Fixed function names in MPPImage Utils

.bazelrc

@@ -98,6 +98,9 @@ build:darwin_arm64 --apple_platform_type=macos
 build:darwin_arm64 --macos_minimum_os=10.16
 build:darwin_arm64 --cpu=darwin_arm64
 
+# Turn off maximum stdout size
+build --experimental_ui_max_stdouterr_bytes=-1
+
 # This bazelrc file is meant to be written by a setup script.
 try-import %workspace%/.configure.bazelrc
 

WORKSPACE

@@ -513,6 +513,9 @@ http_archive(
         "@//third_party:org_tensorflow_system_python.diff",
         # Diff is generated with a script, don't update it manually.
         "@//third_party:org_tensorflow_custom_ops.diff",
+        # Works around Bazel issue with objc_library.
+        # See https://github.com/bazelbuild/bazel/issues/19912
+        "@//third_party:org_tensorflow_objc_build_fixes.diff",
     ],
     patch_args = [
         "-p1",

docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.pbxproj

@@ -0,0 +1,342 @@
+// !$*UTF8*$!
+{
+  archiveVersion = 1;
+  classes = {
+  };
+  objectVersion = 56;
+  objects = {
+
+/* Begin PBXBuildFile section */
+    8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */ = {isa = PBXBuildFile; fileRef = 8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */; settings = {ATTRIBUTES = (Public, ); }; };
+/* End PBXBuildFile section */
+
+/* Begin PBXFileReference section */
+    8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = MediaPipeTasksDocGen.framework; sourceTree = BUILT_PRODUCTS_DIR; };
+    8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MediaPipeTasksDocGen.h; sourceTree = "<group>"; };
+/* End PBXFileReference section */
+
+/* Begin PBXFrameworksBuildPhase section */
+    8566B5562ABABF9A00AAB22A /* Frameworks */ = {
+      isa = PBXFrameworksBuildPhase;
+      buildActionMask = 2147483647;
+      files = (
+      );
+      runOnlyForDeploymentPostprocessing = 0;
+    };
+/* End PBXFrameworksBuildPhase section */
+
+/* Begin PBXGroup section */
+    8566B54F2ABABF9A00AAB22A = {
+      isa = PBXGroup;
+      children = (
+        8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */,
+        8566B55A2ABABF9A00AAB22A /* Products */,
+      );
+      sourceTree = "<group>";
+    };
+    8566B55A2ABABF9A00AAB22A /* Products */ = {
+      isa = PBXGroup;
+      children = (
+        8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */,
+      );
+      name = Products;
+      sourceTree = "<group>";
+    };
+    8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = {
+      isa = PBXGroup;
+      children = (
+        8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */,
+      );
+      path = MediaPipeTasksDocGen;
+      sourceTree = "<group>";
+    };
+/* End PBXGroup section */
+
+/* Begin PBXHeadersBuildPhase section */
+    8566B5542ABABF9A00AAB22A /* Headers */ = {
+      isa = PBXHeadersBuildPhase;
+      buildActionMask = 2147483647;
+      files = (
+        8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */,
+      );
+      runOnlyForDeploymentPostprocessing = 0;
+    };
+/* End PBXHeadersBuildPhase section */
+
+/* Begin PBXNativeTarget section */
+    8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = {
+      isa = PBXNativeTarget;
+      buildConfigurationList = 8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */;
+      buildPhases = (
+        8566B5542ABABF9A00AAB22A /* Headers */,
+        8566B5552ABABF9A00AAB22A /* Sources */,
+        8566B5562ABABF9A00AAB22A /* Frameworks */,
+        8566B5572ABABF9A00AAB22A /* Resources */,
+      );
+      buildRules = (
+      );
+      dependencies = (
+      );
+      name = MediaPipeTasksDocGen;
+      productName = MediaPipeTasksDocGen;
+      productReference = 8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */;
+      productType = "com.apple.product-type.framework";
+    };
+/* End PBXNativeTarget section */
+
+/* Begin PBXProject section */
+    8566B5502ABABF9A00AAB22A /* Project object */ = {
+      isa = PBXProject;
+      attributes = {
+        BuildIndependentTargetsInParallel = 1;
+        LastUpgradeCheck = 1430;
+        TargetAttributes = {
+          8566B5582ABABF9A00AAB22A = {
+            CreatedOnToolsVersion = 14.3.1;
+          };
+        };
+      };
+      buildConfigurationList = 8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */;
+      compatibilityVersion = "Xcode 14.0";
+      developmentRegion = en;
+      hasScannedForEncodings = 0;
+      knownRegions = (
+        en,
+        Base,
+      );
+      mainGroup = 8566B54F2ABABF9A00AAB22A;
+      productRefGroup = 8566B55A2ABABF9A00AAB22A /* Products */;
+      projectDirPath = "";
+      projectRoot = "";
+      targets = (
+        8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */,
+      );
+    };
+/* End PBXProject section */
+
+/* Begin PBXResourcesBuildPhase section */
+    8566B5572ABABF9A00AAB22A /* Resources */ = {
+      isa = PBXResourcesBuildPhase;
+      buildActionMask = 2147483647;
+      files = (
+      );
+      runOnlyForDeploymentPostprocessing = 0;
+    };
+/* End PBXResourcesBuildPhase section */
+
+/* Begin PBXSourcesBuildPhase section */
+    8566B5552ABABF9A00AAB22A /* Sources */ = {
+      isa = PBXSourcesBuildPhase;
+      buildActionMask = 2147483647;
+      files = (
+      );
+      runOnlyForDeploymentPostprocessing = 0;
+    };
+/* End PBXSourcesBuildPhase section */
+
+/* Begin XCBuildConfiguration section */
+    8566B55E2ABABF9A00AAB22A /* Debug */ = {
+      isa = XCBuildConfiguration;
+      buildSettings = {
+        ALWAYS_SEARCH_USER_PATHS = NO;
+        CLANG_ANALYZER_NONNULL = YES;
+        CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+        CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
+        CLANG_ENABLE_MODULES = YES;
+        CLANG_ENABLE_OBJC_ARC = YES;
+        CLANG_ENABLE_OBJC_WEAK = YES;
+        CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+        CLANG_WARN_BOOL_CONVERSION = YES;
+        CLANG_WARN_COMMA = YES;
+        CLANG_WARN_CONSTANT_CONVERSION = YES;
+        CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+        CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+        CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+        CLANG_WARN_EMPTY_BODY = YES;
+        CLANG_WARN_ENUM_CONVERSION = YES;
+        CLANG_WARN_INFINITE_RECURSION = YES;
+        CLANG_WARN_INT_CONVERSION = YES;
+        CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+        CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+        CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+        CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+        CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
+        CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+        CLANG_WARN_STRICT_PROTOTYPES = YES;
+        CLANG_WARN_SUSPICIOUS_MOVE = YES;
+        CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+        CLANG_WARN_UNREACHABLE_CODE = YES;
+        CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+        COPY_PHASE_STRIP = NO;
+        CURRENT_PROJECT_VERSION = 1;
+        DEBUG_INFORMATION_FORMAT = dwarf;
+        ENABLE_STRICT_OBJC_MSGSEND = YES;
+        ENABLE_TESTABILITY = YES;
+        GCC_C_LANGUAGE_STANDARD = gnu11;
+        GCC_DYNAMIC_NO_PIC = NO;
+        GCC_NO_COMMON_BLOCKS = YES;
+        GCC_OPTIMIZATION_LEVEL = 0;
+        GCC_PREPROCESSOR_DEFINITIONS = (
+          "DEBUG=1",
+          "$(inherited)",
+        );
+        GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+        GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+        GCC_WARN_UNDECLARED_SELECTOR = YES;
+        GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+        GCC_WARN_UNUSED_FUNCTION = YES;
+        GCC_WARN_UNUSED_VARIABLE = YES;
+        IPHONEOS_DEPLOYMENT_TARGET = 16.4;
+        MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
+        MTL_FAST_MATH = YES;
+        ONLY_ACTIVE_ARCH = YES;
+        SDKROOT = iphoneos;
+        SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG;
+        SWIFT_OPTIMIZATION_LEVEL = "-Onone";
+        VERSIONING_SYSTEM = "apple-generic";
+        VERSION_INFO_PREFIX = "";
+      };
+      name = Debug;
+    };
+    8566B55F2ABABF9A00AAB22A /* Release */ = {
+      isa = XCBuildConfiguration;
+      buildSettings = {
+        ALWAYS_SEARCH_USER_PATHS = NO;
+        CLANG_ANALYZER_NONNULL = YES;
+        CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE;
+        CLANG_CXX_LANGUAGE_STANDARD = "gnu++20";
+        CLANG_ENABLE_MODULES = YES;
+        CLANG_ENABLE_OBJC_ARC = YES;
+        CLANG_ENABLE_OBJC_WEAK = YES;
+        CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES;
+        CLANG_WARN_BOOL_CONVERSION = YES;
+        CLANG_WARN_COMMA = YES;
+        CLANG_WARN_CONSTANT_CONVERSION = YES;
+        CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES;
+        CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR;
+        CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
+        CLANG_WARN_EMPTY_BODY = YES;
+        CLANG_WARN_ENUM_CONVERSION = YES;
+        CLANG_WARN_INFINITE_RECURSION = YES;
+        CLANG_WARN_INT_CONVERSION = YES;
+        CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
+        CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES;
+        CLANG_WARN_OBJC_LITERAL_CONVERSION = YES;
+        CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR;
+        CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES;
+        CLANG_WARN_RANGE_LOOP_ANALYSIS = YES;
+        CLANG_WARN_STRICT_PROTOTYPES = YES;
+        CLANG_WARN_SUSPICIOUS_MOVE = YES;
+        CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE;
+        CLANG_WARN_UNREACHABLE_CODE = YES;
+        CLANG_WARN__DUPLICATE_METHOD_MATCH = YES;
+        COPY_PHASE_STRIP = NO;
+        CURRENT_PROJECT_VERSION = 1;
+        DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
+        ENABLE_NS_ASSERTIONS = NO;
+        ENABLE_STRICT_OBJC_MSGSEND = YES;
+        GCC_C_LANGUAGE_STANDARD = gnu11;
+        GCC_NO_COMMON_BLOCKS = YES;
+        GCC_WARN_64_TO_32_BIT_CONVERSION = YES;
+        GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR;
+        GCC_WARN_UNDECLARED_SELECTOR = YES;
+        GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE;
+        GCC_WARN_UNUSED_FUNCTION = YES;
+        GCC_WARN_UNUSED_VARIABLE = YES;
+        IPHONEOS_DEPLOYMENT_TARGET = 16.4;
+        MTL_ENABLE_DEBUG_INFO = NO;
+        MTL_FAST_MATH = YES;
+        SDKROOT = iphoneos;
+        SWIFT_COMPILATION_MODE = wholemodule;
+        SWIFT_OPTIMIZATION_LEVEL = "-O";
+        VALIDATE_PRODUCT = YES;
+        VERSIONING_SYSTEM = "apple-generic";
+        VERSION_INFO_PREFIX = "";
+      };
+      name = Release;
+    };
+    8566B5612ABABF9A00AAB22A /* Debug */ = {
+      isa = XCBuildConfiguration;
+      buildSettings = {
+        CODE_SIGN_STYLE = Automatic;
+        CURRENT_PROJECT_VERSION = 1;
+        DEFINES_MODULE = YES;
+        DYLIB_COMPATIBILITY_VERSION = 1;
+        DYLIB_CURRENT_VERSION = 1;
+        DYLIB_INSTALL_NAME_BASE = "@rpath";
+        ENABLE_MODULE_VERIFIER = YES;
+        GENERATE_INFOPLIST_FILE = YES;
+        INFOPLIST_KEY_NSHumanReadableCopyright = "";
+        INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
+        LD_RUNPATH_SEARCH_PATHS = (
+          "$(inherited)",
+          "@executable_path/Frameworks",
+          "@loader_path/Frameworks",
+        );
+        MARKETING_VERSION = 1.0;
+        MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++";
+        MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20";
+        PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen;
+        PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
+        SKIP_INSTALL = YES;
+        SWIFT_EMIT_LOC_STRINGS = YES;
+        SWIFT_VERSION = 5.0;
+        TARGETED_DEVICE_FAMILY = "1,2";
+      };
+      name = Debug;
+    };
+    8566B5622ABABF9A00AAB22A /* Release */ = {
+      isa = XCBuildConfiguration;
+      buildSettings = {
+        CODE_SIGN_STYLE = Automatic;
+        CURRENT_PROJECT_VERSION = 1;
+        DEFINES_MODULE = YES;
+        DYLIB_COMPATIBILITY_VERSION = 1;
+        DYLIB_CURRENT_VERSION = 1;
+        DYLIB_INSTALL_NAME_BASE = "@rpath";
+        ENABLE_MODULE_VERIFIER = YES;
+        GENERATE_INFOPLIST_FILE = YES;
+        INFOPLIST_KEY_NSHumanReadableCopyright = "";
+        INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks";
+        LD_RUNPATH_SEARCH_PATHS = (
+          "$(inherited)",
+          "@executable_path/Frameworks",
+          "@loader_path/Frameworks",
+        );
+        MARKETING_VERSION = 1.0;
+        MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++";
+        MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20";
+        PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen;
+        PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)";
+        SKIP_INSTALL = YES;
+        SWIFT_EMIT_LOC_STRINGS = YES;
+        SWIFT_VERSION = 5.0;
+        TARGETED_DEVICE_FAMILY = "1,2";
+      };
+      name = Release;
+    };
+/* End XCBuildConfiguration section */
+
+/* Begin XCConfigurationList section */
+    8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */ = {
+      isa = XCConfigurationList;
+      buildConfigurations = (
+        8566B55E2ABABF9A00AAB22A /* Debug */,
+        8566B55F2ABABF9A00AAB22A /* Release */,
+      );
+      defaultConfigurationIsVisible = 0;
+      defaultConfigurationName = Release;
+    };
+    8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */ = {
+      isa = XCConfigurationList;
+      buildConfigurations = (
+        8566B5612ABABF9A00AAB22A /* Debug */,
+        8566B5622ABABF9A00AAB22A /* Release */,
+      );
+      defaultConfigurationIsVisible = 0;
+      defaultConfigurationName = Release;
+    };
+/* End XCConfigurationList section */
+  };
+  rootObject = 8566B5502ABABF9A00AAB22A /* Project object */;
+}

docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/contents.xcworkspacedata

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Workspace
+   version = "1.0">
+   <FileRef
+      location = "self:">
+   </FileRef>
+</Workspace>

docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>IDEDidComputeMac32BitWarning</key>
+  <true/>
+</dict>
+</plist>

docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/xcuserdata/macd.xcuserdatad/xcschemes/xcschememanagement.plist

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>SchemeUserState</key>
+  <dict>
+    <key>MediaPipeTasksDocGen.xcscheme_^#shared#^_</key>
+    <dict>
+      <key>orderHint</key>
+      <integer>0</integer>
+    </dict>
+  </dict>
+</dict>
+</plist>

docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen/MediaPipeTasksDocGen.h

@@ -0,0 +1,17 @@
+//
+//  MediaPipeTasksDocGen.h
+//  MediaPipeTasksDocGen
+//
+//  Created by Mark McDonald on 20/9/2023.
+//
+
+#import <Foundation/Foundation.h>
+
+//! Project version number for MediaPipeTasksDocGen.
+FOUNDATION_EXPORT double MediaPipeTasksDocGenVersionNumber;
+
+//! Project version string for MediaPipeTasksDocGen.
+FOUNDATION_EXPORT const unsigned char MediaPipeTasksDocGenVersionString[];
+
+// In this header, you should import all the public headers of your framework using statements like
+// #import <MediaPipeTasksDocGen/PublicHeader.h>

docs/MediaPipeTasksDocGen/Podfile

@@ -0,0 +1,11 @@
+# Uncomment the next line to define a global platform for your project
+platform :ios, '15.0'
+
+target 'MediaPipeTasksDocGen' do
+  # Comment the next line if you don't want to use dynamic frameworks
+  use_frameworks!
+
+  # Pods for MediaPipeTasksDocGen
+  pod 'MediaPipeTasksText'
+  pod 'MediaPipeTasksVision'
+end

docs/MediaPipeTasksDocGen/README.md

@@ -0,0 +1,9 @@
+# MediaPipeTasksDocGen
+
+This empty project is used to generate reference documentation for the
+ObjectiveC and Swift libraries.
+
+Docs are generated using [Jazzy](https://github.com/realm/jazzy) and published
+to [the developer site](https://developers.google.com/mediapipe/solutions/).
+
+To bump the API version used, edit [`Podfile`](./Podfile).

mediapipe/calculators/audio/spectrogram_calculator.proto

@@ -80,7 +80,7 @@ message SpectrogramCalculatorOptions {
   // If use_local_timestamp is true, the output packet's timestamp is based on
   // the last sample of the packet and it's inferred from the latest input
   // packet's timestamp.  If false, the output packet's timestamp is based on
-  // the cumulative timestamping, which is inferred from the intial input
+  // the cumulative timestamping, which is inferred from the initial input
   // timestamp and the cumulative number of samples.
   optional bool use_local_timestamp = 8 [default = false];
 }

mediapipe/calculators/audio/time_series_framer_calculator.proto

@@ -66,7 +66,7 @@ message TimeSeriesFramerCalculatorOptions {
   // If use_local_timestamp is true, the output packet's timestamp is based on
   // the last sample of the packet and it's inferred from the latest input
   // packet's timestamp.  If false, the output packet's timestamp is based on
-  // the cumulative timestamping, which is inferred from the intial input
+  // the cumulative timestamping, which is inferred from the initial input
   // timestamp and the cumulative number of samples.
   optional bool use_local_timestamp = 6 [default = false];
 }

mediapipe/calculators/core/BUILD

@@ -727,6 +727,7 @@ cc_library(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/status",
     ],
     alwayslink = 1,
 )
@@ -742,6 +743,7 @@ cc_test(
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/tool:options_util",
+        "//mediapipe/util:packet_test_util",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],

mediapipe/calculators/core/packet_sequencer_calculator_test.cc

@@ -71,7 +71,7 @@ TEST_F(PacketSequencerCalculatorTest, IsRegistered) {
       CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator"));
 }
 
-// Shows how control packets recieve timestamps before and after frame packets
+// Shows how control packets receive timestamps before and after frame packets
 // have arrived.
 TEST_F(PacketSequencerCalculatorTest, ChannelEarly) {
   CalculatorGraphConfig::Node node_config = BuildNodeConfig();

mediapipe/calculators/core/side_packet_to_stream_calculator.cc

@@ -17,6 +17,7 @@
 #include <set>
 #include <string>
 
+#include "absl/status/status.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/ret_check.h"
@@ -32,6 +33,7 @@ namespace {
 constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
 constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
 constexpr char kTagAtZero[] = "AT_ZERO";
+constexpr char kTagAtFirstTick[] = "AT_FIRST_TICK";
 constexpr char kTagAtTick[] = "AT_TICK";
 constexpr char kTagTick[] = "TICK";
 constexpr char kTagAtTimestamp[] = "AT_TIMESTAMP";
@@ -43,6 +45,7 @@ static std::map<std::string, Timestamp>* kTimestampMap = []() {
   res->emplace(kTagAtPostStream, Timestamp::PostStream());
   res->emplace(kTagAtZero, Timestamp(0));
   res->emplace(kTagAtTick, Timestamp::Unset());
+  res->emplace(kTagAtFirstTick, Timestamp::Unset());
   res->emplace(kTagAtTimestamp, Timestamp::Unset());
   return res;
 }();
@@ -59,8 +62,8 @@ std::string GetOutputTag(const CC& cc) {
 // timestamp, depending on the tag used to define output stream(s). (One tag can
 // be used only.)
 //
-// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_TIMESTAMP
+// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_FIRST_TICK,
-// and corresponding timestamps are Timestamp::PreStream(),
+// AT_TIMESTAMP and corresponding timestamps are Timestamp::PreStream(),
 // Timestamp::PostStream(), Timestamp(0), timestamp of a packet received in TICK
 // input, and timestamp received from a side input.
 //
@@ -96,6 +99,7 @@ class SidePacketToStreamCalculator : public CalculatorBase {
 
  private:
   bool is_tick_processing_ = false;
+  bool close_on_first_tick_ = false;
   std::string output_tag_;
 };
 REGISTER_CALCULATOR(SidePacketToStreamCalculator);
@@ -103,13 +107,16 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator);
 absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
   const auto& tags = cc->Outputs().GetTags();
   RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
-      << "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
+      << "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, "
-         "AT_TIMESTAMP tags is allowed and required to specify output "
+         "AT_FIRST_TICK and AT_TIMESTAMP tags is allowed and required to "
-         "stream(s).";
+         "specify output stream(s).";
-  RET_CHECK(
+  const bool has_tick_output =
-      (cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) ||
+      cc->Outputs().HasTag(kTagAtTick) || cc->Outputs().HasTag(kTagAtFirstTick);
-      (!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick)))
+  const bool has_tick_input = cc->Inputs().HasTag(kTagTick);
-      << "Either both of TICK and AT_TICK should be used or none of them.";
+  RET_CHECK((has_tick_output && has_tick_input) ||
+            (!has_tick_output && !has_tick_input))
+      << "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
+         "should be used or none of them.";
   RET_CHECK((cc->Outputs().HasTag(kTagAtTimestamp) &&
              cc->InputSidePackets().HasTag(kTagSideInputTimestamp)) ||
             (!cc->Outputs().HasTag(kTagAtTimestamp) &&
@@ -148,11 +155,17 @@ absl::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
     // timestamp bound update.
     cc->SetOffset(TimestampDiff(0));
   }
+  if (output_tag_ == kTagAtFirstTick) {
+    close_on_first_tick_ = true;
+  }
   return absl::OkStatus();
 }
 
 absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
   if (is_tick_processing_) {
+    if (cc->Outputs().Get(output_tag_, 0).IsClosed()) {
+      return absl::OkStatus();
+    }
     // TICK input is guaranteed to be non-empty, as it's the only input stream
     // for this calculator.
     const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
@@ -160,6 +173,9 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
       cc->Outputs()
           .Get(output_tag_, i)
           .AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
+      if (close_on_first_tick_) {
+        cc->Outputs().Get(output_tag_, i).Close();
+      }
     }
 
     return absl::OkStatus();
@@ -170,6 +186,7 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
 
 absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
   if (!cc->Outputs().HasTag(kTagAtTick) &&
+      !cc->Outputs().HasTag(kTagAtFirstTick) &&
       !cc->Outputs().HasTag(kTagAtTimestamp)) {
     const auto& timestamp = kTimestampMap->at(output_tag_);
     for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {

mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc

@@ -27,13 +27,17 @@
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/status_matchers.h"
 #include "mediapipe/framework/tool/options_util.h"
+#include "mediapipe/util/packet_test_util.h"
 
 namespace mediapipe {
 namespace {
 
-using testing::HasSubstr;
+using ::testing::ElementsAre;
+using ::testing::Eq;
+using ::testing::HasSubstr;
+using ::testing::IsEmpty;
 
-TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
+TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTick) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
           R"pb(
@@ -52,10 +56,35 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
   EXPECT_THAT(
       status.message(),
       HasSubstr(
-          "Either both of TICK and AT_TICK should be used or none of them."));
+          "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
+          "should be used or none of them."));
 }
 
-TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
+TEST(SidePacketToStreamCalculator,
+     WrongConfigWithMissingTickForFirstTickProcessing) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"pb(
+            input_stream: "tick"
+            input_side_packet: "side_packet"
+            output_stream: "packet"
+            node {
+              calculator: "SidePacketToStreamCalculator"
+              input_side_packet: "side_packet"
+              output_stream: "AT_FIRST_TICK:packet"
+            }
+          )pb");
+  CalculatorGraph graph;
+  auto status = graph.Initialize(graph_config);
+  EXPECT_FALSE(status.ok());
+  EXPECT_THAT(
+      status.message(),
+      HasSubstr(
+          "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
+          "should be used or none of them."));
+}
+
+TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTimestampSideInput) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
           R"pb(
@@ -76,7 +105,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
                         "or none of them."));
 }
 
-TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
+TEST(SidePacketToStreamCalculator, WrongConfigWithNonExistentTag) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
           R"pb(
@@ -92,14 +121,13 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
-  EXPECT_THAT(
+  EXPECT_THAT(status.message(),
-      status.message(),
+              HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
-      HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
+                        "AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
-                "AT_TIMESTAMP tags is allowed and required to specify output "
+                        "allowed and required to specify output stream(s)."));
-                "stream(s)."));
 }
 
-TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
+TEST(SidePacketToStreamCalculator, WrongConfigWithMixedTags) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
           R"pb(
@@ -117,14 +145,13 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
   CalculatorGraph graph;
   auto status = graph.Initialize(graph_config);
   EXPECT_FALSE(status.ok());
-  EXPECT_THAT(
+  EXPECT_THAT(status.message(),
-      status.message(),
+              HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
-      HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
+                        "AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
-                "AT_TIMESTAMP tags is allowed and required to specify output "
+                        "allowed and required to specify output stream(s)."));
-                "stream(s)."));
 }
 
-TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
+TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughSidePackets) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
           R"pb(
@@ -146,7 +173,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
           "Same number of input side packets and output streams is required."));
 }
 
-TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
+TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughOutputStreams) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
           R"pb(
@@ -248,7 +275,50 @@ TEST(SidePacketToStreamCalculator, AtTick) {
   tick_and_verify(/*at_timestamp=*/1025);
 }
 
-TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
+TEST(SidePacketToStreamCalculator, AtFirstTick) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"pb(
+            input_stream: "tick"
+            input_side_packet: "side_packet"
+            output_stream: "packet"
+            node {
+              calculator: "SidePacketToStreamCalculator"
+              input_stream: "TICK:tick"
+              input_side_packet: "side_packet"
+              output_stream: "AT_FIRST_TICK:packet"
+            }
+          )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("packet", &graph_config, &output_packets);
+  CalculatorGraph graph;
+
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  const int expected_value = 20;
+  const Timestamp kTestTimestamp(1234);
+  MP_ASSERT_OK(
+      graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
+
+  auto insert_tick = [&graph](Timestamp at_timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
+    MP_ASSERT_OK(graph.WaitUntilIdle());
+  };
+
+  insert_tick(kTestTimestamp);
+
+  EXPECT_THAT(output_packets,
+              ElementsAre(PacketContainsTimestampAndPayload<int>(
+                  Eq(kTestTimestamp), Eq(expected_value))));
+
+  output_packets.clear();
+
+  // Should not result in an additional output.
+  insert_tick(kTestTimestamp + 1);
+  EXPECT_THAT(output_packets, IsEmpty());
+}
+
+TEST(SidePacketToStreamCalculator, AtTickWithMultipleSidePackets) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
           R"pb(
@@ -302,6 +372,62 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
   tick_and_verify(/*at_timestamp=*/1025);
 }
 
+TEST(SidePacketToStreamCalculator, AtFirstTickWithMultipleSidePackets) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"pb(
+            input_stream: "tick"
+            input_side_packet: "side_packet0"
+            input_side_packet: "side_packet1"
+            output_stream: "packet0"
+            output_stream: "packet1"
+            node {
+              calculator: "SidePacketToStreamCalculator"
+              input_stream: "TICK:tick"
+              input_side_packet: "side_packet0"
+              input_side_packet: "side_packet1"
+              output_stream: "AT_FIRST_TICK:0:packet0"
+              output_stream: "AT_FIRST_TICK:1:packet1"
+            }
+          )pb");
+  std::vector<Packet> output_packets0;
+  tool::AddVectorSink("packet0", &graph_config, &output_packets0);
+  std::vector<Packet> output_packets1;
+  tool::AddVectorSink("packet1", &graph_config, &output_packets1);
+  CalculatorGraph graph;
+
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  const int expected_value0 = 20;
+  const int expected_value1 = 128;
+  const Timestamp kTestTimestamp(1234);
+  MP_ASSERT_OK(
+      graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
+                      {"side_packet1", MakePacket<int>(expected_value1)}}));
+
+  auto insert_tick = [&graph](Timestamp at_timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
+    MP_ASSERT_OK(graph.WaitUntilIdle());
+  };
+
+  insert_tick(kTestTimestamp);
+
+  EXPECT_THAT(output_packets0,
+              ElementsAre(PacketContainsTimestampAndPayload<int>(
+                  Eq(kTestTimestamp), Eq(expected_value0))));
+  EXPECT_THAT(output_packets1,
+              ElementsAre(PacketContainsTimestampAndPayload<int>(
+                  Eq(kTestTimestamp), Eq(expected_value1))));
+
+  output_packets0.clear();
+  output_packets1.clear();
+
+  // Should not result in an additional output.
+  insert_tick(kTestTimestamp + 1);
+  EXPECT_THAT(output_packets0, IsEmpty());
+  EXPECT_THAT(output_packets1, IsEmpty());
+}
+
 TEST(SidePacketToStreamCalculator, AtTimestamp) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
@@ -334,7 +460,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
   EXPECT_EQ(expected_value, output_packets.back().Get<int>());
 }
 
-TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
+TEST(SidePacketToStreamCalculator, AtTimestampWithMultipleOutputs) {
   CalculatorGraphConfig graph_config =
       ParseTextProtoOrDie<CalculatorGraphConfig>(
           R"pb(

mediapipe/calculators/core/value_or_default_calculator_test.cc

@@ -174,7 +174,7 @@ TEST(ValueOrDefaultCalculatorTest, DefaultAndValues) {
       ElementsAre(kDefaultValue, 1, 2, kDefaultValue, 3, kDefaultValue));
 }
 
-TEST(ValueOrDefaultCalculatorTest, TimestampsMissmatch) {
+TEST(ValueOrDefaultCalculatorTest, TimestampsMismatch) {
   // Check that when we provide the inputs not on time - we don't get them.
   ValueOrDefaultRunner runner;
   const std::vector<int64_t> ticks = {1, 2, 5, 8, 12, 33, 231};

mediapipe/calculators/image/affine_transformation_runner_opencv.cc

@@ -59,7 +59,7 @@ class OpenCvRunner
       const ImageFrame& input, const std::array<float, 16>& matrix,
       const AffineTransformation::Size& size,
       AffineTransformation::BorderMode border_mode) override {
-    // OpenCV warpAffine works in absolute coordinates, so the transfom (which
+    // OpenCV warpAffine works in absolute coordinates, so the transform (which
     // accepts and produces relative coordinates) should be adjusted to first
     // normalize coordinates and then scale them.
     // clang-format off

mediapipe/calculators/image/image_clone_calculator.cc

@@ -65,7 +65,7 @@ class ImageCloneCalculator : public Node {
     }
 #else
     MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
-        cc, /*requesst_gpu_as_optional=*/true));
+        cc, /*request_gpu_as_optional=*/true));
 #endif  // MEDIAPIPE_DISABLE_GPU
     return absl::OkStatus();
   }

mediapipe/calculators/image/image_cropping_calculator.proto

@@ -24,7 +24,7 @@ message ImageCroppingCalculatorOptions {
   }
 
   // Output texture buffer dimensions. The values defined in the options will be
-  // overriden by the WIDTH and HEIGHT input streams if they exist.
+  // overridden by the WIDTH and HEIGHT input streams if they exist.
   optional int32 width = 1;
   optional int32 height = 2;
 

mediapipe/calculators/image/image_file_properties_calculator.cc

@@ -77,7 +77,7 @@ absl::StatusOr<double> ComputeFocalLengthInPixels(int image_width,
   return focal_length_pixels;
 }
 
-absl::StatusOr<ImageFileProperties> GetImageFileProperites(
+absl::StatusOr<ImageFileProperties> GetImageFileProperties(
     const std::string& image_bytes) {
   easyexif::EXIFInfo result;
   int code = result.parseFrom(image_bytes);
@@ -151,7 +151,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
     if (cc->InputSidePackets().NumEntries() == 1) {
       const std::string& image_bytes =
           cc->InputSidePackets().Index(0).Get<std::string>();
-      MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
+      MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
       read_properties_ = true;
     }
 
@@ -169,7 +169,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
         return absl::OkStatus();
       }
       const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
-      MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
+      MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
       read_properties_ = true;
     }
     if (read_properties_) {

mediapipe/calculators/image/segmentation_smoothing_calculator.cc

@@ -118,7 +118,7 @@ absl::Status SegmentationSmoothingCalculator::GetContract(
 
 #if !MEDIAPIPE_DISABLE_GPU
   MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
-      cc, /*requesst_gpu_as_optional=*/true));
+      cc, /*request_gpu_as_optional=*/true));
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
   return absl::OkStatus();

mediapipe/calculators/image/warp_affine_calculator.cc

@@ -206,7 +206,7 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
     if constexpr (std::is_same_v<InterfaceT, WarpAffineCalculatorGpu> ||
                   std::is_same_v<InterfaceT, WarpAffineCalculator>) {
       MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
-          cc, /*requesst_gpu_as_optional=*/true));
+          cc, /*request_gpu_as_optional=*/true));
     }
     return absl::OkStatus();
   }

mediapipe/calculators/image/warp_affine_calculator_test.cc

@@ -284,7 +284,7 @@ std::array<float, 16> GetMatrix(cv::Mat input, mediapipe::NormalizedRect roi,
       .IgnoreError();
   mediapipe::GetRotatedSubRectToRectTransformMatrix(
       roi_absolute, input.cols, input.rows,
-      /*flip_horizontaly=*/false, &transform_mat);
+      /*flip_horizontally=*/false, &transform_mat);
   return transform_mat;
 }
 

mediapipe/calculators/image/yuv_to_image_calculator.cc

@@ -49,7 +49,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
 // The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka
 // YV21) format (as per the `fourcc()` property). This covers the most commonly
 // used YUV image formats used on mobile devices. Other formats are not
-// supported and wil result in an `InvalidArgumentError`.
+// supported and will result in an `InvalidArgumentError`.
 class YUVToImageCalculator : public Node {
  public:
   static constexpr Input<YUVImage> kInput{"YUV_IMAGE"};

mediapipe/calculators/tensor/BUILD

@@ -657,6 +657,7 @@ cc_library(
     }),
     deps = [
         ":tensor_converter_calculator_cc_proto",
+        ":tensor_converter_cpu",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:port",
         "//mediapipe/framework/formats:image_frame",
@@ -665,6 +666,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:statusor",
+        "//mediapipe/gpu:gpu_buffer",
         "//mediapipe/gpu:gpu_buffer_format",
         "//mediapipe/gpu:gpu_origin_cc_proto",
         "//mediapipe/util:resource_util",
@@ -674,10 +676,17 @@ cc_library(
         "@com_google_absl//absl/log:check",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",
+        "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
     ] + select({
         "//mediapipe/gpu:disable_gpu": [],
-        "//conditions:default": ["tensor_converter_calculator_gpu_deps"],
+        "//conditions:default": [
+            "tensor_converter_calculator_gpu_deps",
+            "//mediapipe/gpu:gl_base",
+            "//mediapipe/gpu:gl_calculator_helper",
+            "//mediapipe/gpu:gl_simple_shaders",
+            "//mediapipe/gpu:shader_util",
+        ],
     }) + select({
         "//mediapipe:apple": [
             "//third_party/apple_frameworks:MetalKit",
@@ -687,6 +696,35 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "tensor_converter_cpu",
+    srcs = ["tensor_converter_cpu.cc"],
+    hdrs = ["tensor_converter_cpu.h"],
+    deps = [
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+    ],
+)
+
+cc_test(
+    name = "tensor_converter_cpu_test",
+    srcs = ["tensor_converter_cpu_test.cc"],
+    deps = [
+        ":tensor_converter_cpu",
+        "//mediapipe/framework/formats:matrix",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:gtest",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status_matchers",
+        "//mediapipe/util:image_test_utils",
+    ],
+)
+
 cc_library(
     name = "tensor_converter_calculator_gpu_deps",
     visibility = ["//visibility:private"],
@@ -1414,6 +1452,8 @@ cc_library(
     }),
     deps = [
         ":tensors_to_segmentation_calculator_cc_proto",
+        ":tensors_to_segmentation_converter",
+        ":tensors_to_segmentation_utils",
         "//mediapipe/framework:calculator_context",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:port",
@@ -1421,9 +1461,11 @@ cc_library(
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/formats:tensor",
         "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:statusor",
         "//mediapipe/gpu:gpu_origin_cc_proto",
         "//mediapipe/util:resource_util",
+        "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/types:span",
@@ -1434,6 +1476,7 @@ cc_library(
             "//mediapipe/gpu:gl_calculator_helper",
             "//mediapipe/gpu:gl_simple_shaders",
             "//mediapipe/gpu:gpu_buffer",
+            "//mediapipe/gpu:gpu_buffer_format",
             "//mediapipe/gpu:shader_util",
         ],
     }) + selects.with_or({
@@ -1453,19 +1496,96 @@ cc_library(
     }) + select({
         "//mediapipe/framework/port:disable_opencv": [],
         "//conditions:default": [
-            "//mediapipe/framework/formats:image_opencv",
+            ":tensors_to_segmentation_converter_opencv",
-            "//mediapipe/framework/port:opencv_imgproc",
         ],
     }),
     alwayslink = 1,
 )
 
+cc_library(
+    name = "tensors_to_segmentation_utils",
+    srcs = ["tensors_to_segmentation_utils.cc"],
+    hdrs = ["tensors_to_segmentation_utils.h"],
+    deps = [
+        "//mediapipe/framework:port",
+        "//mediapipe/framework/port:ret_check",
+        "@com_google_absl//absl/status:statusor",
+    ],
+)
+
+cc_test(
+    name = "tensors_to_segmentation_utils_test",
+    srcs = ["tensors_to_segmentation_utils_test.cc"],
+    deps = [
+        ":tensors_to_segmentation_utils",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:status_matchers",
+        "@com_google_absl//absl/status:statusor",
+    ],
+)
+
+cc_library(
+    name = "tensors_to_segmentation_converter",
+    hdrs = ["tensors_to_segmentation_converter.h"],
+    deps = [
+        "//mediapipe/framework/formats:image",
+        "//mediapipe/framework/formats:tensor",
+        "@com_google_absl//absl/status:statusor",
+    ],
+)
+
+cc_library(
+    name = "tensors_to_segmentation_converter_opencv",
+    srcs = ["tensors_to_segmentation_converter_opencv.cc"],
+    hdrs = ["tensors_to_segmentation_converter_opencv.h"],
+    deps = [
+        ":tensors_to_segmentation_calculator_cc_proto",
+        ":tensors_to_segmentation_converter",
+        ":tensors_to_segmentation_utils",
+        "//mediapipe/framework/formats:image",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_opencv",
+        "//mediapipe/framework/formats:tensor",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/status:statusor",
+    ],
+)
+
+cc_library(
+    name = "tensors_to_segmentation_calculator_test_utils",
+    testonly = 1,
+    srcs = ["tensors_to_segmentation_calculator_test_utils.cc"],
+    hdrs = ["tensors_to_segmentation_calculator_test_utils.h"],
+    deps = [
+        ":tensors_to_segmentation_calculator_cc_proto",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/log:absl_log",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+cc_test(
+    name = "tensors_to_segmentation_calculator_test_utils_test",
+    srcs = ["tensors_to_segmentation_calculator_test_utils_test.cc"],
+    deps = [
+        ":tensors_to_segmentation_calculator_cc_proto",
+        ":tensors_to_segmentation_calculator_test_utils",
+        "//mediapipe/framework/port:gtest_main",
+    ],
+)
+
 cc_test(
     name = "tensors_to_segmentation_calculator_test",
     srcs = ["tensors_to_segmentation_calculator_test.cc"],
     deps = [
         ":tensors_to_segmentation_calculator",
         ":tensors_to_segmentation_calculator_cc_proto",
+        ":tensors_to_segmentation_calculator_test_utils",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework:packet",
@@ -1476,11 +1596,6 @@ cc_test(
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/formats:tensor",
         "//mediapipe/framework/port:gtest_main",
-        "//mediapipe/framework/port:parse_text_proto",
-        "@com_google_absl//absl/log",
-        "@com_google_absl//absl/log:absl_log",
-        "@com_google_absl//absl/strings",
-        "@com_google_googletest//:gtest_main",
     ],
 )
 

mediapipe/calculators/tensor/audio_to_tensor_calculator.cc

@@ -109,7 +109,7 @@ bool IsValidFftSize(int size) {
 //   Non-streaming mode: when "stream_mode" is set to false in the calculator
 //     options, the calculators treats the packets in the input audio stream as
 //     a batch of unrelated audio buffers. In each Process() call, the input
-//     buffer will be frist resampled, and framed as fixed-sized, possibly
+//     buffer will be first resampled, and framed as fixed-sized, possibly
 //     overlapping tensors. The last tensor produced by a Process() invocation
 //     will be zero-padding if the remaining samples are insufficient. As the
 //     calculator treats the input packets as unrelated, all samples will be
@@ -159,7 +159,7 @@ class AudioToTensorCalculator : public Node {
  public:
   static constexpr Input<Matrix> kAudioIn{"AUDIO"};
   // TODO: Removes this optional input stream when the "AUDIO" stream
-  // uses the new mediapipe audio data containers that carry audio metatdata,
+  // uses the new mediapipe audio data containers that carry audio metadata,
   // such as sample rate.
   static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"};
   static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};

mediapipe/calculators/tensor/audio_to_tensor_calculator.proto

@@ -37,7 +37,7 @@ message AudioToTensorCalculatorOptions {
   // will be converted into tensors.
   optional double target_sample_rate = 4;
 
-  // Whether to treat the input audio stream as a continous stream or a batch
+  // Whether to treat the input audio stream as a continuous stream or a batch
   // of unrelated audio buffers.
   optional bool stream_mode = 5 [default = true];
 

mediapipe/calculators/tensor/image_to_tensor_calculator.cc

@@ -82,7 +82,7 @@ namespace api2 {
 //
 // Outputs:
 //   TENSORS - std::vector<Tensor>
-//     Vector containing a single Tensor populated with an extrated RGB image.
+//     Vector containing a single Tensor populated with an extracted RGB image.
 //   MATRIX - std::array<float, 16> @Optional
 //     An std::array<float, 16> representing a 4x4 row-major-order matrix that
 //     maps a point on the input image to a point on the output tensor, and
@@ -212,7 +212,7 @@ class ImageToTensorCalculator : public Node {
       std::array<float, 16> matrix;
       GetRotatedSubRectToRectTransformMatrix(
           roi, image->width(), image->height(),
-          /*flip_horizontaly=*/false, &matrix);
+          /*flip_horizontally=*/false, &matrix);
       kOutMatrix(cc).Send(std::move(matrix));
     }
 

mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc

@@ -206,7 +206,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
   } else if (image_channels == 1) {
     return ImageFormat::GRAY8;
   }
-  ABSL_CHECK(false) << "Unsupported input image channles: " << image_channels;
+  ABSL_CHECK(false) << "Unsupported input image channels: " << image_channels;
 }
 
 Packet MakeImageFramePacket(cv::Mat input) {

mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc

@@ -57,7 +57,7 @@ class SubRectExtractorGl {
   absl::Status ExtractSubRectToBuffer(
       const tflite::gpu::gl::GlTexture& texture,
       const tflite::gpu::HW& texture_size, const RotatedRect& sub_rect,
-      bool flip_horizontaly, float alpha, float beta,
+      bool flip_horizontally, float alpha, float beta,
       const tflite::gpu::HW& destination_size,
       tflite::gpu::gl::CommandQueue* command_queue,
       tflite::gpu::gl::GlBuffer* destination);
@@ -154,13 +154,13 @@ void main() {
 absl::Status SubRectExtractorGl::ExtractSubRectToBuffer(
     const tflite::gpu::gl::GlTexture& texture,
     const tflite::gpu::HW& texture_size, const RotatedRect& texture_sub_rect,
-    bool flip_horizontaly, float alpha, float beta,
+    bool flip_horizontally, float alpha, float beta,
     const tflite::gpu::HW& destination_size,
     tflite::gpu::gl::CommandQueue* command_queue,
     tflite::gpu::gl::GlBuffer* destination) {
   std::array<float, 16> transform_mat;
   GetRotatedSubRectToRectTransformMatrix(texture_sub_rect, texture_size.w,
-                                         texture_size.h, flip_horizontaly,
+                                         texture_size.h, flip_horizontally,
                                          &transform_mat);
   MP_RETURN_IF_ERROR(texture.BindAsSampler2D(0));
 
@@ -308,7 +308,7 @@ class GlProcessor : public ImageToTensorConverter {
               input_texture,
               tflite::gpu::HW(source_texture.height(), source_texture.width()),
               roi,
-              /*flip_horizontaly=*/false, transform.scale, transform.offset,
+              /*flip_horizontally=*/false, transform.scale, transform.offset,
               tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
               command_queue_.get(), &output));
 

mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc

@@ -199,7 +199,7 @@ class GlProcessor : public ImageToTensorConverter {
                                   range_min, range_max));
           auto tensor_view = output_tensor.GetOpenGlTexture2dWriteView();
           MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi,
-                                            /*flip_horizontaly=*/false,
+                                            /*flip_horizontally=*/false,
                                             transform.scale, transform.offset,
                                             output_shape, &tensor_view));
           return absl::OkStatus();
@@ -210,7 +210,7 @@ class GlProcessor : public ImageToTensorConverter {
 
   absl::Status ExtractSubRect(const mediapipe::GlTexture& texture,
                               const RotatedRect& sub_rect,
-                              bool flip_horizontaly, float alpha, float beta,
+                              bool flip_horizontally, float alpha, float beta,
                               const Tensor::Shape& output_shape,
                               Tensor::OpenGlTexture2dView* output) {
     const int output_height = output_shape.dims[1];
@@ -263,13 +263,13 @@ class GlProcessor : public ImageToTensorConverter {
     ABSL_LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread.";
     if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) {
       GetTransposedRotatedSubRectToRectTransformMatrix(
-          sub_rect, texture.width(), texture.height(), flip_horizontaly,
+          sub_rect, texture.width(), texture.height(), flip_horizontally,
           &transform_mat);
       glUniformMatrix4fv(matrix_id_, 1, GL_FALSE, transform_mat.data());
     } else {
       GetRotatedSubRectToRectTransformMatrix(sub_rect, texture.width(),
-                                             texture.height(), flip_horizontaly,
+                                             texture.height(),
-                                             &transform_mat);
+                                             flip_horizontally, &transform_mat);
       glUniformMatrix4fv(matrix_id_, 1, GL_TRUE, transform_mat.data());
     }
 

mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc

@@ -179,13 +179,13 @@ class SubRectExtractorMetal {
   }
 
   absl::Status Execute(id<MTLTexture> input_texture,
-                       const RotatedRect& sub_rect, bool flip_horizontaly,
+                       const RotatedRect& sub_rect, bool flip_horizontally,
                        float alpha, float beta,
                        const tflite::gpu::HW& destination_size,
                        id<MTLCommandBuffer> command_buffer,
                        id<MTLBuffer> destination) {
     auto output_texture = MTLTextureWithBuffer(destination_size, destination);
-    return InternalExecute(input_texture, sub_rect, flip_horizontaly, alpha,
+    return InternalExecute(input_texture, sub_rect, flip_horizontally, alpha,
                            beta, destination_size, command_buffer,
                            output_texture);
   }
@@ -211,7 +211,7 @@ class SubRectExtractorMetal {
 
   absl::Status InternalExecute(id<MTLTexture> input_texture,
                                const RotatedRect& sub_rect,
-                               bool flip_horizontaly, float alpha, float beta,
+                               bool flip_horizontally, float alpha, float beta,
                                const tflite::gpu::HW& destination_size,
                                id<MTLCommandBuffer> command_buffer,
                                id<MTLTexture> output_texture) {
@@ -223,7 +223,7 @@ class SubRectExtractorMetal {
     std::array<float, 16> transform_mat;
     GetRotatedSubRectToRectTransformMatrix(sub_rect, input_texture.width,
                                            input_texture.height,
-                                           flip_horizontaly, &transform_mat);
+                                           flip_horizontally, &transform_mat);
     id<MTLBuffer> transform_mat_buffer =
         [device_ newBufferWithBytes:&transform_mat
                              length:sizeof(transform_mat)
@@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter {
           MtlBufferView::GetWriteView(output_tensor, command_buffer);
       MP_RETURN_IF_ERROR(extractor_->Execute(
           texture, roi,
-          /*flip_horizontaly=*/false, transform.scale, transform.offset,
+          /*flip_horizontally=*/false, transform.scale, transform.offset,
           tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]),
           command_buffer, buffer_view.buffer()));
       [command_buffer commit];

mediapipe/calculators/tensor/image_to_tensor_utils.cc

@@ -92,7 +92,7 @@ absl::StatusOr<ValueTransformation> GetValueRangeTransformation(
 
 void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
                                             int rect_width, int rect_height,
-                                            bool flip_horizontaly,
+                                            bool flip_horizontally,
                                             std::array<float, 16>* matrix_ptr) {
   std::array<float, 16>& matrix = *matrix_ptr;
   // The resulting matrix is multiplication of below commented out matrices:
@@ -118,7 +118,7 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
   // {0.0f, 0.0f,    a, 0.0f}
   // {0.0f, 0.0f, 0.0f, 1.0f}
 
-  const float flip = flip_horizontaly ? -1 : 1;
+  const float flip = flip_horizontally ? -1 : 1;
   // Matrix for optional horizontal flip around middle of output image.
   // { fl  , 0.0f, 0.0f, 0.0f}
   // { 0.0f, 1.0f, 0.0f, 0.0f}
@@ -177,13 +177,13 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
 
 void GetTransposedRotatedSubRectToRectTransformMatrix(
     const RotatedRect& sub_rect, int rect_width, int rect_height,
-    bool flip_horizontaly, std::array<float, 16>* matrix_ptr) {
+    bool flip_horizontally, std::array<float, 16>* matrix_ptr) {
   std::array<float, 16>& matrix = *matrix_ptr;
   // See comments in GetRotatedSubRectToRectTransformMatrix for detailed
   // calculations.
   const float a = sub_rect.width;
   const float b = sub_rect.height;
-  const float flip = flip_horizontaly ? -1 : 1;
+  const float flip = flip_horizontally ? -1 : 1;
   const float c = std::cos(sub_rect.rotation);
   const float d = std::sin(sub_rect.rotation);
   const float e = sub_rect.center_x;

mediapipe/calculators/tensor/image_to_tensor_utils.h

@@ -74,7 +74,7 @@ absl::StatusOr<std::array<float, 4>> PadRoi(int input_tensor_width,
 // Represents a transformation of value which involves scaling and offsetting.
 // To apply transformation:
 // ValueTransformation transform = ...
-// float transformed_value = transform.scale * value + transfrom.offset;
+// float transformed_value = transform.scale * value + transform.offset;
 struct ValueTransformation {
   float scale;
   float offset;
@@ -99,11 +99,11 @@ absl::StatusOr<ValueTransformation> GetValueRangeTransformation(
 // @sub_rect - rotated sub rect in absolute coordinates
 // @rect_width - rect width
 // @rect_height - rect height
-// @flip_horizontaly - we need to flip the output buffer.
+// @flip_horizontally - we need to flip the output buffer.
 // @matrix - 4x4 matrix (array of 16 elements) to populate
 void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
                                             int rect_width, int rect_height,
-                                            bool flip_horizontaly,
+                                            bool flip_horizontally,
                                             std::array<float, 16>* matrix);
 
 // Returns the transpose of the matrix found with
@@ -118,11 +118,11 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect,
 // @sub_rect - rotated sub rect in absolute coordinates
 // @rect_width - rect width
 // @rect_height - rect height
-// @flip_horizontaly - we need to flip the output buffer.
+// @flip_horizontally - we need to flip the output buffer.
 // @matrix - 4x4 matrix (array of 16 elements) to populate
 void GetTransposedRotatedSubRectToRectTransformMatrix(
     const RotatedRect& sub_rect, int rect_width, int rect_height,
-    bool flip_horizontaly, std::array<float, 16>* matrix);
+    bool flip_horizontally, std::array<float, 16>* matrix);
 
 // Validates the output dimensions set in the option proto. The input option
 // proto is expected to have to following fields:

mediapipe/calculators/tensor/tensor_converter_calculator.cc

@@ -14,6 +14,7 @@
 
 #include <cstdint>
 #include <string>
+#include <utility>
 #include <vector>
 
 #include "absl/log/absl_check.h"
@@ -21,17 +22,22 @@
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/str_format.h"
+#include "absl/strings/substitute.h"
+#include "absl/types/optional.h"
 #include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h"
+#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port.h"
 #include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status_macros.h"
 #include "mediapipe/gpu/gpu_buffer_format.h"
 #include "mediapipe/gpu/gpu_origin.pb.h"
 
 #if !MEDIAPIPE_DISABLE_GPU
+#include "mediapipe/gpu/gl_base.h"
 #include "mediapipe/gpu/gpu_buffer.h"
 #if MEDIAPIPE_METAL_ENABLED
 #import <CoreVideo/CoreVideo.h>
@@ -94,16 +100,13 @@ absl::StatusOr<bool> ShouldFlipVertically(
   }
 }
 
-typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
-    RowMajorMatrixXf;
-typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
-    ColMajorMatrixXf;
-
 constexpr char kImageFrameTag[] = "IMAGE";
 constexpr char kGpuBufferTag[] = "IMAGE_GPU";
 constexpr char kTensorsTag[] = "TENSORS";
 constexpr char kMatrixTag[] = "MATRIX";
 
+constexpr std::pair<float, float> kDefaultOutputRange = {0.0f, 1.0f};
+
 }  // namespace
 
 namespace mediapipe {
@@ -156,10 +159,6 @@ class TensorConverterCalculator : public CalculatorBase {
  private:
   absl::Status InitGpu(CalculatorContext* cc);
   absl::Status LoadOptions(CalculatorContext* cc, bool use_gpu);
-  template <class T>
-  absl::Status NormalizeImage(const ImageFrame& image_frame,
-                              bool flip_vertically, float* tensor_ptr);
-  absl::Status CopyMatrixToTensor(const Matrix& matrix, float* tensor_ptr);
   absl::Status ProcessCPU(CalculatorContext* cc);
   absl::Status ProcessGPU(CalculatorContext* cc);
 
@@ -279,46 +278,21 @@ absl::Status TensorConverterCalculator::ProcessCPU(CalculatorContext* cc) {
     }
     const auto& image_frame =
         cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-    const int height = image_frame.Height();
+    MP_ASSIGN_OR_RETURN(Tensor output,
-    const int width = image_frame.Width();
+                        ConvertImageFrameToTensorOnCpu(
-    const int channels = image_frame.NumberOfChannels();
+                            image_frame,
-    const int channels_preserved = std::min(channels, max_num_channels_);
+                            output_range_.has_value() ? output_range_.value()
-    const mediapipe::ImageFormat::Format format = image_frame.Format();
+                                                      : kDefaultOutputRange,
-
+                            flip_vertically_, max_num_channels_));
-    if (!(format == mediapipe::ImageFormat::SRGBA ||
+    output_tensors->emplace_back(std::move(output));
-          format == mediapipe::ImageFormat::SRGB ||
-          format == mediapipe::ImageFormat::GRAY8 ||
-          format == mediapipe::ImageFormat::VEC32F1))
-      RET_CHECK_FAIL() << "Unsupported CPU input format.";
-
-    output_tensors->emplace_back(
-        Tensor::ElementType::kFloat32,
-        Tensor::Shape{1, height, width, channels_preserved});
-    auto cpu_view = output_tensors->back().GetCpuWriteView();
-
-    // Copy image data into tensor.
-    if (image_frame.ByteDepth() == 1) {
-      MP_RETURN_IF_ERROR(NormalizeImage<uint8_t>(image_frame, flip_vertically_,
-                                                 cpu_view.buffer<float>()));
-    } else if (image_frame.ByteDepth() == 4) {
-      MP_RETURN_IF_ERROR(NormalizeImage<float>(image_frame, flip_vertically_,
-                                               cpu_view.buffer<float>()));
-    } else {
-      return absl::InternalError(
-          "Only byte-based (8 bit) and float (32 bit) images supported.");
-    }
   } else if (cc->Inputs().HasTag(kMatrixTag)) {
     if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) {
       return absl::OkStatus();
     }
     const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get<Matrix>();
-    const int height = matrix.rows();
+    MP_ASSIGN_OR_RETURN(Tensor output,
-    const int width = matrix.cols();
+                        ConvertMatrixToTensorOnCpu(matrix, row_major_matrix_));
-    const int channels = 1;
+    output_tensors->emplace_back(std::move(output));
-    output_tensors->emplace_back(Tensor::ElementType::kFloat32,
-                                 Tensor::Shape{1, height, width, channels});
-    MP_RETURN_IF_ERROR(CopyMatrixToTensor(
-        matrix, output_tensors->back().GetCpuWriteView().buffer<float>()));
   } else {
     return absl::OkStatus();
   }
@@ -669,67 +643,4 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc,
   return absl::OkStatus();
 }
 
-template <class T>
-absl::Status TensorConverterCalculator::NormalizeImage(
-    const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) {
-  const int height = image_frame.Height();
-  const int width = image_frame.Width();
-  const int channels = image_frame.NumberOfChannels();
-  const int channels_preserved = std::min(channels, max_num_channels_);
-  const int channels_ignored = channels - channels_preserved;
-
-  if (output_range_.has_value()) {
-    // If the output float range is set and we are not using custom
-    // normalization, normalize the pixel values from [0, 255] to the specified
-    // output range.
-    RET_CHECK_NE(output_range_->first, output_range_->second);
-    const float scale = (output_range_->second - output_range_->first) / 255.0f;
-    const float bias = output_range_->first;
-
-    for (int i = 0; i < height; ++i) {
-      const T* image_ptr = reinterpret_cast<const T*>(
-          image_frame.PixelData() +
-          (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
-      for (int j = 0; j < width; ++j) {
-        for (int c = 0; c < channels_preserved; ++c) {
-          *tensor_ptr++ = *image_ptr++ * scale + bias;
-        }
-        image_ptr += channels_ignored;
-      }
-    }
-  } else {
-    // [0,1], scale only (bias == 0)
-    // Verified that there are no precision issues with 1.0f / 255.0f expression
-    const float scale = 1.0f / 255.0f;
-    for (int i = 0; i < height; ++i) {
-      const T* image_ptr = reinterpret_cast<const T*>(
-          image_frame.PixelData() +
-          (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
-      for (int j = 0; j < width; ++j) {
-        for (int c = 0; c < channels_preserved; ++c) {
-          *tensor_ptr++ = *image_ptr++ * scale;
-        }
-        image_ptr += channels_ignored;
-      }
-    }
-  }
-
-  return absl::OkStatus();
-}
-
-absl::Status TensorConverterCalculator::CopyMatrixToTensor(const Matrix& matrix,
-                                                           float* tensor_ptr) {
-  if (row_major_matrix_) {
-    auto matrix_map =
-        Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
-    matrix_map = matrix;
-  } else {
-    auto matrix_map =
-        Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
-    matrix_map = matrix;
-  }
-
-  return absl::OkStatus();
-}
-
 }  // namespace mediapipe

mediapipe/calculators/tensor/tensor_converter_calculator.proto

@@ -32,7 +32,7 @@ message TensorConverterCalculatorOptions {
   // Custom settings to override the internal scaling factors `div` and `sub`.
   // Both values must be set to non-negative values. Will only take effect on
   // CPU AND when |use_custom_normalization| is set to true. When these custom
-  // values take effect, the |zero_center| setting above will be overriden, and
+  // values take effect, the |zero_center| setting above will be overridden, and
   // the normalized_value will be calculated as:
   // normalized_value = input / custom_div - custom_sub.
   optional bool use_custom_normalization = 6 [default = false];

mediapipe/calculators/tensor/tensor_converter_calculator_test.cc

@@ -321,6 +321,61 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) {
   }
 }
 
+TEST_F(TensorConverterCalculatorTest,
+       ShouldConvertImageWithDefaultOutputRange) {
+  CalculatorGraph graph;
+  CalculatorGraphConfig graph_config =
+      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"pb(
+            input_stream: "input_image"
+            node {
+              calculator: "TensorConverterCalculator"
+              input_stream: "IMAGE:input_image"
+              output_stream: "TENSORS:tensor"
+              options {
+                [mediapipe.TensorConverterCalculatorOptions.ext] {
+                  zero_center: false
+                }
+              }
+            }
+          )pb");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("tensor", &graph_config, &output_packets);
+
+  // Run the graph.
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  MP_ASSERT_OK(graph.StartRun({}));
+  auto input_image = std::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
+  cv::Mat mat = mediapipe::formats::MatView(input_image.get());
+  mat.at<uint8_t>(0, 0) = 200;
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "input_image", Adopt(input_image.release()).At(Timestamp(0))));
+
+  // Wait until the calculator finishes processing.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  ASSERT_EQ(output_packets.size(), 1);
+
+  // Get and process results.
+  const std::vector<Tensor>& tensor_vec =
+      output_packets[0].Get<std::vector<Tensor>>();
+  ASSERT_EQ(tensor_vec.size(), 1);
+
+  const Tensor* tensor = &tensor_vec[0];
+
+  // Calculate the expected normalized value:
+  float expected_value = 200.0 / 255.0;
+
+  EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32);
+  auto view = tensor->GetCpuReadView();
+  float actual_value = *view.buffer<float>();
+  EXPECT_FLOAT_EQ(actual_value, expected_value);
+
+  // Fully close graph at end, otherwise calculator+tensors are destroyed
+  // after calling WaitUntilDone().
+  MP_ASSERT_OK(graph.CloseInputStream("input_image"));
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
 TEST_F(TensorConverterCalculatorTest, FlipVertically) {
   CalculatorGraph graph;
   CalculatorGraphConfig graph_config =

mediapipe/calculators/tensor/tensor_converter_cpu.cc

@@ -0,0 +1,145 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
+
+#include <algorithm>
+#include <cstdint>
+#include <utility>
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status_macros.h"
+
+namespace mediapipe {
+namespace {
+
+typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
+    RowMajorMatrixXf;
+typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
+    ColMajorMatrixXf;
+
+template <class T>
+absl::Status NormalizeImage(const ImageFrame& image_frame, bool flip_vertically,
+                            const std::pair<float, float>& output_range,
+                            int max_num_channels, float* tensor_ptr) {
+  const int height = image_frame.Height();
+  const int width = image_frame.Width();
+  const int channels = image_frame.NumberOfChannels();
+  const int channels_preserved = std::min(channels, max_num_channels);
+  const int channels_ignored = channels - channels_preserved;
+
+  RET_CHECK_NE(output_range.first, output_range.second);
+  const float scale = (output_range.second - output_range.first) / 255.0f;
+  const float bias = output_range.first;
+
+  for (int i = 0; i < height; ++i) {
+    const T* image_ptr = reinterpret_cast<const T*>(
+        image_frame.PixelData() +
+        (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
+    for (int j = 0; j < width; ++j) {
+      for (int c = 0; c < channels_preserved; ++c) {
+        *tensor_ptr++ = *image_ptr++ * scale + bias;
+      }
+      image_ptr += channels_ignored;
+    }
+  }
+  return absl::OkStatus();
+}
+
+}  // namespace
+
+absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
+                                 bool flip_vertically,
+                                 const std::pair<float, float>& output_range,
+                                 int max_num_channels, float* tensor_ptr) {
+  return NormalizeImage<uint8_t>(image_frame, flip_vertically, output_range,
+                                 max_num_channels, tensor_ptr);
+}
+
+absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
+                                 bool flip_vertically,
+                                 const std::pair<float, float>& output_range,
+                                 int max_num_channels, float* tensor_ptr) {
+  return NormalizeImage<float>(image_frame, flip_vertically, output_range,
+                               max_num_channels, tensor_ptr);
+}
+
+absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
+                                float* tensor_ptr) {
+  if (is_row_major_matrix) {
+    auto matrix_map =
+        Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
+    matrix_map = matrix;
+  } else {
+    auto matrix_map =
+        Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
+    matrix_map = matrix;
+  }
+  return absl::OkStatus();
+}
+
+absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
+    const ImageFrame& image_frame, const std::pair<float, float>& output_range,
+    bool flip_vertically, int max_num_channels) {
+  const int height = image_frame.Height();
+  const int width = image_frame.Width();
+  const int channels = image_frame.NumberOfChannels();
+  const int channels_preserved = std::min(channels, max_num_channels);
+  const mediapipe::ImageFormat::Format format = image_frame.Format();
+
+  if (!(format == mediapipe::ImageFormat::SRGBA ||
+        format == mediapipe::ImageFormat::SRGB ||
+        format == mediapipe::ImageFormat::GRAY8 ||
+        format == mediapipe::ImageFormat::VEC32F1))
+    RET_CHECK_FAIL() << "Unsupported CPU input format.";
+
+  Tensor output_tensor(Tensor::ElementType::kFloat32,
+                       Tensor::Shape{1, height, width, channels_preserved});
+  auto cpu_view = output_tensor.GetCpuWriteView();
+
+  // Copy image data into tensor.
+  if (image_frame.ByteDepth() == 1) {
+    MP_RETURN_IF_ERROR(NormalizeUInt8Image(image_frame, flip_vertically,
+                                           output_range, max_num_channels,
+                                           cpu_view.buffer<float>()));
+  } else if (image_frame.ByteDepth() == 4) {
+    MP_RETURN_IF_ERROR(NormalizeFloatImage(image_frame, flip_vertically,
+                                           output_range, max_num_channels,
+                                           cpu_view.buffer<float>()));
+  } else {
+    return absl::InternalError(
+        "Only byte-based (8 bit) and float (32 bit) images supported.");
+  }
+  return output_tensor;
+}
+
+absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
+                                                  bool row_major_matrix) {
+  const int height = matrix.rows();
+  const int width = matrix.cols();
+  const int channels = 1;
+  Tensor output_tensor(Tensor::ElementType::kFloat32,
+                       Tensor::Shape{1, height, width, channels});
+  MP_RETURN_IF_ERROR(
+      CopyMatrixToTensor(matrix, row_major_matrix,
+                         output_tensor.GetCpuWriteView().buffer<float>()));
+  return output_tensor;
+}
+
+}  // namespace mediapipe

mediapipe/calculators/tensor/tensor_converter_cpu.h

@@ -0,0 +1,61 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
+#define MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
+
+#include <utility>
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/tensor.h"
+
+namespace mediapipe {
+
+// Converts an ImageFrame to a vector of Tensors.
+// @flip_vertically enables to flip the image during conversion.
+// @max_num_channels can be used to reserve extra channels in the output
+// tensors.
+// Returns output Tensor.
+absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
+    const ImageFrame& image_frame, const std::pair<float, float>& output_range,
+    bool flip_vertically, int max_num_channels);
+
+// Converts a Matrix to a vector of Tensors.
+// @row_major_matrix defines the ordering in the input matrix.
+// @max_num_channels can be used to reserve extra channels in the output
+// tensors.
+// Returns output Tensor.
+absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
+                                                  bool row_major_matrix);
+
+// For testing only below.
+absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
+                                 bool flip_vertically,
+                                 const std::pair<float, float>& output_range,
+                                 int max_num_channels, float* tensor_ptr);
+
+absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
+                                 bool flip_vertically,
+                                 const std::pair<float, float>& output_range,
+                                 int max_num_channels, float* tensor_ptr);
+
+absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
+                                float* tensor_ptr);
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_

mediapipe/calculators/tensor/tensor_converter_cpu_test.cc

@@ -0,0 +1,175 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
+
+#include <cstdint>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/util/image_test_utils.h"
+
+namespace mediapipe {
+namespace {
+
+Matrix CreateTestMatrix(int num_rows, int num_columns) {
+  Matrix matrix(num_rows, num_columns);
+  for (int r = 0; r < num_rows; ++r) {
+    for (int c = 0; c < num_columns; ++c) {
+      matrix(r, c) = r * num_columns + c;
+    }
+  }
+  return matrix;
+}
+
+TEST(TensorConverterCpuTest, ShouldCopyMatrixInRowMajorFormatToTensor) {
+  auto test_matrix = CreateTestMatrix(/* num_rows=*/3, /*num_columns=*/4);
+  std::vector<float> tensor_data(test_matrix.size(), 0.0f);
+
+  MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/true,
+                                  tensor_data.data()));
+
+  for (int i = 0; i < tensor_data.size(); ++i) {
+    const int row = i / test_matrix.cols();
+    const int column = i % test_matrix.cols();
+    EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
+  }
+}
+
+TEST(TensorConverterCpuTest, ShouldCopyMatrixInColumnMajorFormatToTensor) {
+  auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
+  std::vector<float> tensor_data(test_matrix.size(), 0.0f);
+
+  MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/false,
+                                  tensor_data.data()));
+
+  for (int i = 0; i < tensor_data.size(); ++i) {
+    const int row = i % test_matrix.rows();
+    const int column = i / test_matrix.rows();
+    EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
+  }
+}
+
+TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithDefaultRange) {
+  auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
+  std::vector<float> tensor_data(
+      grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
+
+  MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false,
+                                   {0.0f, 1.0f}, /*num_tensor_channels=*/1,
+                                   tensor_data.data()));
+
+  for (int i = 0; i < tensor_data.size(); ++i) {
+    EXPECT_FLOAT_EQ(
+        tensor_data[i],
+        static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
+  }
+}
+
+TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithSpecifiedRange) {
+  auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
+  std::vector<float> tensor_data(
+      grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
+  const auto range = std::make_pair(2.0f, 3.0f);
+
+  MP_EXPECT_OK(
+      NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, range,
+                          /*num_tensor_channels=*/1, tensor_data.data()));
+
+  for (int i = 0; i < tensor_data.size(); ++i) {
+    EXPECT_FLOAT_EQ(tensor_data[i],
+                    static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) /
+                            255.0f * (range.second - range.first) +
+                        range.first);
+  }
+}
+
+TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageFlipped) {
+  auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
+  std::vector<float> tensor_data(
+      grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
+
+  MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/true,
+                                   {0.0f, 1.0f}, /*num_tensor_channels=*/1,
+                                   tensor_data.data()));
+
+  for (int i = 0; i < tensor_data.size(); ++i) {
+    const int x = i % grey8_image_frame.Width();
+    const int y = i / grey8_image_frame.Width();
+    const int flipped_y = grey8_image_frame.Height() - y - 1;
+
+    const int index = flipped_y * grey8_image_frame.Width() + x;
+    EXPECT_FLOAT_EQ(
+        tensor_data[index],
+        static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
+  }
+}
+
+TEST(TensorConverterCpuTest, ShouldNormalizeFloatImageWithDefaultRange) {
+  auto float_image_frame =
+      CreateTestFloat32ImageFrame(/*width=*/3, /*height=*/4);
+  std::vector<float> tensor_data(
+      float_image_frame.Width() * float_image_frame.Height(), 0.0f);
+
+  MP_EXPECT_OK(NormalizeFloatImage(float_image_frame, /*flip_vertically=*/false,
+                                   {0.0f, 1.0f}, /*num_tensor_channels=*/1,
+                                   tensor_data.data()));
+
+  for (int i = 0; i < tensor_data.size(); ++i) {
+    EXPECT_FLOAT_EQ(tensor_data[i], reinterpret_cast<const float*>(
+                                        float_image_frame.PixelData())[i] /
+                                        255.0f);
+  }
+}
+
+TEST(TensorConverterCpuTest, ConvertImageFrameToTensorOnCpu) {
+  auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
+
+  MP_ASSERT_OK_AND_ASSIGN(Tensor output, ConvertImageFrameToTensorOnCpu(
+                                             grey8_image_frame, {0.0f, 1.0f},
+                                             /*flip_vertically=*/false,
+                                             /*max_num_channels=*/1));
+
+  const auto cpu_read_view = output.GetCpuReadView();
+  const float* tensor_ptr = cpu_read_view.buffer<float>();
+  for (int i = 0; i < grey8_image_frame.Width() * grey8_image_frame.Height();
+       ++i) {
+    EXPECT_FLOAT_EQ(
+        tensor_ptr[i],
+        static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0);
+  }
+}
+
+TEST(TensorConverterCpuTest, ConvertMatrixToTensorOnCpu) {
+  auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
+
+  MP_ASSERT_OK_AND_ASSIGN(
+      Tensor output, ConvertMatrixToTensorOnCpu(test_matrix,
+                                                /*row_major_matrix=*/false));
+
+  const auto cpu_read_view = output.GetCpuReadView();
+  const float* tensor_ptr = cpu_read_view.buffer<float>();
+  for (int i = 0; i < test_matrix.size(); ++i) {
+    EXPECT_FLOAT_EQ(tensor_ptr[i], test_matrix.data()[i]);
+  }
+}
+
+}  // namespace
+
+}  // namespace mediapipe

mediapipe/calculators/tensor/tensors_to_classification_calculator.proto

@@ -34,7 +34,7 @@ message TensorsToClassificationCalculatorOptions {
     repeated Entry entries = 1;
   }
 
-  // Score threshold for perserving the class.
+  // Score threshold for preserving the class.
   optional float min_score_threshold = 1;
   // Number of highest scoring labels to output.  If top_k is not positive then
   // all labels are used.

mediapipe/calculators/tensor/tensors_to_detections_calculator.cc

@@ -15,7 +15,6 @@
 #include <unordered_map>
 #include <vector>
 
-#include "absl/log/absl_log.h"
 #include "absl/strings/str_format.h"
 #include "absl/types/span.h"
 #include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h"
@@ -147,7 +146,7 @@ BoxFormat GetBoxFormat(const TensorsToDetectionsCalculatorOptions& options) {
 //  TENSORS - Vector of Tensors of type kFloat32. The vector of tensors can have
 //            2 or 3 tensors. First tensor is the predicted raw boxes/keypoints.
 //            The size of the values must be (num_boxes * num_predicted_values).
-//            Second tensor is the score tensor. The size of the valuse must be
+//            Second tensor is the score tensor. The size of the values must be
 //            (num_boxes * num_classes). It's optional to pass in a third tensor
 //            for anchors (e.g. for SSD models) depend on the outputs of the
 //            detection model. The size of anchor tensor must be (num_boxes *
@@ -215,7 +214,8 @@ class TensorsToDetectionsCalculator : public Node {
                                    const int* detection_classes,
                                    std::vector<Detection>* output_detections);
   Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax,
-                               float box_xmax, float score, int class_id,
+                               float box_xmax, absl::Span<const float> scores,
+                               absl::Span<const int> class_ids,
                                bool flip_vertically);
   bool IsClassIndexAllowed(int class_index);
 
@@ -223,6 +223,7 @@ class TensorsToDetectionsCalculator : public Node {
   int num_boxes_ = 0;
   int num_coords_ = 0;
   int max_results_ = -1;
+  int classes_per_detection_ = 1;
   BoxFormat box_output_format_ =
       mediapipe::TensorsToDetectionsCalculatorOptions::YXHW;
 
@@ -267,7 +268,7 @@ absl::Status TensorsToDetectionsCalculator::UpdateContract(
   if (CanUseGpu()) {
 #ifndef MEDIAPIPE_DISABLE_GL_COMPUTE
     MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
-        cc, /*requesst_gpu_as_optional=*/true));
+        cc, /*request_gpu_as_optional=*/true));
 #elif MEDIAPIPE_METAL_ENABLED
     MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
 #endif  // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
@@ -484,6 +485,16 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
     auto num_boxes_view = num_boxes_tensor->GetCpuReadView();
     auto num_boxes = num_boxes_view.buffer<float>();
     num_boxes_ = num_boxes[0];
+    // The detection model with Detection_PostProcess op may output duplicate
+    // boxes with different classes, in the following format:
+    //   num_boxes_tensor = [num_boxes]
+    //   detection_classes_tensor = [box_1_class_1, box_1_class_2, ...]
+    //   detection_scores_tensor = [box_1_score_1, box_1_score_2, ... ]
+    //   detection_boxes_tensor = [box_1, box1, ... ]
+    // Each box repeats classes_per_detection_ times.
+    // Note Detection_PostProcess op is only supported in CPU.
+    RET_CHECK_EQ(max_detections % num_boxes_, 0);
+    classes_per_detection_ = max_detections / num_boxes_;
 
     auto detection_boxes_view = detection_boxes_tensor->GetCpuReadView();
     auto detection_boxes = detection_boxes_view.buffer<float>();
@@ -493,8 +504,8 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU(
 
     auto detection_classes_view = detection_classes_tensor->GetCpuReadView();
     auto detection_classes_ptr = detection_classes_view.buffer<float>();
-    std::vector<int> detection_classes(num_boxes_);
+    std::vector<int> detection_classes(num_boxes_ * classes_per_detection_);
-    for (int i = 0; i < num_boxes_; ++i) {
+    for (int i = 0; i < detection_classes.size(); ++i) {
       detection_classes[i] = static_cast<int>(detection_classes_ptr[i]);
     }
     MP_RETURN_IF_ERROR(ConvertToDetections(detection_boxes, detection_scores,
@@ -863,24 +874,25 @@ absl::Status TensorsToDetectionsCalculator::DecodeBoxes(
 absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
     const float* detection_boxes, const float* detection_scores,
     const int* detection_classes, std::vector<Detection>* output_detections) {
-  for (int i = 0; i < num_boxes_; ++i) {
+  for (int i = 0; i < num_boxes_ * classes_per_detection_;
+       i += classes_per_detection_) {
     if (max_results_ > 0 && output_detections->size() == max_results_) {
       break;
     }
-    if (options_.has_min_score_thresh() &&
-        detection_scores[i] < options_.min_score_thresh()) {
-      continue;
-    }
-    if (!IsClassIndexAllowed(detection_classes[i])) {
-      continue;
-    }
     const int box_offset = i * num_coords_;
     Detection detection = ConvertToDetection(
         /*box_ymin=*/detection_boxes[box_offset + box_indices_[0]],
         /*box_xmin=*/detection_boxes[box_offset + box_indices_[1]],
         /*box_ymax=*/detection_boxes[box_offset + box_indices_[2]],
         /*box_xmax=*/detection_boxes[box_offset + box_indices_[3]],
-        detection_scores[i], detection_classes[i], options_.flip_vertically());
+        absl::MakeConstSpan(detection_scores + i, classes_per_detection_),
+        absl::MakeConstSpan(detection_classes + i, classes_per_detection_),
+        options_.flip_vertically());
+    // if all the scores and classes are filtered out, we skip the empty
+    // detection.
+    if (detection.score().empty()) {
+      continue;
+    }
     const auto& bbox = detection.location_data().relative_bounding_box();
     if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) ||
         std::isnan(bbox.height())) {
@@ -910,11 +922,21 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
 }
 
 Detection TensorsToDetectionsCalculator::ConvertToDetection(
-    float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score,
+    float box_ymin, float box_xmin, float box_ymax, float box_xmax,
-    int class_id, bool flip_vertically) {
+    absl::Span<const float> scores, absl::Span<const int> class_ids,
+    bool flip_vertically) {
   Detection detection;
-  detection.add_score(score);
+  for (int i = 0; i < scores.size(); ++i) {
-  detection.add_label_id(class_id);
+    if (!IsClassIndexAllowed(class_ids[i])) {
+      continue;
+    }
+    if (options_.has_min_score_thresh() &&
+        scores[i] < options_.min_score_thresh()) {
+      continue;
+    }
+    detection.add_score(scores[i]);
+    detection.add_label_id(class_ids[i]);
+  }
 
   LocationData* location_data = detection.mutable_location_data();
   location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX);

mediapipe/calculators/tensor/tensors_to_detections_calculator.proto

@@ -75,7 +75,7 @@ message TensorsToDetectionsCalculatorOptions {
   // representation has a bottom-left origin (e.g., in OpenGL).
   optional bool flip_vertically = 18 [default = false];
 
-  // Score threshold for perserving decoded detections.
+  // Score threshold for preserving decoded detections.
   optional float min_score_thresh = 19;
 
   // The maximum number of the detection results to return. If < 0, all

mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc

@@ -124,7 +124,7 @@ absl::Status TensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
        kFlipVertically(cc).IsConnected())) {
     RET_CHECK(options_.has_input_image_height() &&
               options_.has_input_image_width())
-        << "Must provide input width/height for using flipping when outputing "
+        << "Must provide input width/height for using flipping when outputting "
            "landmarks in absolute coordinates.";
   }
   return absl::OkStatus();

mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc

@@ -12,32 +12,35 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <memory>
+#include <string>
+#include <tuple>
+#include <utility>
 #include <vector>
 
-#include "absl/strings/str_format.h"
+#include "absl/status/status.h"
-#include "absl/types/span.h"
+#include "absl/strings/str_cat.h"
 #include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image.h"
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/port.h"
 #include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/framework/port/status_macros.h"
 #include "mediapipe/gpu/gpu_origin.pb.h"
-#include "mediapipe/util/resource_util.h"
-#include "tensorflow/lite/interpreter.h"
 
 #if !MEDIAPIPE_DISABLE_GPU
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gl_simple_shaders.h"
-#include "mediapipe/gpu/gpu_buffer.h"
+#include "mediapipe/gpu/gpu_buffer_format.h"
 #include "mediapipe/gpu/shader_util.h"
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
 #if !MEDIAPIPE_DISABLE_OPENCV
-#include "mediapipe/framework/formats/image_opencv.h"
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
-#include "mediapipe/framework/port/opencv_imgproc_inc.h"
 #endif  // !MEDIAPIPE_DISABLE_OPENCV
 
 #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
@@ -62,37 +65,9 @@ namespace {
 constexpr int kWorkgroupSize = 8;  // Block size for GPU shader.
 enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
 
-// Commonly used to compute the number of blocks to launch in a kernel.
-int NumGroups(const int size, const int group_size) {  // NOLINT
-  return (size + group_size - 1) / group_size;
-}
-
-bool CanUseGpu() {
-#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
-  // TODO: Configure GPU usage policy in individual calculators.
-  constexpr bool kAllowGpuProcessing = true;
-  return kAllowGpuProcessing;
-#else
-  return false;
-#endif  // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
-}
-
 constexpr char kTensorsTag[] = "TENSORS";
 constexpr char kOutputSizeTag[] = "OUTPUT_SIZE";
 constexpr char kMaskTag[] = "MASK";
-
-absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
-    const std::vector<int>& dims) {
-  if (dims.size() == 3) {
-    return std::make_tuple(dims[0], dims[1], dims[2]);
-  } else if (dims.size() == 4) {
-    // BHWC format check B == 1
-    RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap";
-    return std::make_tuple(dims[1], dims[2], dims[3]);
-  } else {
-    RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
-  }
-}
 }  // namespace
 
 namespace mediapipe {
@@ -156,19 +131,28 @@ class TensorsToSegmentationCalculator : public CalculatorBase {
  private:
   absl::Status LoadOptions(CalculatorContext* cc);
   absl::Status InitGpu(CalculatorContext* cc);
-  absl::Status ProcessGpu(CalculatorContext* cc);
+  absl::Status ProcessGpu(CalculatorContext* cc,
-  absl::Status ProcessCpu(CalculatorContext* cc);
+                          const std::vector<Tensor>& input_tensors,
+                          std::tuple<int, int, int> hwc, int output_width,
+                          int output_height);
   void GlRender();
 
   bool DoesGpuTextureStartAtBottom() {
     return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
   }
-
+  absl::Status InitConverterIfNecessary() {
 #if !MEDIAPIPE_DISABLE_OPENCV
-  template <class T>
+    if (!cpu_converter_) {
-  absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat);
+      MP_ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(options_));
+    }
+#else
+    RET_CHECK_FAIL() << "OpenCV processing disabled.";
 #endif  // !MEDIAPIPE_DISABLE_OPENCV
-  ::mediapipe::TensorsToSegmentationCalculatorOptions options_;
+    return absl::OkStatus();
+  }
+
+  mediapipe::TensorsToSegmentationCalculatorOptions options_;
+  std::unique_ptr<TensorsToSegmentationConverter> cpu_converter_;
 
 #if !MEDIAPIPE_DISABLE_GPU
   mediapipe::GlCalculatorHelper gpu_helper_;
@@ -208,7 +192,7 @@ absl::Status TensorsToSegmentationCalculator::GetContract(
   if (CanUseGpu()) {
 #if !MEDIAPIPE_DISABLE_GPU
     MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
-        cc, /*requesst_gpu_as_optional=*/true));
+        cc, /*request_gpu_as_optional=*/true));
 #if MEDIAPIPE_METAL_ENABLED
     MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]);
 #endif  // MEDIAPIPE_METAL_ENABLED
@@ -261,7 +245,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
     MP_ASSIGN_OR_RETURN(auto hwc,
                         GetHwcFromDims(input_tensors[0].shape().dims));
     int tensor_channels = std::get<2>(hwc);
-    typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
+    using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
     switch (options_.activation()) {
       case Options::NONE:
         RET_CHECK_EQ(tensor_channels, 1);
@@ -275,6 +259,17 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
     }
   }
 
+  // Get dimensions.
+  MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
+  auto [tensor_height, tensor_width, tensor_channels] = hwc;
+  int output_width = tensor_width, output_height = tensor_height;
+  if (cc->Inputs().HasTag(kOutputSizeTag)) {
+    const auto& size =
+        cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
+    output_width = size.first;
+    output_height = size.second;
+  }
+
   if (use_gpu) {
 #if !MEDIAPIPE_DISABLE_GPU
     if (!gpu_initialized_) {
@@ -286,16 +281,25 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
 #endif  // !MEDIAPIPE_DISABLE_GPU
 
 #if !MEDIAPIPE_DISABLE_GPU
-    MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status {
+    MP_RETURN_IF_ERROR(
-      MP_RETURN_IF_ERROR(ProcessGpu(cc));
+        gpu_helper_.RunInGlContext([this, cc, &input_tensors, output_width,
-      return absl::OkStatus();
+                                    output_height, hwc]() -> absl::Status {
-    }));
+          MP_RETURN_IF_ERROR(
+              ProcessGpu(cc, input_tensors, hwc, output_width, output_height));
+          return absl::OkStatus();
+        }));
 #else
     RET_CHECK_FAIL() << "GPU processing disabled.";
 #endif  // !MEDIAPIPE_DISABLE_GPU
   } else {
 #if !MEDIAPIPE_DISABLE_OPENCV
-    MP_RETURN_IF_ERROR(ProcessCpu(cc));
+    // Lazily initialize converter.
+    MP_RETURN_IF_ERROR(InitConverterIfNecessary());
+    MP_ASSIGN_OR_RETURN(
+        std::unique_ptr<Image> output_mask,
+        cpu_converter_->Convert(input_tensors, output_width, output_height));
+    cc->Outputs().Tag(kMaskTag).Add(output_mask.release(),
+                                    cc->InputTimestamp());
 #else
     RET_CHECK_FAIL() << "OpenCV processing disabled.";
 #endif  // !MEDIAPIPE_DISABLE_OPENCV
@@ -329,132 +333,15 @@ absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) {
   return absl::OkStatus();
 }
 
-absl::Status TensorsToSegmentationCalculator::ProcessCpu(
-    CalculatorContext* cc) {
-#if !MEDIAPIPE_DISABLE_OPENCV
-  // Get input streams, and dimensions.
-  const auto& input_tensors =
-      cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
-  MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
-  auto [tensor_height, tensor_width, tensor_channels] = hwc;
-  int output_width = tensor_width, output_height = tensor_height;
-  if (cc->Inputs().HasTag(kOutputSizeTag)) {
-    const auto& size =
-        cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
-    output_width = size.first;
-    output_height = size.second;
-  }
-
-  // Create initial working mask.
-  cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
-
-  // Wrap input tensor.
-  auto raw_input_tensor = &input_tensors[0];
-  auto raw_input_view = raw_input_tensor->GetCpuReadView();
-  const float* raw_input_data = raw_input_view.buffer<float>();
-  cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
-                     CV_MAKETYPE(CV_32F, tensor_channels),
-                     const_cast<float*>(raw_input_data));
-
-  // Process mask tensor and apply activation function.
-  if (tensor_channels == 2) {
-    MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
-  } else if (tensor_channels == 1) {
-    RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
-              options_.activation());  // Requires 2 channels.
-    if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
-        options_.activation())  // Pass-through optimization.
-      tensor_mat.copyTo(small_mask_mat);
-    else
-      MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
-  } else {
-    RET_CHECK_FAIL() << "Unsupported number of tensor channels "
-                     << tensor_channels;
-  }
-
-  // Send out image as CPU packet.
-  std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
-      ImageFormat::VEC32F1, output_width, output_height);
-  std::unique_ptr<Image> output_mask = absl::make_unique<Image>(mask_frame);
-  auto output_mat = formats::MatView(output_mask.get());
-  // Upsample small mask into output.
-  cv::resize(small_mask_mat, *output_mat,
-             cv::Size(output_width, output_height));
-  cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp());
-#endif  // !MEDIAPIPE_DISABLE_OPENCV
-
-  return absl::OkStatus();
-}
-
-#if !MEDIAPIPE_DISABLE_OPENCV
-template <class T>
-absl::Status TensorsToSegmentationCalculator::ApplyActivation(
-    cv::Mat& tensor_mat, cv::Mat* small_mask_mat) {
-  // Configure activation function.
-  const int output_layer_index = options_.output_layer_index();
-  typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
-  const auto activation_fn = [&](const cv::Vec2f& mask_value) {
-    float new_mask_value = 0;
-    // TODO consider moving switch out of the loop,
-    // and also avoid float/Vec2f casting.
-    switch (options_.activation()) {
-      case Options::NONE: {
-        new_mask_value = mask_value[0];
-        break;
-      }
-      case Options::SIGMOID: {
-        const float pixel0 = mask_value[0];
-        new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
-        break;
-      }
-      case Options::SOFTMAX: {
-        const float pixel0 = mask_value[0];
-        const float pixel1 = mask_value[1];
-        const float max_pixel = std::max(pixel0, pixel1);
-        const float min_pixel = std::min(pixel0, pixel1);
-        const float softmax_denom =
-            /*exp(max_pixel - max_pixel)=*/1.0f +
-            std::exp(min_pixel - max_pixel);
-        new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
-                         softmax_denom;
-        break;
-      }
-    }
-    return new_mask_value;
-  };
-
-  // Process mask tensor.
-  for (int i = 0; i < tensor_mat.rows; ++i) {
-    for (int j = 0; j < tensor_mat.cols; ++j) {
-      const T& input_pix = tensor_mat.at<T>(i, j);
-      const float mask_value = activation_fn(input_pix);
-      small_mask_mat->at<float>(i, j) = mask_value;
-    }
-  }
-
-  return absl::OkStatus();
-}
-#endif  // !MEDIAPIPE_DISABLE_OPENCV
-
 // Steps:
 // 1. receive tensor
 // 2. process segmentation tensor into small mask
 // 3. upsample small mask into output mask to be same size as input image
 absl::Status TensorsToSegmentationCalculator::ProcessGpu(
-    CalculatorContext* cc) {
+    CalculatorContext* cc, const std::vector<Tensor>& input_tensors,
+    std::tuple<int, int, int> hwc, int output_width, int output_height) {
 #if !MEDIAPIPE_DISABLE_GPU
-  // Get input streams, and dimensions.
-  const auto& input_tensors =
-      cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
-  MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
   auto [tensor_height, tensor_width, tensor_channels] = hwc;
-  int output_width = tensor_width, output_height = tensor_height;
-  if (cc->Inputs().HasTag(kOutputSizeTag)) {
-    const auto& size =
-        cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
-    output_width = size.first;
-    output_height = size.second;
-  }
 
   // Create initial working mask texture.
 #if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31)
@@ -632,7 +519,7 @@ void TensorsToSegmentationCalculator::GlRender() {
 absl::Status TensorsToSegmentationCalculator::LoadOptions(
     CalculatorContext* cc) {
   // Get calculator options specified in the graph.
-  options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>();
+  options_ = cc->Options<mediapipe::TensorsToSegmentationCalculatorOptions>();
 
   return absl::OkStatus();
 }
@@ -826,7 +713,7 @@ void main() {
 #endif  // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
 
     // Shader defines.
-    typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
+    using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
     const std::string output_layer_index =
         "\n#define OUTPUT_LAYER_INDEX int(" +
         std::to_string(options_.output_layer_index()) + ")";

mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc

@@ -17,10 +17,8 @@
 #include <utility>
 #include <vector>
 
-#include "absl/log/absl_log.h"
-#include "absl/log/log.h"
-#include "absl/strings/substitute.h"
 #include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
 #include "mediapipe/framework/formats/image.h"
@@ -30,7 +28,6 @@
 #include "mediapipe/framework/formats/tensor.h"
 #include "mediapipe/framework/packet.h"
 #include "mediapipe/framework/port/gtest.h"
-#include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/port/status_matchers.h"
 #include "mediapipe/framework/timestamp.h"
 
@@ -40,62 +37,17 @@ namespace {
 using ::testing::SizeIs;
 using ::testing::TestWithParam;
 using Options = mediapipe::TensorsToSegmentationCalculatorOptions;
+namespace test_utils = ::mediapipe::tensors_to_segmentation_utils;
 
-std::string ActivationTypeToString(Options::Activation activation) {
+using TensorsToSegmentationCalculatorTest =
-  switch (activation) {
+    TestWithParam<test_utils::FormattingTestCase>;
-    case Options::NONE:
-      return "NONE";
-    case Options::SIGMOID:
-      return "SIGMOID";
-    case Options::SOFTMAX:
-      return "SOFTMAX";
-    default:
-      ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
-      return "UNKNOWN";
-  }
-}
 
-struct FormattingTestCase {
-  std::string test_name;
-  std::vector<float> inputs;
-  std::vector<float> expected_outputs;
-  Options::Activation activation;
-  int rows;
-  int cols;
-  int channels;
-};
-
-using TensorsToSegmentationCalculatorTest = TestWithParam<FormattingTestCase>;
-
-// Currently only useable for tests with no output resize.
 TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
-  const FormattingTestCase& test_case = GetParam();
+  const auto& [test_name, inputs, expected_outputs, activation, rows, cols,
-  std::vector<float> inputs = test_case.inputs;
+               rows_new, cols_new, channels, max_abs_diff] = GetParam();
-  std::vector<float> expected_outputs = test_case.expected_outputs;
-  Options::Activation activation = test_case.activation;
-  int rows = test_case.rows;
-  int cols = test_case.cols;
-  int channels = test_case.channels;
 
-  std::string string_config = absl::Substitute(
-      R"pb(
-        input_stream: "tensors"
-        input_stream: "size"
-        node {
-          calculator: "TensorsToSegmentationCalculator"
-          input_stream: "TENSORS:tensors"
-          input_stream: "OUTPUT_SIZE:size"
-          output_stream: "MASK:image_as_mask"
-          options: {
-            [mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
-              activation: $0
-            }
-          }
-        }
-      )pb",
-      ActivationTypeToString(activation));
   auto graph_config =
-      mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(string_config);
+      test_utils::CreateGraphConfigForTest(/*test_gpu=*/false, activation);
 
   std::vector<Packet> output_packets;
   tool::AddVectorSink("image_as_mask", &graph_config, &output_packets);
@@ -119,28 +71,34 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
     MP_ASSERT_OK(graph.AddPacketToInputStream(
         "tensors", mediapipe::Adopt(tensors.release()).At(Timestamp(0))));
   }
+
+  // The output size is defined as pair(new_width, new_height).
   MP_ASSERT_OK(graph.AddPacketToInputStream(
-      "size",
+      "size", mediapipe::Adopt(new std::pair<int, int>(cols_new, rows_new))
-      mediapipe::Adopt(new std::pair<int, int>(rows, cols)).At(Timestamp(0))));
+                  .At(Timestamp(0))));
   MP_ASSERT_OK(graph.WaitUntilIdle());
 
   ASSERT_THAT(output_packets, SizeIs(1));
   const Image& image_as_mask = output_packets[0].Get<Image>();
+  EXPECT_FALSE(image_as_mask.UsesGpu());
+
   std::shared_ptr<cv::Mat> result_mat = formats::MatView(&image_as_mask);
-  EXPECT_EQ(result_mat->rows, rows);
+  EXPECT_EQ(result_mat->rows, rows_new);
-  EXPECT_EQ(result_mat->cols, cols);
+  EXPECT_EQ(result_mat->cols, cols_new);
-  EXPECT_EQ(result_mat->channels(), channels);
+  EXPECT_EQ(result_mat->channels(), 1);
 
   // Compare the real result with the expected result.
-  cv::Mat expected_result = cv::Mat(
+  cv::Mat expected_result =
-      rows, cols, CV_32FC1, const_cast<float*>(expected_outputs.data()));
+      cv::Mat(rows_new, cols_new, CV_32FC1,
+              const_cast<float*>(expected_outputs.data()));
   cv::Mat diff;
   cv::absdiff(*result_mat, expected_result, diff);
   double max_val;
   cv::minMaxLoc(diff, nullptr, &max_val);
-  // Expects the maximum absolute pixel-by-pixel difference is less than 1e-5.
+
-  // This delta is for passthorugh accuracy only.
+  // The max allowable diff between output and expected output varies between
-  EXPECT_LE(max_val, 1e-5);
+  // tests.
+  EXPECT_LE(max_val, max_abs_diff);
 
   MP_ASSERT_OK(graph.CloseInputStream("tensors"));
   MP_ASSERT_OK(graph.CloseInputStream("size"));
@@ -149,18 +107,97 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
 
 INSTANTIATE_TEST_SUITE_P(
     TensorsToSegmentationCalculatorTests, TensorsToSegmentationCalculatorTest,
-    testing::ValuesIn<FormattingTestCase>({
+    testing::ValuesIn<test_utils::FormattingTestCase>({
-        {/*test_name=*/"NoActivationAndNoOutputResize",
+        {.test_name = "NoActivationAndNoOutputResize",
-         /*inputs=*/
+         .inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
-         {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0,
+                    12.0, 13.0, 14.0, 15.0, 16.0},
-          14.0, 15.0, 16.0},
+         .expected_outputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0,
-         /*expected_outputs=*/
+                              11.0, 12.0, 13.0, 14.0, 15.0, 16.0},
-         {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0,
+         .activation = Options::NONE,
-          14.0, 15.0, 16.0},
+         .rows = 4,
-         /*activation=*/Options::NONE,
+         .cols = 4,
-         /*rows=*/4,
+         .rows_new = 4,
-         /*cols=*/4,
+         .cols_new = 4,
-         /*channels=*/1},
+         .channels = 1,
+         .max_abs_diff = 1e-7},
+        {.test_name = "OutputResizeOnly",
+         .inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
+                    12.0, 13.0, 14.0, 15.0, 16.0},
+         .expected_outputs = {1,    1.5,  2.166667,  2.833333,  3.5,  4,
+                              3.8,  4.3,  4.966667,  5.633333,  6.3,  6.8,
+                              7,    7.5,  8.166667,  8.833333,  9.5,  10,
+                              10.2, 10.7, 11.366667, 12.033333, 12.7, 13.2,
+                              13,   13.5, 14.166667, 14.833333, 15.5, 16},
+         .activation = Options::NONE,
+         .rows = 4,
+         .cols = 4,
+         .rows_new = 5,
+         .cols_new = 6,
+         .channels = 1,
+         .max_abs_diff = 1e-6},
+        {.test_name = "SigmoidActivationWithNoOutputResize",
+         .inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
+                    12.0, 13.0, 14.0, 15.0, 16.0},
+         .expected_outputs = {0.731059, 0.880797, 0.952574, 0.982014, 0.993307,
+                              0.997527, 0.999089, 0.999665, 0.999877, 0.999955,
+                              0.999983, 0.999994, 0.999998, 0.999999, 1.0, 1.0},
+         .activation = Options::SIGMOID,
+         .rows = 4,
+         .cols = 4,
+         .rows_new = 4,
+         .cols_new = 4,
+         .channels = 1,
+         .max_abs_diff = 1e-6},
+        {.test_name = "SigmoidActivationWithOutputResize",
+         .inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
+                    12.0, 13.0, 14.0, 15.0, 16.0},
+         .expected_outputs = {0.731059, 0.805928, 0.89276,  0.940611, 0.967294,
+                              0.982014, 0.914633, 0.93857,  0.966279, 0.981363,
+                              0.989752, 0.994369, 0.996592, 0.997666, 0.998873,
+                              0.999404, 0.999683, 0.999829, 0.999913, 0.99994,
+                              0.999971, 0.999985, 0.999992, 0.999996, 0.999998,
+                              0.999998, 0.999999, 1.0,      1.0,      1.0},
+         .activation = Options::SIGMOID,
+         .rows = 4,
+         .cols = 4,
+         .rows_new = 5,
+         .cols_new = 6,
+         .channels = 1,
+         .max_abs_diff = 1e-6},
+        {.test_name = "SoftmaxActivationWithNoOutputResize",
+         .inputs = {1.0,  2.0,  4.0,  2.0,  3.0,  5.0,  6.0,  1.5,
+                    7.0,  10.0, 11.0, 4.0,  12.0, 15.0, 16.0, 18.5,
+                    19.0, 20.0, 22.0, 23.0, 24.5, 23.4, 25.6, 28.3,
+                    29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3},
+         .expected_outputs = {0.731059, 0.119203, 0.880797, 0.0109869, 0.952574,
+                              0.000911051, 0.952574, 0.924142, 0.731059,
+                              0.731059, 0.24974, 0.937027, 0.689974, 0.990048,
+                              0.0060598, 0.28905},
+         .activation = Options::SOFTMAX,
+         .rows = 4,
+         .cols = 4,
+         .rows_new = 4,
+         .cols_new = 4,
+         .channels = 2,
+         .max_abs_diff = 1e-6},
+        {.test_name = "SoftmaxActivationWithOutputResize",
+         .inputs = {1.0,  2.0,  4.0,  2.0,  3.0,  5.0,  6.0,  1.5,
+                    7.0,  10.0, 11.0, 4.0,  12.0, 15.0, 16.0, 18.5,
+                    19.0, 20.0, 22.0, 23.0, 24.5, 23.4, 25.6, 28.3,
+                    29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3},
+         .expected_outputs = {0.731059,  0.425131, 0.246135, 0.753865, 0.445892,
+                              0.0109869, 0.886119, 0.461259, 0.185506, 0.781934,
+                              0.790618,  0.650195, 0.841816, 0.603901, 0.40518,
+                              0.561962,  0.765871, 0.930584, 0.718733, 0.763744,
+                              0.703402,  0.281989, 0.459635, 0.742634, 0.689974,
+                              0.840011,  0.82605,  0.170058, 0.147555, 0.28905},
+         .activation = Options::SOFTMAX,
+         .rows = 4,
+         .cols = 4,
+         .rows_new = 5,
+         .cols_new = 6,
+         .channels = 2,
+         .max_abs_diff = 1e-6},
     }),
     [](const testing::TestParamInfo<
         TensorsToSegmentationCalculatorTest::ParamType>& info) {

mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.cc

@@ -0,0 +1,111 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
+
+#include <string>
+#include <vector>
+
+#include "absl/log/absl_log.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+
+namespace mediapipe {
+namespace tensors_to_segmentation_utils {
+
+std::string ActivationTypeToString(
+    const TensorsToSegmentationCalculatorOptions::Activation& activation) {
+  switch (activation) {
+    case TensorsToSegmentationCalculatorOptions::NONE:
+      return "NONE";
+    case TensorsToSegmentationCalculatorOptions::SIGMOID:
+      return "SIGMOID";
+    case TensorsToSegmentationCalculatorOptions::SOFTMAX:
+      return "SOFTMAX";
+  }
+  ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
+  return "UNKNOWN";
+}
+
+std::vector<unsigned char> ArrayFloatToUnsignedChar(
+    const std::vector<float>& array) {
+  std::vector<unsigned char> result;
+  result.reserve(array.size());
+  for (int i = 0; i < array.size(); ++i) {
+    result.push_back(static_cast<unsigned char>(array[i]));
+  }
+  return result;
+}
+
+std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values) {
+  std::vector<float> result;
+  result.reserve(values.size() * 4);
+  for (const float& value : values) {
+    result.push_back(value);
+    result.push_back(0);
+    result.push_back(0);
+    result.push_back(value);
+  }
+  return result;
+}
+
+// For GPU tests, the input tensor needs to be moved to GPU, using
+// TensorViewRequestor. After calculation, the output needs to be moved back
+// to CPU, using ToImageCalculator. The output is an ImageFrame.
+mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
+    bool test_gpu,
+    const TensorsToSegmentationCalculatorOptions::Activation& activation) {
+  std::string pre_process = R"pb(
+    node {
+      calculator: "mediapipe.aimatter.TensorViewRequestor"
+      input_stream: "TENSORS:tensors"
+      output_stream: "TENSORS:tensors_gpu"
+      options {
+        [mediapipe.aimatter.TensorViewRequestorOptions.ext] { gpu {} }
+      }
+    }
+  )pb";
+  std::string post_process = R"pb(
+    node {
+      calculator: "FromImageCalculator"
+      input_stream: "IMAGE:image_as_mask_gpu"
+      output_stream: "IMAGE_CPU:image_as_mask"
+    }
+  )pb";
+  return mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
+      absl::Substitute(
+          R"pb(
+            input_stream: "tensors"
+            input_stream: "size" $0
+            node {
+              calculator: "TensorsToSegmentationCalculator"
+              input_stream: "TENSORS:tensors$1"
+              input_stream: "OUTPUT_SIZE:size"
+              output_stream: "MASK:image_as_mask$2"
+              options: {
+                [mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
+                  activation: $3
+                  gpu_origin: TOP_LEFT
+                }
+              }
+            } $4
+          )pb",
+          test_gpu ? pre_process : "", test_gpu ? "_gpu" : "",
+          test_gpu ? "_gpu" : "", ActivationTypeToString(activation),
+          test_gpu ? post_process : ""));
+}
+}  // namespace tensors_to_segmentation_utils
+}  // namespace mediapipe

mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h

@@ -0,0 +1,57 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
+#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
+#include "mediapipe/framework/calculator.pb.h"
+
+namespace mediapipe {
+namespace tensors_to_segmentation_utils {
+std::string ActivationTypeToString(
+    const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
+        activation);
+
+std::vector<unsigned char> ArrayFloatToUnsignedChar(
+    const std::vector<float>& array);
+
+std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values);
+
+mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
+    bool test_gpu,
+    const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
+        activation);
+
+struct FormattingTestCase {
+  std::string test_name;
+  std::vector<float> inputs;
+  std::vector<float> expected_outputs;
+  mediapipe::TensorsToSegmentationCalculatorOptions::Activation activation;
+  int rows = 1;
+  int cols = 1;
+  int rows_new = 1;
+  int cols_new = 1;
+  int channels = 1;
+  double max_abs_diff = 1e-7;
+};
+}  // namespace tensors_to_segmentation_utils
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_

mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils_test.cc

@@ -0,0 +1,50 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
+
+#include <vector>
+
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
+#include "mediapipe/framework/port/gtest.h"
+
+namespace mediapipe::tensors_to_segmentation_utils {
+namespace {
+
+using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
+
+TEST(TensorsToSegmentationCalculatorTestUtilsTest,
+     ActivationTypeToStringWorksCorrectly) {
+  EXPECT_EQ(ActivationTypeToString(Options::NONE), "NONE");
+  EXPECT_EQ(ActivationTypeToString(Options::SIGMOID), "SIGMOID");
+  EXPECT_EQ(ActivationTypeToString(Options::SOFTMAX), "SOFTMAX");
+}
+
+TEST(TensorsToSegmentationCalculatorTestUtilsTest,
+     ArrayFloatToUnsignedCharWorksCorrectly) {
+  std::vector<float> input = {1.0, 2.0, 3.0};
+  std::vector<unsigned char> expected = {1, 2, 3};
+  EXPECT_EQ(ArrayFloatToUnsignedChar(input), expected);
+}
+
+TEST(TensorsToSegmentationCalculatorTestUtilsTest,
+     MakeRedAlphaMatrixWorksCorrectly) {
+  std::vector<float> input = {1.0, 2.0, 3.0};
+  std::vector<float> expected = {1.0, 0.0, 0.0, 1.0, 2.0, 0.0,
+                                 0.0, 2.0, 3.0, 0.0, 0.0, 3.0};
+  EXPECT_EQ(MakeRedAlphaMatrix(input), expected);
+}
+
+}  // namespace
+}  // namespace mediapipe::tensors_to_segmentation_utils

mediapipe/calculators/tensor/tensors_to_segmentation_converter.h

@@ -0,0 +1,43 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
+#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
+
+#include <memory>
+#include <vector>
+
+#include "absl/status/statusor.h"
+#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/tensor.h"
+
+namespace mediapipe {
+
+class TensorsToSegmentationConverter {
+ public:
+  virtual ~TensorsToSegmentationConverter() = default;
+
+  // Converts tensors to image mask.
+  // Returns a unique pointer containing the converted image.
+  // @input_tensors contains the tensors needed to be processed.
+  // @output_width/height describes output dimensions to reshape the output mask
+  // into.
+  virtual absl::StatusOr<std::unique_ptr<Image>> Convert(
+      const std::vector<Tensor>& input_tensors, int output_width,
+      int output_height) = 0;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_

mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc

@@ -0,0 +1,157 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
+
+#include <algorithm>
+#include <cmath>
+#include <memory>
+#include <vector>
+
+#include "absl/status/status.h"
+#include "absl/status/statusor.h"
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
+#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_opencv.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status_macros.h"
+
+namespace mediapipe {
+namespace {
+
+class OpenCvProcessor : public TensorsToSegmentationConverter {
+ public:
+  absl::Status Init(const TensorsToSegmentationCalculatorOptions& options) {
+    options_ = options;
+    return absl::OkStatus();
+  }
+
+  absl::StatusOr<std::unique_ptr<Image>> Convert(
+      const std::vector<Tensor>& input_tensors, int output_width,
+      int output_height) override;
+
+ private:
+  template <class T>
+  absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat);
+
+  TensorsToSegmentationCalculatorOptions options_;
+};
+
+absl::StatusOr<std::unique_ptr<Image>> OpenCvProcessor::Convert(
+    const std::vector<Tensor>& input_tensors, int output_width,
+    int output_height) {
+  MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
+  auto [tensor_height, tensor_width, tensor_channels] = hwc;
+  // Create initial working mask.
+  cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
+
+  // Wrap input tensor.
+  auto raw_input_tensor = &input_tensors[0];
+  auto raw_input_view = raw_input_tensor->GetCpuReadView();
+  const float* raw_input_data = raw_input_view.buffer<float>();
+  cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
+                     CV_MAKETYPE(CV_32F, tensor_channels),
+                     const_cast<float*>(raw_input_data));
+
+  // Process mask tensor and apply activation function.
+  if (tensor_channels == 2) {
+    MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
+  } else if (tensor_channels == 1) {
+    RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
+              options_.activation());  // Requires 2 channels.
+    if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
+        options_.activation())  // Pass-through optimization.
+      tensor_mat.copyTo(small_mask_mat);
+    else
+      MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
+  } else {
+    RET_CHECK_FAIL() << "Unsupported number of tensor channels "
+                     << tensor_channels;
+  }
+
+  // Send out image as CPU packet.
+  std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
+      ImageFormat::VEC32F1, output_width, output_height);
+  auto output_mask = std::make_unique<Image>(mask_frame);
+  auto output_mat = formats::MatView(output_mask.get());
+  // Upsample small mask into output.
+  cv::resize(small_mask_mat, *output_mat,
+             cv::Size(output_width, output_height));
+  return output_mask;
+}
+
+template <class T>
+absl::Status OpenCvProcessor::ApplyActivation(cv::Mat& tensor_mat,
+                                              cv::Mat* small_mask_mat) {
+  // Configure activation function.
+  const int output_layer_index = options_.output_layer_index();
+  using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
+  const auto activation_fn = [&](const cv::Vec2f& mask_value) {
+    float new_mask_value = 0;
+    // TODO consider moving switch out of the loop,
+    // and also avoid float/Vec2f casting.
+    switch (options_.activation()) {
+      case Options::NONE: {
+        new_mask_value = mask_value[0];
+        break;
+      }
+      case Options::SIGMOID: {
+        const float pixel0 = mask_value[0];
+        new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
+        break;
+      }
+      case Options::SOFTMAX: {
+        const float pixel0 = mask_value[0];
+        const float pixel1 = mask_value[1];
+        const float max_pixel = std::max(pixel0, pixel1);
+        const float min_pixel = std::min(pixel0, pixel1);
+        const float softmax_denom =
+            /*exp(max_pixel - max_pixel)=*/1.0f +
+            std::exp(min_pixel - max_pixel);
+        new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
+                         softmax_denom;
+        break;
+      }
+    }
+    return new_mask_value;
+  };
+
+  // Process mask tensor.
+  for (int i = 0; i < tensor_mat.rows; ++i) {
+    for (int j = 0; j < tensor_mat.cols; ++j) {
+      const T& input_pix = tensor_mat.at<T>(i, j);
+      const float mask_value = activation_fn(input_pix);
+      small_mask_mat->at<float>(i, j) = mask_value;
+    }
+  }
+
+  return absl::OkStatus();
+}
+
+}  // namespace
+
+absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
+CreateOpenCvConverter(const TensorsToSegmentationCalculatorOptions& options) {
+  auto converter = std::make_unique<OpenCvProcessor>();
+  MP_RETURN_IF_ERROR(converter->Init(options));
+  return converter;
+}
+
+}  // namespace mediapipe

mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h

@@ -0,0 +1,31 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
+#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
+
+#include <memory>
+
+#include "absl/status/statusor.h"
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
+
+namespace mediapipe {
+// Creates OpenCV tensors-to-segmentation converter.
+absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
+CreateOpenCvConverter(
+    const mediapipe::TensorsToSegmentationCalculatorOptions& options);
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_

mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc

@@ -0,0 +1,52 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
+
+#include <tuple>
+#include <vector>
+
+#include "absl/status/statusor.h"
+#include "mediapipe/framework/port.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+
+int NumGroups(int size, int group_size) {
+  return (size + group_size - 1) / group_size;
+}
+
+bool CanUseGpu() {
+#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
+  // TODO: Configure GPU usage policy in individual calculators.
+  constexpr bool kAllowGpuProcessing = true;
+  return kAllowGpuProcessing;
+#else
+  return false;
+#endif  // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
+}
+
+absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
+    const std::vector<int>& dims) {
+  if (dims.size() == 3) {
+    return std::make_tuple(dims[0], dims[1], dims[2]);
+  } else if (dims.size() == 4) {
+    // BHWC format check B == 1
+    RET_CHECK_EQ(dims[0], 1) << "Expected batch to be 1 for BHWC heatmap";
+    return std::make_tuple(dims[1], dims[2], dims[3]);
+  } else {
+    RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
+  }
+}
+}  // namespace mediapipe

mediapipe/calculators/tensor/tensors_to_segmentation_utils.h

@@ -0,0 +1,34 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
+#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
+
+#include <tuple>
+#include <vector>
+
+#include "absl/status/statusor.h"
+
+namespace mediapipe {
+
+// Commonly used to compute the number of blocks to launch in a kernel.
+int NumGroups(const int size, const int group_size);  // NOLINT
+
+bool CanUseGpu();
+
+absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
+    const std::vector<int>& dims);
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_

mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc

@@ -0,0 +1,63 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
+
+#include <tuple>
+#include <vector>
+
+#include "absl/status/statusor.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+using ::testing::HasSubstr;
+
+TEST(TensorsToSegmentationUtilsTest, NumGroupsWorksProperly) {
+  EXPECT_EQ(NumGroups(13, 4), 4);
+  EXPECT_EQ(NumGroups(4, 13), 1);
+}
+
+TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsWorksProperly) {
+  std::vector<int> dims_3 = {2, 3, 4};
+  absl::StatusOr<std::tuple<int, int, int>> result_1 = GetHwcFromDims(dims_3);
+  MP_ASSERT_OK(result_1);
+  EXPECT_EQ(result_1.value(), (std::make_tuple(2, 3, 4)));
+  std::vector<int> dims_4 = {1, 3, 4, 5};
+  absl::StatusOr<std::tuple<int, int, int>> result_2 = GetHwcFromDims(dims_4);
+  MP_ASSERT_OK(result_2);
+  EXPECT_EQ(result_2.value(), (std::make_tuple(3, 4, 5)));
+}
+
+TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsBatchCheckFail) {
+  std::vector<int> dims_4 = {2, 3, 4, 5};
+  absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_4);
+  EXPECT_FALSE(result.ok());
+  EXPECT_THAT(result.status().message(),
+              HasSubstr("Expected batch to be 1 for BHWC heatmap"));
+}
+
+TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsInvalidShape) {
+  std::vector<int> dims_5 = {1, 2, 3, 4, 5};
+  absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_5);
+  EXPECT_FALSE(result.ok());
+  EXPECT_THAT(result.status().message(),
+              HasSubstr("Invalid shape for segmentation tensor"));
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc

@@ -79,7 +79,7 @@ namespace mpms = mediapipe::mediasequence;
 //   and label and label_id are optional but at least one of them should be set.
 // "IMAGE_${NAME}", "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store
 // prefixed versions of each stream, which allows for multiple image streams to
-// be included. However, the default names are suppored by more tools.
+// be included. However, the default names are supported by more tools.
 //
 // Example config:
 // node {

mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc

@@ -67,8 +67,8 @@ absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet,
 // -- 1-D or 2-D Tensor
 // Output:
 // -- Matrix with the same values as the Tensor
-// If input tensor is 1 dimensional, the ouput Matrix is of (1xn) shape.
+// If input tensor is 1 dimensional, the output Matrix is of (1xn) shape.
-// If input tensor is 2 dimensional (batched), the ouput Matrix is (mxn) shape.
+// If input tensor is 2 dimensional (batched), the output Matrix is (mxn) shape.
 //
 // Example Config
 // node: {

mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator.cc

@@ -15,9 +15,9 @@
 // Calculator converts from one-dimensional Tensor of DT_FLOAT to vector<float>
 // OR from (batched) two-dimensional Tensor of DT_FLOAT to vector<vector<float>.
 
+#include <cstdint>
 #include <memory>
 
-#include "absl/base/integral_types.h"
 #include "mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/status.h"

mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc

@@ -111,8 +111,8 @@ class InferenceState {
 // input_side_packet.
 //
 // The input and output streams are TensorFlow tensors labeled by tags. The tags
-// for the streams are matched to feeds and fetchs in a TensorFlow session using
+// for the streams are matched to feeds and fetches in a TensorFlow session
-// a named_signature.generic_signature in the ModelManifest. The
+// using a named_signature.generic_signature in the ModelManifest. The
 // generic_signature is used as key-value pairs between the MediaPipe tag and
 // the TensorFlow tensor. The signature_name in the options proto determines
 // which named_signature is used. The keys in the generic_signature must be
@@ -128,7 +128,7 @@ class InferenceState {
 // addition. Once batch_size inputs have been provided, the batch will be run
 // and the output tensors sent out on the output streams with timestamps
 // corresponding to the input stream packets. Setting the batch_size to 1
-// completely disables batching, but is indepdent of add_batch_dim_to_tensors.
+// completely disables batching, but is independent of add_batch_dim_to_tensors.
 //
 // The TensorFlowInferenceCalculator also support feeding states recurrently for
 // RNNs and LSTMs. Simply set the recurrent_tag_pair options to define the

mediapipe/calculators/tensorflow/tensorflow_inference_calculator.proto

@@ -42,7 +42,7 @@ message TensorFlowInferenceCalculatorOptions {
   // If the 0th dimension is the batch dimension, then the tensors are
   // concatenated on that dimension. If the 0th is a data dimension, then a 0th
   // dimension is added before concatenating. If added, the extra dimension is
-  // removed before outputing the tensor. Examples of each case: If you want
+  // removed before outputting the tensor. Examples of each case: If you want
   // to batch spectra of audio over time for an LSTM, a time-frequency
   // representation has a 0th dimension as the batch dimension. If you want to
   // batch frames of video that are [width, height, channels], the batch

mediapipe/calculators/tflite/testdata/README.md

@@ -1,2 +1,2 @@
 The model files add.bin, add_quantized.bin
-(and corresponding metatada json files) come from tensorflow/lite/testdata/
+(and corresponding metadata json files) come from tensorflow/lite/testdata/

mediapipe/calculators/tflite/tflite_converter_calculator.cc

@@ -95,7 +95,7 @@ struct GPUData {
 // into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer
 // or MTLBuffer.
 //
-// This calculator is designed to be used with the TfLiteInferenceCalcualtor,
+// This calculator is designed to be used with the TfLiteInferenceCalculator,
 // as a pre-processing step for calculator inputs.
 //
 // IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1],

mediapipe/calculators/tflite/tflite_converter_calculator.proto

@@ -31,7 +31,7 @@ message TfLiteConverterCalculatorOptions {
   // Custom settings to override the internal scaling factors `div` and `sub`.
   // Both values must be set to non-negative values. Will only take effect on
   // CPU AND when |use_custom_normalization| is set to true. When these custom
-  // values take effect, the |zero_center| setting above will be overriden, and
+  // values take effect, the |zero_center| setting above will be overridden, and
   // the normalized_value will be calculated as:
   // normalized_value = input / custom_div - custom_sub.
   optional bool use_custom_normalization = 6 [default = false];

mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto

@@ -25,7 +25,7 @@ message TfLiteTensorsToClassificationCalculatorOptions {
     optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463;
   }
 
-  // Score threshold for perserving the class.
+  // Score threshold for preserving the class.
   optional float min_score_threshold = 1;
   // Number of highest scoring labels to output.  If top_k is not positive then
   // all labels are used.

mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc

@@ -116,7 +116,7 @@ void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors,
 //               tensors can have 2 or 3 tensors. First tensor is the predicted
 //               raw boxes/keypoints. The size of the values must be (num_boxes
 //               * num_predicted_values). Second tensor is the score tensor. The
-//               size of the valuse must be (num_boxes * num_classes). It's
+//               size of the values must be (num_boxes * num_classes). It's
 //               optional to pass in a third tensor for anchors (e.g. for SSD
 //               models) depend on the outputs of the detection model. The size
 //               of anchor tensor must be (num_boxes * 4).

mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto

@@ -69,6 +69,6 @@ message TfLiteTensorsToDetectionsCalculatorOptions {
   // representation has a bottom-left origin (e.g., in OpenGL).
   optional bool flip_vertically = 18 [default = false];
 
-  // Score threshold for perserving decoded detections.
+  // Score threshold for preserving decoded detections.
   optional float min_score_thresh = 19;
 }

mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc

@@ -158,7 +158,7 @@ absl::Status TfLiteTensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
     RET_CHECK(options_.has_input_image_height() &&
               options_.has_input_image_width())
         << "Must provide input width/height for using flip_vertically option "
-           "when outputing landmarks in absolute coordinates.";
+           "when outputting landmarks in absolute coordinates.";
   }
 
   flip_horizontally_ =

mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
 networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

mediapipe/examples/android/solutions/gradlew

@@ -83,10 +83,8 @@ done
 # This is normally unused
 # shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
-
+APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD=maximum
@@ -133,10 +131,13 @@ location of your Java installation."
     fi
 else
     JAVACMD=java
-    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
 
 Please set the JAVA_HOME variable in your environment to match the
 location of your Java installation."
+    fi
 fi
 
 # Increase the maximum file descriptors if we can.
@@ -144,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
         # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045 
+        # shellcheck disable=SC2039,SC3045
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
@@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
       '' | soft) :;; #(
       *)
         # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
-        # shellcheck disable=SC3045 
+        # shellcheck disable=SC2039,SC3045
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
@@ -197,11 +198,15 @@ if "$cygwin" || "$msys" ; then
     done
 fi
 
-# Collect all arguments for the java command;
+
-#   * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-#     shell script including quotes and variable substitutions, so put them in
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
-#     double quotes to make sure that they get re-expanded; and
+
-#   * put everything else in single quotes, so that it's not re-expanded.
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
 
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \

mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc

@@ -55,7 +55,7 @@ absl::Status RunMPPGraph() {
   for (const std::string& kv_pair : kv_pairs) {
     std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
     RET_CHECK(name_and_value.size() == 2);
-    RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0]));
+    RET_CHECK(!input_side_packets.contains(name_and_value[0]));
     std::string input_side_packet_contents;
     MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
         name_and_value[1], &input_side_packet_contents));

mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc

@@ -56,7 +56,7 @@ absl::Status RunMPPGraph() {
   for (const std::string& kv_pair : kv_pairs) {
     std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
     RET_CHECK(name_and_value.size() == 2);
-    RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0]));
+    RET_CHECK(!input_side_packets.contains(name_and_value[0]));
     std::string input_side_packet_contents;
     MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
         name_and_value[1], &input_side_packet_contents));

mediapipe/framework/api2/builder.h

@@ -330,6 +330,14 @@ using MultiSideDestination = MultiPort<SideDestination<T>>;
 
 class NodeBase {
  public:
+  NodeBase() = default;
+  ~NodeBase() = default;
+  NodeBase(NodeBase&&) = default;
+  NodeBase& operator=(NodeBase&&) = default;
+  // Explicitly delete copies to improve error messages.
+  NodeBase(const NodeBase&) = delete;
+  NodeBase& operator=(const NodeBase&) = delete;
+
   // TODO: right now access to an indexed port is made directly by
   // specifying both a tag and an index. It would be better to represent this
   // as a two-step lookup, first getting a multi-port, and then accessing one
@@ -585,6 +593,14 @@ class PacketGenerator {
 
 class Graph {
  public:
+  Graph() = default;
+  ~Graph() = default;
+  Graph(Graph&&) = default;
+  Graph& operator=(Graph&&) = default;
+  // Explicitly delete copies to improve error messages.
+  Graph(const Graph&) = delete;
+  Graph& operator=(const Graph&) = delete;
+
   void SetType(std::string type) { type_ = std::move(type); }
 
   // Creates a node of a specific type. Should be used for calculators whose

mediapipe/framework/formats/BUILD

@@ -124,6 +124,15 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "ahwb_view",
+    hdrs = ["ahwb_view.h"],
+    deps = [
+        "//mediapipe/framework:port",
+        "//mediapipe/gpu:gpu_buffer_storage",
+    ],
+)
+
 cc_library(
     name = "affine_transform",
     srcs = ["affine_transform.cc"],

mediapipe/framework/formats/ahwb_view.h

@@ -0,0 +1,54 @@
+#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
+#define MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
+
+#include "mediapipe/framework/port.h"
+#ifdef MEDIAPIPE_GPU_BUFFER_USE_AHWB
+#include <android/hardware_buffer.h>
+
+#include "mediapipe/gpu/gpu_buffer_storage.h"
+
+namespace mediapipe {
+
+// Wrapper to facilitate short lived access to Android Hardware Buffer objects.
+// Intended use cases:
+// - Extracting an AHWB for processing in another library after it's produced by
+// MediaPipe.
+// - Sending AHWBs to compute devices that are able to map the memory for their
+// own usage.
+// The AHWB abstractions in GpuBuffer and Tensor are likely more suitable for
+// other CPU/GPU uses of AHWBs.
+class AhwbView {
+ public:
+  explicit AhwbView(AHardwareBuffer* handle) : handle_(handle) {}
+  // Non-copyable
+  AhwbView(const AhwbView&) = delete;
+  AhwbView& operator=(const AhwbView&) = delete;
+  // Non-movable
+  AhwbView(AhwbView&&) = delete;
+
+  // Only supports synchronous usage. All users of GetHandle must finish
+  // accessing the buffer before this view object is destroyed to avoid race
+  // conditions.
+  // TODO: Support asynchronous usage.
+  const AHardwareBuffer* GetHandle() const { return handle_; }
+
+ private:
+  const AHardwareBuffer* handle_;
+};
+
+namespace internal {
+// Makes this class available as a GpuBuffer view.
+template <>
+class ViewProvider<AhwbView> {
+ public:
+  virtual ~ViewProvider() = default;
+  virtual const AhwbView GetReadView(types<AhwbView>) const = 0;
+  virtual AhwbView GetWriteView(types<AhwbView>) = 0;
+};
+
+}  // namespace internal
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_GPU_BUFFER_USE_AHWB
+#endif  // MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_

mediapipe/framework/port.h

@@ -50,7 +50,7 @@
 // but may or may not still be able to run other OpenGL code.
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) &&                                  \
     (defined(__APPLE__) || defined(__EMSCRIPTEN__) || MEDIAPIPE_DISABLE_GPU || \
-     MEDIAPIPE_USING_SWIFTSHADER)
+     MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
 #define MEDIAPIPE_DISABLE_GL_COMPUTE
 #endif
 
@@ -104,4 +104,9 @@
 #endif
 #endif  // MEDIAPIPE_HAS_RTTI
 
+// AHardware buffers are only available since Android API 26.
+#if (__ANDROID_API__ >= 26)
+#define MEDIAPIPE_GPU_BUFFER_USE_AHWB 1
+#endif
+
 #endif  // MEDIAPIPE_FRAMEWORK_PORT_H_

mediapipe/framework/tool/BUILD

@@ -616,6 +616,7 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/port:gtest_main",
         "//mediapipe/framework/port:parse_text_proto",
+        "@com_google_absl//absl/functional:bind_front",
         "@com_google_absl//absl/strings",
     ],
 )
@@ -856,6 +857,7 @@ cc_library(
 mediapipe_cc_test(
     name = "switch_demux_calculator_test",
     srcs = ["switch_demux_calculator_test.cc"],
+    requires_full_emulation = False,
     deps = [
         ":container_util",
         ":switch_demux_calculator",
@@ -891,6 +893,7 @@ cc_library(
 mediapipe_cc_test(
     name = "switch_mux_calculator_test",
     srcs = ["switch_mux_calculator_test.cc"],
+    requires_full_emulation = False,
     deps = [
         ":container_util",
         ":switch_mux_calculator",

mediapipe/framework/tool/sink_test.cc

@@ -17,6 +17,7 @@
 #include <memory>
 #include <vector>
 
+#include "absl/functional/bind_front.h"
 #include "absl/strings/string_view.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"

mediapipe/framework/tool/validate_name.cc

@@ -134,7 +134,7 @@ absl::Status ParseTagAndName(absl::string_view tag_and_name, std::string* tag,
   RET_CHECK(name);
   absl::Status tag_status = absl::OkStatus();
   absl::Status name_status = absl::UnknownError("");
-  int name_index = 0;
+  int name_index = -1;
   std::vector<std::string> v = absl::StrSplit(tag_and_name, ':');
   if (v.size() == 1) {
     name_status = ValidateName(v[0]);
@@ -143,7 +143,7 @@ absl::Status ParseTagAndName(absl::string_view tag_and_name, std::string* tag,
     tag_status = ValidateTag(v[0]);
     name_status = ValidateName(v[1]);
     name_index = 1;
-  }
+  }  // else omitted, name_index == -1, triggering error.
   if (name_index == -1 || tag_status != absl::OkStatus() ||
       name_status != absl::OkStatus()) {
     tag->clear();

mediapipe/gpu/BUILD

@@ -511,11 +511,19 @@ cc_library(
         ],
     }),
     deps = [
+        ":gl_base_hdr",
+        ":gl_context",
         ":gl_texture_buffer",
+        ":gl_texture_view",
         ":gpu_buffer_format",
         ":gpu_buffer_storage",
         ":image_frame_view",
+        "//mediapipe/framework:port",
+        "//mediapipe/framework/formats:ahwb_view",
         "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:ret_check",
+        "//third_party/GL:EGL_headers",
+        "@com_google_absl//absl/log:absl_check",
         "@com_google_absl//absl/strings:str_format",
     ],
 )
@@ -526,12 +534,14 @@ mediapipe_proto_library(
     visibility = ["//visibility:public"],
 )
 
-objc_library(
+cc_library(
     name = "pixel_buffer_pool_util",
-    srcs = ["pixel_buffer_pool_util.mm"],
+    srcs = ["pixel_buffer_pool_util.cc"],
     hdrs = ["pixel_buffer_pool_util.h"],
     copts = [
+        "-x objective-c++",
         "-Wno-shorten-64-to-32",
+        "-fobjc-arc",  # enable reference-counting
     ],
     visibility = ["//visibility:public"],
     deps = [
@@ -542,13 +552,14 @@ objc_library(
     ],
 )
 
-objc_library(
+cc_library(
     name = "metal_shared_resources",
-    srcs = ["metal_shared_resources.mm"],
+    srcs = ["metal_shared_resources.cc"],
     hdrs = ["metal_shared_resources.h"],
     copts = [
         "-x objective-c++",
         "-Wno-shorten-64-to-32",
+        "-fobjc-arc",  # enable reference-counting
     ],
     features = ["-layering_check"],
     visibility = ["//visibility:public"],
@@ -557,15 +568,17 @@ objc_library(
         "@google_toolbox_for_mac//:GTM_Defines",
     ] + [
     ],
+    alwayslink = 1,
 )
 
-objc_library(
+cc_library(
     name = "MPPMetalUtil",
-    srcs = ["MPPMetalUtil.mm"],
+    srcs = ["MPPMetalUtil.cc"],
     hdrs = ["MPPMetalUtil.h"],
     copts = [
         "-x objective-c++",
         "-Wno-shorten-64-to-32",
+        "-fobjc-arc",  # enable reference-counting
     ],
     visibility = ["//visibility:public"],
     deps = [
@@ -575,6 +588,7 @@ objc_library(
         "@com_google_absl//absl/time",
         "@google_toolbox_for_mac//:GTM_Defines",
     ],
+    alwayslink = 1,
 )
 
 mediapipe_proto_library(
@@ -655,6 +669,8 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/log:absl_check",
+        "@com_google_absl//absl/log:absl_log",
+        "@com_google_absl//absl/status",
     ] + select({
         "//conditions:default": [],
         "//mediapipe:apple": [
@@ -857,12 +873,14 @@ cc_library(
     }),
 )
 
-objc_library(
+cc_library(
     name = "MPPMetalHelper",
-    srcs = ["MPPMetalHelper.mm"],
+    srcs = ["MPPMetalHelper.cc"],
     hdrs = ["MPPMetalHelper.h"],
     copts = [
         "-Wno-shorten-64-to-32",
+        "-x objective-c++",
+        "-fobjc-arc",
     ],
     features = ["-layering_check"],
     visibility = ["//visibility:public"],
@@ -1215,9 +1233,13 @@ mediapipe_cc_test(
     ],
     requires_full_emulation = True,
     deps = [
+        ":gl_texture_buffer",
+        ":gl_texture_util",
         ":gpu_buffer_format",
         ":gpu_buffer_storage_ahwb",
+        ":gpu_test_base",
         "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/tool:test_util",
     ],
 )
 

mediapipe/gpu/MPPMetalHelper.cc

@@ -14,15 +14,14 @@
 
 #import "mediapipe/gpu/MPPMetalHelper.h"
 
+#import "GTMDefines.h"
 #include "absl/log/absl_check.h"
 #include "absl/log/absl_log.h"
+#include "mediapipe/framework/port/ret_check.h"
 #import "mediapipe/gpu/gpu_buffer.h"
 #import "mediapipe/gpu/gpu_service.h"
 #import "mediapipe/gpu/graph_support.h"
 #import "mediapipe/gpu/metal_shared_resources.h"
-#import "GTMDefines.h"
-
-#include "mediapipe/framework/port/ret_check.h"
 
 @interface MPPMetalHelper () {
   mediapipe::GpuResources* _gpuResources;
@@ -31,7 +30,8 @@
 
 namespace mediapipe {
 
-// Using a C++ class so it can be declared as a friend of LegacyCalculatorSupport.
+// Using a C++ class so it can be declared as a friend of
+// LegacyCalculatorSupport.
 class MetalHelperLegacySupport {
  public:
   static CalculatorContract* GetCalculatorContract() {
@@ -61,7 +61,8 @@ class MetalHelperLegacySupport {
 
 - (instancetype)initWithCalculatorContext:(mediapipe::CalculatorContext*)cc {
   if (!cc) return nil;
-  return [self initWithGpuResources:&cc->Service(mediapipe::kGpuService).GetObject()];
+  return [self
+      initWithGpuResources:&cc->Service(mediapipe::kGpuService).GetObject()];
 }
 
 + (absl::Status)updateContract:(mediapipe::CalculatorContract*)cc {
@@ -77,7 +78,8 @@ class MetalHelperLegacySupport {
 }
 
 // Legacy support.
-- (instancetype)initWithSidePackets:(const mediapipe::PacketSet&)inputSidePackets {
+- (instancetype)initWithSidePackets:
+    (const mediapipe::PacketSet&)inputSidePackets {
   auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContext();
   if (cc) {
     ABSL_CHECK_EQ(&inputSidePackets, &cc->InputSidePackets());
@@ -85,16 +87,19 @@ class MetalHelperLegacySupport {
   }
 
   // TODO: remove when we can.
-  ABSL_LOG(WARNING) << "CalculatorContext not available. If this calculator uses "
+  ABSL_LOG(WARNING)
-                       "CalculatorBase, call initWithCalculatorContext instead.";
+      << "CalculatorContext not available. If this calculator uses "
+         "CalculatorBase, call initWithCalculatorContext instead.";
   mediapipe::GpuSharedData* gpu_shared =
-      inputSidePackets.Tag(mediapipe::kGpuSharedTagName).Get<mediapipe::GpuSharedData*>();
+      inputSidePackets.Tag(mediapipe::kGpuSharedTagName)
+          .Get<mediapipe::GpuSharedData*>();
 
   return [self initWithGpuResources:gpu_shared->gpu_resources.get()];
 }
 
 // Legacy support.
-+ (absl::Status)setupInputSidePackets:(mediapipe::PacketTypeSet*)inputSidePackets {
++ (absl::Status)setupInputSidePackets:
+    (mediapipe::PacketTypeSet*)inputSidePackets {
   auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContract();
   if (cc) {
     ABSL_CHECK_EQ(inputSidePackets, &cc->InputSidePackets());
@@ -102,12 +107,12 @@ class MetalHelperLegacySupport {
   }
 
   // TODO: remove when we can.
-  ABSL_LOG(WARNING) << "CalculatorContract not available. If you're calling this "
+  ABSL_LOG(WARNING)
-                       "from a GetContract method, call updateContract instead.";
+      << "CalculatorContract not available. If you're calling this "
+         "from a GetContract method, call updateContract instead.";
   auto id = inputSidePackets->GetId(mediapipe::kGpuSharedTagName, 0);
-  RET_CHECK(id.IsValid())
+  RET_CHECK(id.IsValid()) << "A " << mediapipe::kGpuSharedTagName
-      << "A " << mediapipe::kGpuSharedTagName
+                          << " input side packet is required here.";
-      << " input side packet is required here.";
   inputSidePackets->Get(id).Set<mediapipe::GpuSharedData*>();
   return absl::OkStatus();
 }
@@ -125,10 +130,12 @@ class MetalHelperLegacySupport {
 }
 
 - (id<MTLCommandBuffer>)commandBuffer {
-  return [_gpuResources->metal_shared().resources().mtlCommandQueue commandBuffer];
+  return
+      [_gpuResources->metal_shared().resources().mtlCommandQueue commandBuffer];
 }
 
-- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer
+- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:
+                         (const mediapipe::GpuBuffer&)gpuBuffer
                                                plane:(size_t)plane {
   CVPixelBufferRef pixel_buffer = mediapipe::GetCVPixelBufferRef(gpuBuffer);
   OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer);
@@ -178,41 +185,48 @@ class MetalHelperLegacySupport {
   CVMetalTextureRef texture;
   CVReturn err = CVMetalTextureCacheCreateTextureFromImage(
       NULL, _gpuResources->metal_shared().resources().mtlTextureCache,
-      mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width, height, plane,
+      mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width,
-      &texture);
+      height, plane, &texture);
   ABSL_CHECK_EQ(err, kCVReturnSuccess);
   return texture;
 }
 
-- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer {
+- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:
+    (const mediapipe::GpuBuffer&)gpuBuffer {
   return [self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:0];
 }
 
-- (id<MTLTexture>)metalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer {
+- (id<MTLTexture>)metalTextureWithGpuBuffer:
+    (const mediapipe::GpuBuffer&)gpuBuffer {
   return [self metalTextureWithGpuBuffer:gpuBuffer plane:0];
 }
 
-- (id<MTLTexture>)metalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer
+- (id<MTLTexture>)metalTextureWithGpuBuffer:
-                                 plane:(size_t)plane {
+                      (const mediapipe::GpuBuffer&)gpuBuffer
+                                      plane:(size_t)plane {
   CFHolder<CVMetalTextureRef> cvTexture;
   cvTexture.adopt([self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:plane]);
   return CVMetalTextureGetTexture(*cvTexture);
 }
 
-- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width height:(int)height {
+- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width
+                                             height:(int)height {
   return _gpuResources->gpu_buffer_pool().GetBuffer(width, height);
 }
 
 - (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width
-                                         height:(int)height
+                                             height:(int)height
-                                         format:(mediapipe::GpuBufferFormat)format {
+                                             format:(mediapipe::GpuBufferFormat)
+                                                        format {
   return _gpuResources->gpu_buffer_pool().GetBuffer(width, height, format);
 }
 
-- (id<MTLLibrary>)newLibraryWithResourceName:(NSString*)name error:(NSError * _Nullable *)error {
+- (id<MTLLibrary>)newLibraryWithResourceName:(NSString*)name
+                                       error:(NSError* _Nullable*)error {
   return [_gpuResources->metal_shared().resources().mtlDevice
-      newLibraryWithFile:[[NSBundle bundleForClass:[self class]] pathForResource:name
+      newLibraryWithFile:[[NSBundle bundleForClass:[self class]]
-                                                                          ofType:@"metallib"]
+                             pathForResource:name
+                                      ofType:@"metallib"]
                    error:error];
 }
 

mediapipe/gpu/MPPMetalUtil.cc

@@ -69,10 +69,10 @@
   while (!bufferCompleted) {
     auto duration = absl::Now() - start_time;
     // If the spin-lock takes more than 5 ms then go to blocking wait:
-    // - it frees the CPU core for another threads: increase the performance/decrease power
+    // - it frees the CPU core for another threads: increase the
-    // consumption.
+    // performance/decrease power consumption.
-    // - if a driver thread that notifies that the GPU buffer is completed has lower priority then
+    // - if a driver thread that notifies that the GPU buffer is completed has
-    // the CPU core is allocated for the thread.
+    // lower priority then the CPU core is allocated for the thread.
     if (duration >= absl::Milliseconds(5)) {
       [commandBuffer waitUntilCompleted];
       break;

mediapipe/gpu/gl_calculator_helper.cc

@@ -57,8 +57,8 @@ void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) {
 
 // static
 absl::Status GlCalculatorHelper::UpdateContract(CalculatorContract* cc,
-                                                bool requesst_gpu_as_optional) {
+                                                bool request_gpu_as_optional) {
-  if (requesst_gpu_as_optional) {
+  if (request_gpu_as_optional) {
     cc->UseService(kGpuService).Optional();
   } else {
     cc->UseService(kGpuService);

mediapipe/gpu/gl_calculator_helper.h

@@ -68,7 +68,7 @@ class GlCalculatorHelper {
   // This method can be called from GetContract to set up the needed GPU
   // resources.
   static absl::Status UpdateContract(CalculatorContract* cc,
-                                     bool requesst_gpu_as_optional = false);
+                                     bool request_gpu_as_optional = false);
 
   // This method can be called from FillExpectations to set the correct types
   // for the shared GL input side packet(s).

mediapipe/gpu/gl_texture_buffer.cc

@@ -14,6 +14,8 @@
 
 #include "mediapipe/gpu/gl_texture_buffer.h"
 
+#include <cstdint>
+
 #include "absl/log/absl_check.h"
 #include "absl/log/absl_log.h"
 #include "mediapipe/framework/formats/image_frame.h"
@@ -131,6 +133,13 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) {
       SymbolAvailable(&glTexStorage2D)) {
     ABSL_CHECK(data == nullptr) << "unimplemented";
     glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
+  } else if (info.immutable) {
+    ABSL_CHECK(SymbolAvailable(&glTexStorage2D) &&
+               context->GetGlVersion() != GlVersion::kGLES2)
+        << "Immutable GpuBuffer format requested is not supported in this "
+        << "GlContext. Format was " << static_cast<uint32_t>(format_);
+    ABSL_CHECK(data == nullptr) << "unimplemented";
+    glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
   } else {
     glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_,
                  height_, 0 /* border */, info.gl_format, info.gl_type, data);

mediapipe/gpu/gl_texture_buffer.h

@@ -19,6 +19,7 @@
 #define MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_
 
 #include <atomic>
+#include <memory>
 
 #include "absl/memory/memory.h"
 #include "mediapipe/framework/formats/image_frame.h"

mediapipe/gpu/gl_thread_collector.h

@@ -17,7 +17,7 @@
 
 #include <cstdlib>
 
-#if defined(MEDIAPIPE_USING_SWIFTSHADER)
+#if defined(MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
 #define MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR 1
 #endif
 

mediapipe/gpu/gpu_buffer_format.cc

@@ -35,6 +35,10 @@ namespace mediapipe {
 #endif  // GL_HALF_FLOAT_OES
 #endif  // __EMSCRIPTEN__
 
+#ifndef GL_RGBA8
+#define GL_RGBA8 0x8058
+#endif  // GL_RGBA8
+
 #if !MEDIAPIPE_DISABLE_GPU
 #ifdef GL_ES_VERSION_2_0
 static void AdaptGlTextureInfoForGLES2(GlTextureInfo* info) {
@@ -163,6 +167,14 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
            {
                {GL_RGBA32F, GL_RGBA, GL_FLOAT, 1},
            }},
+          {GpuBufferFormat::kImmutableRGBAFloat128,
+           {
+               {GL_RGBA32F, GL_RGBA, GL_FLOAT, 1, true /* immutable */},
+           }},
+          {GpuBufferFormat::kImmutableRGBA32,
+           {
+               {GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE, 1, true /* immutable */},
+           }},
       }};
 
   static const auto* gles2_format_info = ([] {
@@ -206,6 +218,7 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
 
 ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
   switch (format) {
+    case GpuBufferFormat::kImmutableRGBA32:
     case GpuBufferFormat::kBGRA32:
       // TODO: verify we are handling order of channels correctly.
       return ImageFormat::SRGBA;
@@ -221,10 +234,11 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
       return ImageFormat::SRGB;
     case GpuBufferFormat::kTwoComponentFloat32:
       return ImageFormat::VEC32F2;
+    case GpuBufferFormat::kImmutableRGBAFloat128:
     case GpuBufferFormat::kRGBAFloat128:
       return ImageFormat::VEC32F4;
     case GpuBufferFormat::kRGBA32:
-      // TODO: this likely maps to ImageFormat::SRGBA
+      return ImageFormat::SRGBA;
     case GpuBufferFormat::kGrayHalf16:
     case GpuBufferFormat::kOneComponent8Alpha:
     case GpuBufferFormat::kOneComponent8Red:

mediapipe/gpu/gpu_buffer_format.h

@@ -53,6 +53,10 @@ enum class GpuBufferFormat : uint32_t {
   kRGB24 = 0x00000018,  // Note: prefer BGRA32 whenever possible.
   kRGBAHalf64 = MEDIAPIPE_FOURCC('R', 'G', 'h', 'A'),
   kRGBAFloat128 = MEDIAPIPE_FOURCC('R', 'G', 'f', 'A'),
+  // Immutable version of kRGBA32
+  kImmutableRGBA32 = MEDIAPIPE_FOURCC('4', 'C', 'I', '8'),
+  // Immutable version of kRGBAFloat128
+  kImmutableRGBAFloat128 = MEDIAPIPE_FOURCC('4', 'C', 'I', 'f'),
   // 8-bit Y plane + interleaved 8-bit U/V plane with 2x2 subsampling.
   kNV12 = MEDIAPIPE_FOURCC('N', 'V', '1', '2'),
   // 8-bit Y plane + interleaved 8-bit V/U plane with 2x2 subsampling.
@@ -78,6 +82,9 @@ struct GlTextureInfo {
   // For multiplane buffers, this represents how many times smaller than
   // the nominal image size a plane is.
   int downscale;
+  // For GLES3.1+ compute shaders, users may explicitly request immutable
+  // textures.
+  bool immutable = false;
 };
 
 const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
@@ -121,6 +128,8 @@ inline OSType CVPixelFormatForGpuBufferFormat(GpuBufferFormat format) {
       return kCVPixelFormatType_64RGBAHalf;
     case GpuBufferFormat::kRGBAFloat128:
       return kCVPixelFormatType_128RGBAFloat;
+    case GpuBufferFormat::kImmutableRGBA32:
+    case GpuBufferFormat::kImmutableRGBAFloat128:
     case GpuBufferFormat::kNV12:
     case GpuBufferFormat::kNV21:
     case GpuBufferFormat::kI420:

mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc

@@ -151,7 +151,7 @@ static std::shared_ptr<GpuBufferStorageCvPixelBuffer> ConvertFromImageFrame(
     std::shared_ptr<GpuBufferStorageImageFrame> frame) {
   auto status_or_buffer =
       CreateCVPixelBufferForImageFrame(frame->image_frame());
-  ABSL_CHECK(status_or_buffer.ok());
+  ABSL_CHECK_OK(status_or_buffer);
   return std::make_shared<GpuBufferStorageCvPixelBuffer>(
       std::move(status_or_buffer).value());
 }

mediapipe/gpu/gpu_origin.proto

@@ -16,6 +16,9 @@ syntax = "proto3";
 
 package mediapipe;
 
+option java_package = "com.google.mediapipe.gpu";
+option java_outer_classname = "GpuOriginProto";
+
 message GpuOrigin {
   enum Mode {
     DEFAULT = 0;

mediapipe/gpu/gpu_shared_data_internal.cc

@@ -14,10 +14,14 @@
 
 #include "mediapipe/gpu/gpu_shared_data_internal.h"
 
+#include <memory>
+#include <utility>
+
 #include "absl/base/attributes.h"
 #include "absl/log/absl_check.h"
+#include "absl/log/absl_log.h"
+#include "absl/status/status.h"
 #include "mediapipe/framework/deps/no_destructor.h"
-#include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/gpu/gl_context.h"
 #include "mediapipe/gpu/gl_context_options.pb.h"
 #include "mediapipe/gpu/graph_support.h"
@@ -83,8 +87,25 @@ GpuResources::StatusOrGpuResources GpuResources::Create(
 }
 
 GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
+    : gl_key_context_(new GlContextMapType(),
+                      [](auto* map) {
+                        // This flushes all pending jobs in all GL contexts,
+                        // ensuring that all GL contexts not referenced
+                        // elsewhere are destroyed as part of this destructor.
+                        // Failure to do this may cause GL threads to outlast
+                        // this destructor and execute jobs after the
+                        // GpuResources object is destroyed.
+                        for (auto& [key, context] : *map) {
+                          const auto status = std::move(context)->Run(
+                              []() { return absl::OkStatus(); });
+                          ABSL_LOG_IF(ERROR, !status.ok())
+                              << "Failed to flush GlContext jobs: " << status;
+                        }
+                        delete map;
+                      })
 #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
-    : texture_caches_(std::make_shared<CvTextureCacheManager>()),
+      ,
+      texture_caches_(std::make_shared<CvTextureCacheManager>()),
       gpu_buffer_pool_(
           [tc = texture_caches_](const internal::GpuBufferSpec& spec,
                                  const MultiPoolOptions& options) {
@@ -92,7 +113,7 @@ GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
           })
 #endif  // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
 {
-  gl_key_context_[SharedContextKey()] = gl_context;
+  gl_key_context_->insert({SharedContextKey(), gl_context});
   named_executors_[kGpuExecutorName] =
       std::make_shared<GlContextExecutor>(gl_context.get());
 #if __APPLE__
@@ -104,6 +125,15 @@ GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
 }
 
 GpuResources::~GpuResources() {
+  // This flushes all pending jobs in all GL contexts,
+  // ensuring that all existing jobs, which may refer GpuResource and kept their
+  // gpu resources (e.g. GpuResources::gpu_buffer_pool_) through a raw pointer,
+  // have finished before kept gpu resources get deleted.
+  for (auto& [key, context] : *gl_key_context_) {
+    const auto status = context->Run([]() { return absl::OkStatus(); });
+    ABSL_LOG_IF(ERROR, !status.ok())
+        << "Failed to flush GlContext jobs: " << status;
+  }
 #if __APPLE__
   // Note: on Apple platforms, this object contains Objective-C objects.
   // The destructor will release them, but ARC must be on.
@@ -111,7 +141,7 @@ GpuResources::~GpuResources() {
 #error This file must be built with ARC.
 #endif
 #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
-  for (auto& kv : gl_key_context_) {
+  for (auto& kv : *gl_key_context_) {
     texture_caches_->UnregisterTextureCache(kv.second->cv_texture_cache());
   }
 #endif  // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
@@ -173,23 +203,24 @@ absl::Status GpuResources::PrepareGpuNode(CalculatorNode* node) {
 const std::shared_ptr<GlContext>& GpuResources::gl_context(
     CalculatorContext* cc) {
   if (cc) {
-    auto it = gl_key_context_.find(node_key_[cc->NodeName()]);
+    auto it = gl_key_context_->find(node_key_[cc->NodeName()]);
-    if (it != gl_key_context_.end()) {
+    if (it != gl_key_context_->end()) {
       return it->second;
     }
   }
 
-  return gl_key_context_[SharedContextKey()];
+  return gl_key_context_->at(SharedContextKey());
 }
 
 GlContext::StatusOrGlContext GpuResources::GetOrCreateGlContext(
     const std::string& key) {
-  auto it = gl_key_context_.find(key);
+  auto it = gl_key_context_->find(key);
-  if (it == gl_key_context_.end()) {
+  if (it == gl_key_context_->end()) {
-    MP_ASSIGN_OR_RETURN(std::shared_ptr<GlContext> new_context,
+    MP_ASSIGN_OR_RETURN(
-                        GlContext::Create(*gl_key_context_[SharedContextKey()],
+        std::shared_ptr<GlContext> new_context,
-                                          kGlContextUseDedicatedThread));
+        GlContext::Create(*gl_key_context_->at(SharedContextKey()),
-    it = gl_key_context_.emplace(key, new_context).first;
+                          kGlContextUseDedicatedThread));
+    it = gl_key_context_->emplace(key, new_context).first;
 #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
     texture_caches_->RegisterTextureCache(it->second->cv_texture_cache());
 #endif  // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER

mediapipe/gpu/gpu_shared_data_internal.h

@@ -21,6 +21,8 @@
 #ifndef MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_
 #define MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_
 
+#include <memory>
+
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/framework/calculator_node.h"
 #include "mediapipe/framework/executor.h"
@@ -82,7 +84,10 @@ class GpuResources {
   const std::string& ContextKey(const std::string& canonical_node_name);
 
   std::map<std::string, std::string> node_key_;
-  std::map<std::string, std::shared_ptr<GlContext>> gl_key_context_;
+
+  using GlContextMapType = std::map<std::string, std::shared_ptr<GlContext>>;
+  std::unique_ptr<GlContextMapType, void (*)(GlContextMapType*)>
+      gl_key_context_;
 
 #ifdef MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
   std::shared_ptr<CvTextureCacheManager> texture_caches_;

mediapipe/gpu/gpu_test_base.h

@@ -15,6 +15,9 @@
 #ifndef MEDIAPIPE_GPU_GPU_TEST_BASE_H_
 #define MEDIAPIPE_GPU_GPU_TEST_BASE_H_
 
+#include <functional>
+#include <memory>
+
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/gpu/gl_calculator_helper.h"
@@ -22,9 +25,9 @@
 
 namespace mediapipe {
 
-class GpuTestBase : public ::testing::Test {
+class GpuTestEnvironment {
  protected:
-  GpuTestBase() { helper_.InitializeForTest(gpu_resources_.get()); }
+  GpuTestEnvironment() { helper_.InitializeForTest(gpu_resources_.get()); }
 
   void RunInGlContext(std::function<void(void)> gl_func) {
     helper_.RunInGlContext(std::move(gl_func));
@@ -35,6 +38,12 @@ class GpuTestBase : public ::testing::Test {
   GlCalculatorHelper helper_;
 };
 
+class GpuTestBase : public testing::Test, public GpuTestEnvironment {};
+
+template <typename T>
+class GpuTestWithParamBase : public testing::TestWithParam<T>,
+                             public GpuTestEnvironment {};
+
 }  // namespace mediapipe
 
 #endif  // MEDIAPIPE_GPU_GPU_TEST_BASE_H_

mediapipe/gpu/metal.bzl

@@ -171,7 +171,7 @@ METAL_LIBRARY_ATTRS = dicts.add(apple_support.action_required_attrs(), {
 metal_library = rule(
     implementation = _metal_library_impl,
     attrs = METAL_LIBRARY_ATTRS,
-    fragments = ["apple", "objc", "swift"],
+    fragments = ["apple", "objc"],
     output_to_genfiles = True,
 )
 """

mediapipe/gpu/metal_shared_resources.cc

@@ -50,9 +50,10 @@
 - (CVMetalTextureCacheRef)mtlTextureCache {
   @synchronized(self) {
     if (!_mtlTextureCache) {
-      CVReturn __unused err =
+      CVReturn __unused err = CVMetalTextureCacheCreate(
-          CVMetalTextureCacheCreate(NULL, NULL, self.mtlDevice, NULL, &_mtlTextureCache);
+          NULL, NULL, self.mtlDevice, NULL, &_mtlTextureCache);
-      NSAssert(err == kCVReturnSuccess, @"Error at CVMetalTextureCacheCreate %d ; device %@", err,
+      NSAssert(err == kCVReturnSuccess,
+               @"Error at CVMetalTextureCacheCreate %d ; device %@", err,
                self.mtlDevice);
       // TODO: register and flush metal caches too.
     }

mediapipe/gpu/pixel_buffer_pool_util.cc

@@ -24,23 +24,27 @@
 
 namespace mediapipe {
 
-CVPixelBufferPoolRef CreateCVPixelBufferPool(
+CVPixelBufferPoolRef CreateCVPixelBufferPool(int width, int height,
-    int width, int height, OSType pixelFormat, int keepCount,
+                                             OSType pixelFormat, int keepCount,
-    CFTimeInterval maxAge) {
+                                             CFTimeInterval maxAge) {
   CVPixelBufferPoolRef pool = NULL;
 
   NSMutableDictionary *sourcePixelBufferOptions =
-      [(__bridge NSDictionary*)GetCVPixelBufferAttributesForGlCompatibility() mutableCopy];
+      [(__bridge NSDictionary *)GetCVPixelBufferAttributesForGlCompatibility()
+          mutableCopy];
   [sourcePixelBufferOptions addEntriesFromDictionary:@{
     (id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
     (id)kCVPixelBufferWidthKey : @(width),
     (id)kCVPixelBufferHeightKey : @(height),
   }];
 
-  NSMutableDictionary *pixelBufferPoolOptions = [[NSMutableDictionary alloc] init];
+  NSMutableDictionary *pixelBufferPoolOptions =
-  pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] = @(keepCount);
+      [[NSMutableDictionary alloc] init];
+  pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] =
+      @(keepCount);
   if (maxAge > 0) {
-    pixelBufferPoolOptions[(id)kCVPixelBufferPoolMaximumBufferAgeKey] = @(maxAge);
+    pixelBufferPoolOptions[(id)kCVPixelBufferPoolMaximumBufferAgeKey] =
+        @(maxAge);
   }
 
   CVPixelBufferPoolCreate(
@@ -50,8 +54,9 @@ CVPixelBufferPoolRef CreateCVPixelBufferPool(
   return pool;
 }
 
-OSStatus PreallocateCVPixelBufferPoolBuffers(
+OSStatus PreallocateCVPixelBufferPoolBuffers(CVPixelBufferPoolRef pool,
-    CVPixelBufferPoolRef pool, int count, CFDictionaryRef auxAttributes) {
+                                             int count,
+                                             CFDictionaryRef auxAttributes) {
   CVReturn err = kCVReturnSuccess;
   NSMutableArray *pixelBuffers = [[NSMutableArray alloc] init];
   for (int i = 0; i < count && err == kCVReturnSuccess; i++) {
@@ -68,30 +73,37 @@ OSStatus PreallocateCVPixelBufferPoolBuffers(
   return err;
 }
 
-CFDictionaryRef CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold(int allocationThreshold) {
+CFDictionaryRef CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold(
+    int allocationThreshold) {
   if (allocationThreshold > 0) {
-    return (CFDictionaryRef)CFBridgingRetain(
+    return (CFDictionaryRef)CFBridgingRetain(@{
-        @{(id)kCVPixelBufferPoolAllocationThresholdKey: @(allocationThreshold)});
+      (id)kCVPixelBufferPoolAllocationThresholdKey : @(allocationThreshold)
+    });
   } else {
     return nil;
   }
 }
 
-CVReturn CreateCVPixelBufferWithPool(
+CVReturn CreateCVPixelBufferWithPool(CVPixelBufferPoolRef pool,
-    CVPixelBufferPoolRef pool, CFDictionaryRef auxAttributes,
+                                     CFDictionaryRef auxAttributes,
-    CVTextureCacheType textureCache, CVPixelBufferRef* outBuffer) {
+                                     CVTextureCacheType textureCache,
-  return CreateCVPixelBufferWithPool(pool, auxAttributes, [textureCache](){
+                                     CVPixelBufferRef *outBuffer) {
+  return CreateCVPixelBufferWithPool(
+      pool, auxAttributes,
+      [textureCache]() {
 #if TARGET_OS_OSX
-      CVOpenGLTextureCacheFlush(textureCache, 0);
+        CVOpenGLTextureCacheFlush(textureCache, 0);
 #else
-      CVOpenGLESTextureCacheFlush(textureCache, 0);
+        CVOpenGLESTextureCacheFlush(textureCache, 0);
 #endif  // TARGET_OS_OSX
-  }, outBuffer);
+      },
+      outBuffer);
 }
 
-CVReturn CreateCVPixelBufferWithPool(
+CVReturn CreateCVPixelBufferWithPool(CVPixelBufferPoolRef pool,
-    CVPixelBufferPoolRef pool, CFDictionaryRef auxAttributes,
+                                     CFDictionaryRef auxAttributes,
-    std::function<void(void)> flush, CVPixelBufferRef* outBuffer) {
+                                     std::function<void(void)> flush,
+                                     CVPixelBufferRef *outBuffer) {
   CVReturn err = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(
       kCFAllocatorDefault, pool, auxAttributes, outBuffer);
   if (err == kCVReturnWouldExceedAllocationThreshold) {
@@ -103,11 +115,13 @@ CVReturn CreateCVPixelBufferWithPool(
           kCFAllocatorDefault, pool, auxAttributes, outBuffer);
     }
     if (err == kCVReturnWouldExceedAllocationThreshold) {
-      // TODO: allow the application to set the threshold. For now, disable it by
+      // TODO: allow the application to set the threshold. For now, disable it
-      // default, since the threshold we are using is arbitrary and some graphs routinely cross it.
+      // by default, since the threshold we are using is arbitrary and some
+      // graphs routinely cross it.
 #ifdef ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK
-      NSLog(@"Using more buffers than expected! This is a debug-only warning, "
+      NSLog(
-            "you can ignore it if your app works fine otherwise.");
+          @"Using more buffers than expected! This is a debug-only warning, "
+           "you can ignore it if your app works fine otherwise.");
 #ifdef DEBUG
       NSLog(@"Pool status: %@", ((__bridge NSObject *)pool).description);
 #endif  // DEBUG

mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java

@@ -17,7 +17,6 @@ package com.google.mediapipe.components;
 import static java.lang.Math.max;
 import static java.lang.Math.min;
 
-import android.graphics.Bitmap;
 import android.graphics.SurfaceTexture;
 import android.opengl.GLES11Ext;
 import android.opengl.GLES20;
@@ -25,6 +24,7 @@ import android.opengl.GLES31;
 import android.opengl.GLSurfaceView;
 import android.opengl.Matrix;
 import android.util.Log;
+import android.util.Pair;
 import com.google.mediapipe.framework.TextureFrame;
 import com.google.mediapipe.glutil.CommonShaders;
 import com.google.mediapipe.glutil.ShaderUtil;
@@ -51,11 +51,9 @@ import javax.microedition.khronos.opengles.GL10;
  * {@link TextureFrame} (call {@link #setNextFrame(TextureFrame)}).
  */
 public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
-  /**
+  /** Listener for image capture requests. */
-   * Listener for Bitmap capture requests.
+  public interface ImageCaptureListener {
-   */
+    void onImageCaptured(int width, int height, int[] data);
-  public interface BitmapCaptureListener {
-    void onBitmapCaptured(Bitmap result);
   }
 
   /**
@@ -86,27 +84,28 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
   private float[] textureTransformMatrix = new float[16];
   private SurfaceTexture surfaceTexture = null;
   private final AtomicReference<TextureFrame> nextFrame = new AtomicReference<>();
-  private final AtomicBoolean captureNextFrameBitmap = new AtomicBoolean();
+  private final AtomicBoolean captureNextFrameImage = new AtomicBoolean();
-  private BitmapCaptureListener bitmapCaptureListener;
+  private ImageCaptureListener imageCaptureListener;
   // Specifies whether a black CLAMP_TO_BORDER effect should be used.
   private boolean shouldClampToBorder = false;
   private Scale scale = Scale.FILL;
 
-  /**
+  private float zoomFactor = 1.0f;
-   * Sets the {@link BitmapCaptureListener}.
+  private Pair<Float, Float> zoomLocation = new Pair<>(0.5f, 0.5f);
-   */
+
-  public void setBitmapCaptureListener(BitmapCaptureListener bitmapCaptureListener) {
+  /** Sets the {@link ImageCaptureListener}. */
-    this.bitmapCaptureListener = bitmapCaptureListener;
+  public void setImageCaptureListener(ImageCaptureListener imageCaptureListener) {
+    this.imageCaptureListener = imageCaptureListener;
   }
 
   /**
-   * Request to capture Bitmap of the next frame.
+   * Request to capture Image of the next frame.
    *
-   * The result will be provided to the {@link BitmapCaptureListener} if one is set. Please note
+   * <p>The result will be provided to the {@link ImageCaptureListener} if one is set. Please note
    * this is an expensive operation and the result may not be available for a while.
    */
-  public void captureNextFrameBitmap() {
+  public void captureNextFrameImage() {
-    captureNextFrameBitmap.set(true);
+    captureNextFrameImage.set(true);
   }
 
   @Override
@@ -202,9 +201,9 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
     GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
     ShaderUtil.checkGlError("glDrawArrays");
 
-    // Capture Bitmap if requested.
+    // Capture image if requested.
-    BitmapCaptureListener bitmapCaptureListener = this.bitmapCaptureListener;
+    ImageCaptureListener imageCaptureListener = this.imageCaptureListener;
-    if (captureNextFrameBitmap.getAndSet(false) && bitmapCaptureListener != null) {
+    if (captureNextFrameImage.getAndSet(false) && imageCaptureListener != null) {
       // Find the name of the bound texture.
       int[] texName = new int[1];
       if (surfaceTexture != null) {
@@ -219,8 +218,8 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
       GLES31.glGetTexLevelParameteriv(textureTarget, 0, GLES31.GL_TEXTURE_HEIGHT, texDims, 1);
       int texWidth = texDims[0];
       int texHeight = texDims[1];
-      int bitmapSize = texWidth * texHeight;
+      int imageSize = texWidth * texHeight;
-      ByteBuffer byteBuffer = ByteBuffer.allocateDirect(bitmapSize * 4);
+      ByteBuffer byteBuffer = ByteBuffer.allocateDirect(imageSize * 4);
       byteBuffer.order(ByteOrder.nativeOrder());
 
       // Read pixels from texture.
@@ -235,27 +234,9 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
       GLES20.glDeleteFramebuffers(1, fbo, 0);
       ShaderUtil.checkGlError("capture frame");
 
-      int[] pixelBuffer = new int[bitmapSize];
+      int[] data = new int[imageSize];
-      byteBuffer.asIntBuffer().get(pixelBuffer);
+      byteBuffer.asIntBuffer().get(data);
-      for (int i = 0; i < bitmapSize; i++) {
+      imageCaptureListener.onImageCaptured(texWidth, texHeight, data);
-        // Swap R and B channels.
-        pixelBuffer[i] =
-            (pixelBuffer[i] & 0xff00ff00)
-                | ((pixelBuffer[i] & 0x000000ff) << 16)
-                | ((pixelBuffer[i] & 0x00ff0000) >> 16);
-      }
-
-      // Send bitmap.
-      Bitmap bitmap = Bitmap.createBitmap(texWidth, texHeight, Bitmap.Config.ARGB_8888);
-      bitmap.setPixels(
-          pixelBuffer,
-          /* offset= */ bitmapSize - texWidth,
-          /* stride= */ -texWidth,
-          /* x= */ 0,
-          /* y= */ 0,
-          texWidth,
-          texHeight);
-      bitmapCaptureListener.onBitmapCaptured(bitmap);
     }
 
     GLES20.glBindTexture(textureTarget, 0);
@@ -294,6 +275,15 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
     float textureBottom = (1.0f - scaleHeight) * alignmentVertical;
     float textureTop = textureBottom + scaleHeight;
 
+    Pair<Float, Float> currentZoomLocation = this.zoomLocation;
+    float zoomLocationX = currentZoomLocation.first;
+    float zoomLocationY = currentZoomLocation.second;
+
+    textureLeft = (textureLeft - 0.5f) / zoomFactor + zoomLocationX;
+    textureRight = (textureRight - 0.5f) / zoomFactor + zoomLocationX;
+    textureBottom = (textureBottom - 0.5f) / zoomFactor + zoomLocationY;
+    textureTop = (textureTop - 0.5f) / zoomFactor + zoomLocationY;
+
     return new float[] {textureLeft, textureRight, textureBottom, textureTop};
   }
 
@@ -380,6 +370,22 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
     this.scale = scale;
   }
 
+  /** Zoom factor applied to the frame, must not be 0. */
+  public void setZoomFactor(float zoomFactor) {
+    if (zoomFactor == 0.f) {
+      return;
+    }
+    this.zoomFactor = zoomFactor;
+  }
+
+  /**
+   * Location where to apply the zooming of the frame to. Default is 0.5, 0.5 (scaling is applied to
+   * the center).
+   */
+  public void setZoomLocation(float zoomLocationX, float zoomLocationY) {
+    this.zoomLocation = new Pair<>(zoomLocationX, zoomLocationY);
+  }
+
   private boolean isExternalTexture() {
     return textureTarget == GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
   }

mediapipe/java/com/google/mediapipe/framework/PacketGetter.java

@@ -128,6 +128,16 @@ public final class PacketGetter {
     return ProtoUtil.unpack(result, defaultInstance);
   }
 
+  public static <T extends MessageLite> T getProto(final Packet packet, Parser<T> messageParser) {
+    SerializedMessage result = new SerializedMessage();
+    nativeGetProto(packet.getNativeHandle(), result);
+    try {
+      return messageParser.parseFrom(result.value);
+    } catch (InvalidProtocolBufferException e) {
+      throw new IllegalArgumentException(e);
+    }
+  }
+
   /**
    * @deprecated {@link #getProto(Packet, MessageLite)} is safer to use in obfuscated builds.
    */