cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove DrishtiGraphGPUData, add MetalSharedResources

Browse Source 
This class is unused except by the Metal helper; let's narrow it down and simplify gpu_shared_data.

PiperOrigin-RevId: 490531767
This commit is contained in:
Camillo Lugaresi 2022-11-23 10:13:48 -08:00 committed by Copybara-Service
parent 3c53ec2cdb
commit 54d1744c8f
8 changed files with 46 additions and 349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
mediapipe/gpu/BUILD
View File

@ -470,12 +470,9 @@ objc_library(
)
objc_library(
name = "MPPGraphGPUData",
srcs = [
"MPPGraphGPUData.mm",
"gpu_shared_data_internal.cc",
],
hdrs = ["MPPGraphGPUData.h"],
name = "metal_shared_resources",
srcs = ["metal_shared_resources.mm"],
hdrs = ["metal_shared_resources.h"],
copts = [
"-x objective-c++",
"-Wno-shorten-64-to-32",
@ -484,25 +481,9 @@ objc_library(
sdk_frameworks = [
"CoreVideo",
"Metal",
] + select({
"//conditions:default": [
"OpenGLES",
],
"//mediapipe:macos": [
"OpenGL",
"AppKit",
],
}),
visibility = ["//visibility:public"],
deps = [
":gl_base",
":gl_context",
":gpu_buffer_multi_pool",
":gpu_shared_data_header",
":graph_support",
":cv_texture_cache_manager",
"//mediapipe/gpu:gl_context_options_cc_proto",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework/port:ret_check",
"@google_toolbox_for_mac//:GTM_Defines",
] + [
@ -584,16 +565,19 @@ cc_library(
cc_library(
name = "gpu_shared_data_internal_actual",
srcs = select({
"//conditions:default": [
srcs = [
"gpu_shared_data_internal.cc",
],
# iOS uses an Objective-C++ version of this, built in MPPGraphGPUData.
"//mediapipe:apple": [],
}),
hdrs = [
"gpu_shared_data_internal.h",
],
copts = select({
"//conditions:default": [],
"//mediapipe:apple": [
"-x objective-c++",
"-fobjc-arc", # enable reference-counting
],
}),
visibility = ["//visibility:private"],
deps = [
"//mediapipe/gpu:gl_context_options_cc_proto",
@ -610,7 +594,7 @@ cc_library(
] + select({
"//conditions:default": [],
"//mediapipe:apple": [
":MPPGraphGPUData",
":metal_shared_resources",
":cv_texture_cache_manager",
],
}),
@ -1139,8 +1123,8 @@ objc_library(
name = "gl_ios_test_lib",
testonly = 1,
srcs = [
"MPPGraphGPUDataTests.mm",
"gl_ios_test.mm",
"metal_shared_resources_test.mm",
],
copts = [
"-Wno-shorten-64-to-32",
@ -1150,7 +1134,7 @@ objc_library(
],
features = ["-layering_check"],
deps = [
":MPPGraphGPUData",
":metal_shared_resources",
":gl_scaler_calculator",
":gpu_buffer_to_image_frame_calculator",
":gpu_shared_data_internal",

71
mediapipe/gpu/MPPGraphGPUData.h
View File

@ -1,71 +0,0 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_MPPGRAPHGPUDATA_H_
#define MEDIAPIPE_GPU_MPPGRAPHGPUDATA_H_
#import <CoreVideo/CVMetalTextureCache.h>
#import <CoreVideo/CoreVideo.h>
#import <Metal/Metal.h>
#import "mediapipe/gpu/gl_base.h"
#import "mediapipe/gpu/gl_context.h"
namespace mediapipe {
class GlContext;
class GpuBufferMultiPool;
} // namespace mediapipe
@interface MPPGraphGPUData : NSObject {
// Shared buffer pool for GPU calculators.
mediapipe::GpuBufferMultiPool* _gpuBufferPool;
mediapipe::GlContext* _glContext;
}
- (instancetype)init NS_UNAVAILABLE;
/// Initialize. The provided multipool pointer must remain valid throughout
/// this object's lifetime.
- (instancetype)initWithContext:(mediapipe::GlContext*)context
multiPool:(mediapipe::GpuBufferMultiPool*)pool NS_DESIGNATED_INITIALIZER;
/// Shared texture pool for GPU calculators.
/// For internal use by GlCalculatorHelper.
@property(readonly) mediapipe::GpuBufferMultiPool* gpuBufferPool;
/// Shared OpenGL context.
#if TARGET_OS_OSX
@property(readonly) NSOpenGLContext* glContext;
@property(readonly) NSOpenGLPixelFormat* glPixelFormat;
#else
@property(readonly) EAGLContext* glContext;
#endif // TARGET_OS_OSX
/// Shared texture cache.
#if TARGET_OS_OSX
@property(readonly) CVOpenGLTextureCacheRef textureCache;
#else
@property(readonly) CVOpenGLESTextureCacheRef textureCache;
#endif // TARGET_OS_OSX
/// Shared Metal resources.
@property(readonly) id<MTLDevice> mtlDevice;
@property(readonly) id<MTLCommandQueue> mtlCommandQueue;
#if COREVIDEO_SUPPORTS_METAL
@property(readonly) CVMetalTextureCacheRef mtlTextureCache;
#endif
@end
#endif // MEDIAPIPE_GPU_MPPGRAPHGPUDATA_H_

124
mediapipe/gpu/MPPGraphGPUData.mm
View File

@ -1,124 +0,0 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "mediapipe/gpu/MPPGraphGPUData.h"
#import "GTMDefines.h"
#include "mediapipe/gpu/gl_context.h"
#include "mediapipe/gpu/gpu_buffer_multi_pool.h"
#if TARGET_OS_OSX
#import <AppKit/NSOpenGL.h>
#else
#import <OpenGLES/EAGL.h>
#endif // TARGET_OS_OSX
@implementation MPPGraphGPUData
@synthesize textureCache = _textureCache;
@synthesize mtlDevice = _mtlDevice;
@synthesize mtlCommandQueue = _mtlCommandQueue;
#if COREVIDEO_SUPPORTS_METAL
@synthesize mtlTextureCache = _mtlTextureCache;
#endif
#if TARGET_OS_OSX
typedef CVOpenGLTextureCacheRef CVTextureCacheType;
#else
typedef CVOpenGLESTextureCacheRef CVTextureCacheType;
#endif // TARGET_OS_OSX
- (instancetype)initWithContext:(mediapipe::GlContext *)context
multiPool:(mediapipe::GpuBufferMultiPool *)pool {
self = [super init];
if (self) {
_gpuBufferPool = pool;
_glContext = context;
}
return self;
}
- (void)dealloc {
if (_textureCache) {
_textureCache = NULL;
}
#if COREVIDEO_SUPPORTS_METAL
if (_mtlTextureCache) {
CFRelease(_mtlTextureCache);
_mtlTextureCache = NULL;
}
#endif
}
#if TARGET_OS_OSX
- (NSOpenGLContext *)glContext {
return _glContext->nsgl_context();
}
- (NSOpenGLPixelFormat *) glPixelFormat {
return _glContext->nsgl_pixel_format();
}
#else
- (EAGLContext *)glContext {
return _glContext->eagl_context();
}
#endif // TARGET_OS_OSX
- (CVTextureCacheType)textureCache {
@synchronized(self) {
if (!_textureCache) {
_textureCache = _glContext->cv_texture_cache();
}
}
return _textureCache;
}
- (mediapipe::GpuBufferMultiPool *)gpuBufferPool {
return _gpuBufferPool;
}
- (id<MTLDevice>)mtlDevice {
@synchronized(self) {
if (!_mtlDevice) {
_mtlDevice = MTLCreateSystemDefaultDevice();
}
}
return _mtlDevice;
}
- (id<MTLCommandQueue>)mtlCommandQueue {
@synchronized(self) {
if (!_mtlCommandQueue) {
_mtlCommandQueue = [self.mtlDevice newCommandQueue];
}
}
return _mtlCommandQueue;
}
#if COREVIDEO_SUPPORTS_METAL
- (CVMetalTextureCacheRef)mtlTextureCache {
@synchronized(self) {
if (!_mtlTextureCache) {
CVReturn __unused err =
CVMetalTextureCacheCreate(NULL, NULL, self.mtlDevice, NULL, &_mtlTextureCache);
NSAssert(err == kCVReturnSuccess, @"Error at CVMetalTextureCacheCreate %d", err);
// TODO: register and flush metal caches too.
}
}
return _mtlTextureCache;
}
#endif
@end

86
mediapipe/gpu/MPPGraphGPUDataTests.mm
View File

@ -1,86 +0,0 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <UIKit/UIKit.h>
#import <XCTest/XCTest.h>
#include <memory>
#include "absl/memory/memory.h"
#include "mediapipe/framework/port/threadpool.h"
#import "mediapipe/gpu/MPPGraphGPUData.h"
#import "mediapipe/gpu/gpu_shared_data_internal.h"
@interface MPPGraphGPUDataTests : XCTestCase {
}
@end
@implementation MPPGraphGPUDataTests
// This test verifies that the internal Objective-C object is correctly
// released when the C++ wrapper is released.
- (void)testCorrectlyReleased {
__weak id gpuData = nil;
std::weak_ptr<mediapipe::GpuResources> gpuRes;
@autoreleasepool {
mediapipe::GpuSharedData gpu_shared;
gpuRes = gpu_shared.gpu_resources;
gpuData = gpu_shared.gpu_resources->ios_gpu_data();
XCTAssertNotEqual(gpuRes.lock(), nullptr);
XCTAssertNotNil(gpuData);
}
XCTAssertEqual(gpuRes.lock(), nullptr);
XCTAssertNil(gpuData);
}
// This test verifies that the lazy initialization of the glContext instance
// variable is thread-safe. All threads should read the same value.
- (void)testGlContextThreadSafeLazyInitialization {
mediapipe::GpuSharedData gpu_shared;
constexpr int kNumThreads = 10;
EAGLContext* ogl_context[kNumThreads];
auto pool = absl::make_unique<mediapipe::ThreadPool>(kNumThreads);
pool->StartWorkers();
for (int i = 0; i < kNumThreads; ++i) {
pool->Schedule([&gpu_shared, &ogl_context, i] {
ogl_context[i] = gpu_shared.gpu_resources->ios_gpu_data().glContext;
});
}
pool.reset();
for (int i = 0; i < kNumThreads - 1; ++i) {
XCTAssertEqual(ogl_context[i], ogl_context[i + 1]);
}
}
// This test verifies that the lazy initialization of the textureCache instance
// variable is thread-safe. All threads should read the same value.
- (void)testTextureCacheThreadSafeLazyInitialization {
mediapipe::GpuSharedData gpu_shared;
constexpr int kNumThreads = 10;
CFHolder<CVOpenGLESTextureCacheRef> texture_cache[kNumThreads];
auto pool = absl::make_unique<mediapipe::ThreadPool>(kNumThreads);
pool->StartWorkers();
for (int i = 0; i < kNumThreads; ++i) {
pool->Schedule([&gpu_shared, &texture_cache, i] {
texture_cache[i].reset(gpu_shared.gpu_resources->ios_gpu_data().textureCache);
});
}
pool.reset();
for (int i = 0; i < kNumThreads - 1; ++i) {
XCTAssertEqual(*texture_cache[i], *texture_cache[i + 1]);
}
}
@end

29
mediapipe/gpu/MPPMetalHelper.mm
View File

@ -14,14 +14,15 @@
#import "mediapipe/gpu/MPPMetalHelper.h"
#import "mediapipe/gpu/MPPGraphGPUData.h"
#import "mediapipe/gpu/gpu_buffer.h"
#import "mediapipe/gpu/graph_support.h"
#import "mediapipe/gpu/metal_shared_resources.h"
#import "GTMDefines.h"
#include "mediapipe/framework/port/ret_check.h"
@interface MPPMetalHelper () {
MPPGraphGPUData* _gpuShared;
mediapipe::GpuResources* _gpuResources;
}
@end
@ -46,7 +47,7 @@ class MetalHelperLegacySupport {
- (instancetype)initWithGpuResources:(mediapipe::GpuResources*)gpuResources {
self = [super init];
if (self) {
_gpuShared = gpuResources->ios_gpu_data();
_gpuResources = gpuResources;
}
return self;
}
@ -111,19 +112,19 @@ class MetalHelperLegacySupport {
}
- (id<MTLDevice>)mtlDevice {
return _gpuShared.mtlDevice;
return _gpuResources->metal_shared().resources().mtlDevice;
}
- (id<MTLCommandQueue>)mtlCommandQueue {
return _gpuShared.mtlCommandQueue;
return _gpuResources->metal_shared().resources().mtlCommandQueue;
}
- (CVMetalTextureCacheRef)mtlTextureCache {
return _gpuShared.mtlTextureCache;
return _gpuResources->metal_shared().resources().mtlTextureCache;
}
- (id<MTLCommandBuffer>)commandBuffer {
return [_gpuShared.mtlCommandQueue commandBuffer];
return [_gpuResources->metal_shared().resources().mtlCommandQueue commandBuffer];
}
- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer
@ -175,8 +176,9 @@ class MetalHelperLegacySupport {
CVMetalTextureRef texture;
CVReturn err = CVMetalTextureCacheCreateTextureFromImage(
NULL, _gpuShared.mtlTextureCache, mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL,
metalPixelFormat, width, height, plane, &texture);
NULL, _gpuResources->metal_shared().resources().mtlTextureCache,
mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width, height, plane,
&texture);
CHECK_EQ(err, kCVReturnSuccess);
return texture;
}
@ -197,18 +199,19 @@ class MetalHelperLegacySupport {
}
- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width height:(int)height {
return _gpuShared.gpuBufferPool->GetBuffer(width, height);
return _gpuResources->gpu_buffer_pool().GetBuffer(width, height);
}
- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width
height:(int)height
format:(mediapipe::GpuBufferFormat)format {
return _gpuShared.gpuBufferPool->GetBuffer(width, height, format);
return _gpuResources->gpu_buffer_pool().GetBuffer(width, height, format);
}
- (id<MTLLibrary>)newLibraryWithResourceName:(NSString*)name error:(NSError * _Nullable *)error {
return [_gpuShared.mtlDevice newLibraryWithFile:[[NSBundle bundleForClass:[self class]]
pathForResource:name ofType:@"metallib"]
return [_gpuResources->metal_shared().resources().mtlDevice
newLibraryWithFile:[[NSBundle bundleForClass:[self class]] pathForResource:name
ofType:@"metallib"]
error:error];
}

13
mediapipe/gpu/gpu_shared_data_internal.cc
View File

@ -21,7 +21,7 @@
#include "mediapipe/gpu/graph_support.h"
#if __APPLE__
#import "mediapipe/gpu/MPPGraphGPUData.h"
#include "mediapipe/gpu/metal_shared_resources.h"
#endif // __APPLE__
namespace mediapipe {
@ -97,15 +97,14 @@ GpuResources::GpuResources(std::shared_ptr<GlContext> gl_context)
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
texture_caches_->RegisterTextureCache(gl_context->cv_texture_cache());
#endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
ios_gpu_data_ = [[MPPGraphGPUData alloc] initWithContext:gl_context.get()
multiPool:&gpu_buffer_pool_];
metal_shared_ = std::make_unique<MetalSharedResources>();
#endif // __APPLE__
}
GpuResources::~GpuResources() {
#if __APPLE__
// Note: on Apple platforms, this object contains Objective-C objects. The
// destructor will release them, but ARC must be on.
// Note: on Apple platforms, this object contains Objective-C objects.
// The destructor will release them, but ARC must be on.
#if !__has_feature(objc_arc)
#error This file must be built with ARC.
#endif
@ -196,10 +195,6 @@ GlContext::StatusOrGlContext GpuResources::GetOrCreateGlContext(
GpuSharedData::GpuSharedData() : GpuSharedData(kPlatformGlContextNone) {}
#if __APPLE__
MPPGraphGPUData* GpuResources::ios_gpu_data() { return ios_gpu_data_; }
#endif // __APPLE__
extern const GraphService<GpuResources> kGpuService;
#if !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER

18
mediapipe/gpu/gpu_shared_data_internal.h
View File

@ -31,15 +31,14 @@
#ifdef __APPLE__
#include "mediapipe/gpu/cv_texture_cache_manager.h"
#ifdef __OBJC__
@class MPPGraphGPUData;
#else
struct MPPGraphGPUData;
#endif // __OBJC__
#endif // defined(__APPLE__)
namespace mediapipe {
#ifdef __APPLE__
class MetalSharedResources;
#endif // defined(__APPLE__)
// TODO: rename to GpuService or GpuManager or something.
class GpuResources {
public:
@ -56,9 +55,7 @@ class GpuResources {
// Shared GL context for calculators.
// TODO: require passing a context or node identifier.
const std::shared_ptr<GlContext>& gl_context() {
return gl_context(nullptr);
};
const std::shared_ptr<GlContext>& gl_context() { return gl_context(nullptr); }
const std::shared_ptr<GlContext>& gl_context(CalculatorContext* cc);
@ -66,7 +63,7 @@ class GpuResources {
GpuBufferMultiPool& gpu_buffer_pool() { return gpu_buffer_pool_; }
#ifdef __APPLE__
MPPGraphGPUData* ios_gpu_data();
MetalSharedResources& metal_shared() { return *metal_shared_; }
#endif // defined(__APPLE__)§
absl::Status PrepareGpuNode(CalculatorNode* node);
@ -96,8 +93,7 @@ class GpuResources {
GpuBufferMultiPool gpu_buffer_pool_;
#ifdef __APPLE__
// Note that this is an Objective-C object.
MPPGraphGPUData* ios_gpu_data_;
std::unique_ptr<MetalSharedResources> metal_shared_;
#endif // defined(__APPLE__)
std::map<std::string, std::shared_ptr<Executor>> named_executors_;

2
mediapipe/objc/BUILD
View File

@ -83,11 +83,11 @@ objc_library(
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/framework/port:threadpool",
"//mediapipe/gpu:MPPGraphGPUData",
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_shared_data_internal",
"//mediapipe/gpu:graph_support",
"//mediapipe/gpu:metal_shared_resources",
"//mediapipe/gpu:pixel_buffer_pool_util",
"//mediapipe/util:cpu_util",
"@com_google_absl//absl/base:core_headers",