cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge 0493a6070d into e23fa531e1

Browse Source
This commit is contained in:
priankakariatyml 2024-01-01 23:40:09 -08:00 committed by GitHub
commit 9c94a1feec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 80 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc
View File

@ -185,9 +185,34 @@ class SubRectExtractorMetal {
id<MTLCommandBuffer> command_buffer,
id<MTLBuffer> destination) {
auto output_texture = MTLTextureWithBuffer(destination_size, destination);
return InternalExecute(input_texture, sub_rect, flip_horizontally, alpha,
beta, destination_size, command_buffer,
output_texture);
absl::Status status =
InternalExecute(input_texture, sub_rect, flip_horizontally, alpha, beta,
destination_size, command_buffer, output_texture);
// On the simulator the `output_texture` cannot share it's underlying storage
// with the tensor's CPU Buffer. Hence the contents of the `output_texture`
// after sub rect extraction must be copied to the tensor's CPU buffer held by
// the `destination` buffer.
#if TARGET_IPHONE_SIMULATOR
NSUInteger output_bytes_per_row =
GetBytesPerRaw(output_format_, destination_size);
id<MTLBlitCommandEncoder> blitCommandEncoder =
command_buffer.blitCommandEncoder;
[blitCommandEncoder copyFromTexture:output_texture
sourceSlice:0
sourceLevel:0
sourceOrigin:MTLOriginMake(0, 0, 0)
sourceSize:MTLSizeMake(output_texture.width,
output_texture.height,
output_texture.depth)
toBuffer:destination
destinationOffset:0
destinationBytesPerRow:output_bytes_per_row
destinationBytesPerImage:0];
[blitCommandEncoder endEncoding];
#endif
return status;
}
private:
@ -201,11 +226,21 @@ class SubRectExtractorMetal {
texture_desc.usage = MTLTextureUsageRenderTarget;
NSUInteger output_bytes_per_row = GetBytesPerRaw(output_format_, size);
// Creating a no copy `MTLTexture` from an `MTLBuffer` with
// MTLStorageModeShared on the simultor results in the following error:
// "Linear textures from shared buffers is not supported on this device."
// To mitigate this crash, an empty `MTLTexture` is created using the
// `MTLDevice`. The solution invovles an extra copy for copying the
// underlying buffer of the texture back to the tensor's CPU Buffer.
id<MTLTexture> texture;
#if TARGET_IPHONE_SIMULATOR
texture = [buffer.device newTextureWithDescriptor:texture_desc];
#else
texture = [buffer newTextureWithDescriptor:texture_desc
offset:0
bytesPerRow:output_bytes_per_row];
#endif
id<MTLTexture> texture =
[buffer newTextureWithDescriptor:texture_desc
offset:0
bytesPerRow:output_bytes_per_row];
return texture;
}

28
mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc
View File

@ -50,11 +50,38 @@ GlTextureView GpuBufferStorageCvPixelBuffer::GetTexture(
const GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
format(), plane, gl_context->GetGlVersion());
CVTextureType cv_texture_temp;
// The current pixel buffer was created by `CVPixelBufferCreate` with attribute
// `kCVPixelBufferIOSurfacePropertiesKey` to ensure that it can be used to
// create a `CVMetalTextureCache`. But creating an OPENGL ES texture from a
// pixel buffer with IOSurface crashes on the simulator. To workaround this, a
// new pixel buffer sharing the same storage of the current pixel buffer eith
// IOSurface is created using `CVPixelBufferCreateWithBytes`. This pixel buffer
// is then used to create the OPENGL ES texture on the simulator. On the device
// OPENGL ES texture creation requires a pixel buffer with IOSurface and hence
// the current piel buffer can be used.
#if TARGET_IPHONE_SIMULATOR
CVPixelBufferRef simulator_pixel_buffer;
CVPixelBufferLockBaseAddress(**this, 0);
CVPixelBufferCreateWithBytes(
NULL, CVPixelBufferGetWidth(**this), CVPixelBufferGetHeight(**this),
CVPixelBufferGetPixelFormatType(**this),
CVPixelBufferGetBaseAddress(**this), CVPixelBufferGetBytesPerRow(**this),
NULL, NULL, NULL, &simulator_pixel_buffer);
CVPixelBufferUnlockBaseAddress(**this, 0);
err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, gl_context->cv_texture_cache(),
simulator_pixel_buffer, NULL, GL_TEXTURE_2D, info.gl_internal_format,
width() / info.downscale, height() / info.downscale, info.gl_format,
info.gl_type, plane, &cv_texture_temp);
#else
err = CVOpenGLESTextureCacheCreateTextureFromImage(
kCFAllocatorDefault, gl_context->cv_texture_cache(), **this, NULL,
GL_TEXTURE_2D, info.gl_internal_format, width() / info.downscale,
height() / info.downscale, info.gl_format, info.gl_type, plane,
&cv_texture_temp);
#endif
ABSL_CHECK(cv_texture_temp && !err)
<< "CVOpenGLESTextureCacheCreateTextureFromImage failed: " << err;
CFHolder<CVTextureType> cv_texture;
@ -102,6 +129,7 @@ static void ViewDoneWritingSimulatorWorkaround(CVPixelBufferRef pixel_buffer,
std::vector<uint8_t> contiguous_buffer(contiguous_bytes_per_row *
view.height());
uint8_t* temp_ptr = contiguous_buffer.data();
glPixelStorei(GL_PACK_ALIGNMENT, 4);
glReadPixels(0, 0, view.width(), view.height(), GL_BGRA,
GL_UNSIGNED_BYTE, temp_ptr);
for (int i = 0; i < view.height(); ++i) {

35
mediapipe/objc/util.cc
View File

@ -251,25 +251,9 @@ static void FreeRefConReleaseCallback(void* refCon, const void* baseAddress) {
CVReturn CreateCVPixelBufferWithoutPool(int width, int height, OSType cv_format,
CVPixelBufferRef* out_buffer) {
#if TARGET_IPHONE_SIMULATOR
// On the simulator, syncing the texture with the pixelbuffer does not work,
// and we have to use glReadPixels. Since GL_UNPACK_ROW_LENGTH is not
// available in OpenGL ES 2, we should create the buffer so the pixels are
// contiguous.
//
// TODO: verify if we can use kIOSurfaceBytesPerRow to force
// CoreVideo to give us contiguous data.
size_t bytes_per_row = width * 4;
void* data = malloc(bytes_per_row * height);
return CVPixelBufferCreateWithBytes(
kCFAllocatorDefault, width, height, cv_format, data, bytes_per_row,
FreeRefConReleaseCallback, data,
GetCVPixelBufferAttributesForGlCompatibility(), out_buffer);
#else
return CVPixelBufferCreate(kCFAllocatorDefault, width, height, cv_format,
GetCVPixelBufferAttributesForGlCompatibility(),
out_buffer);
#endif
}
absl::StatusOr<CFHolder<CVPixelBufferRef>> CreateCVPixelBufferWithoutPool(
@ -655,13 +639,16 @@ CFDictionaryRef GetCVPixelBufferAttributesForGlCompatibility() {
kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks,
&kCFTypeDictionaryValueCallBacks);
// To ensure compatibility with CVOpenGLESTextureCache, these attributes
// should be present. However, on simulator this IOSurface attribute
// actually causes CVOpenGLESTextureCache to fail. b/144850076
// To ensure compatibility with CVMetalTextureCache
// kCVPixelBufferIOSurfacePropertiesKey must be present. To ensure
// compatibility with CVOpenGLESTextureCache all the listed property keys
// must be present. However, on simulator this IOSurface attribute actually
// causes CVOpenGLESTextureCache to fail. b/144850076 We will use the pixel
// buffer created using these attributes to create CVOpenGLESTextureCache
// only on the device. For simulator, a different pixel buffer will be
// created.
const void* keys[] = {
#if !TARGET_IPHONE_SIMULATOR
kCVPixelBufferIOSurfacePropertiesKey,
#endif // !TARGET_IPHONE_SIMULATOR
#if TARGET_OS_OSX
kCVPixelFormatOpenGLCompatibility,
@ -671,10 +658,8 @@ CFDictionaryRef GetCVPixelBufferAttributesForGlCompatibility() {
};
const void* values[] = {
#if !TARGET_IPHONE_SIMULATOR
empty_dict,
#endif // !TARGET_IPHONE_SIMULATOR
kCFBooleanTrue
empty_dict,
kCFBooleanTrue,
};
attrs = CFDictionaryCreate(