From 8f084e6d8872cdaf670a57b02187912db74e5a73 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 16 Nov 2023 00:59:03 +0530 Subject: [PATCH] Fixed crash in creation of OpenGL ES texture from CVPixelBuffer on iOS simulator --- .../gpu/gpu_buffer_storage_cv_pixel_buffer.cc | 28 +++++++++++++++ mediapipe/objc/util.cc | 35 ++++++------------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc b/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc index 5983758f9..2324895f7 100644 --- a/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc +++ b/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc @@ -50,11 +50,38 @@ GlTextureView GpuBufferStorageCvPixelBuffer::GetTexture( const GlTextureInfo info = GlTextureInfoForGpuBufferFormat( format(), plane, gl_context->GetGlVersion()); CVTextureType cv_texture_temp; + +// The current pixel buffer was created by `CVPixelBufferCreate` with attribute +// `kCVPixelBufferIOSurfacePropertiesKey` to ensure that it can be used to +// create a `CVMetalTextureCache`. But creating an OPENGL ES texture from a +// pixel buffer with IOSurface crashes on the simulator. To workaround this, a +// new pixel buffer sharing the same storage of the current pixel buffer eith +// IOSurface is created using `CVPixelBufferCreateWithBytes`. This pixel buffer +// is then used to create the OPENGL ES texture on the simulator. On the device +// OPENGL ES texture creation requires a pixel buffer with IOSurface and hence +// the current piel buffer can be used. +#if TARGET_IPHONE_SIMULATOR + CVPixelBufferRef simulator_pixel_buffer; + CVPixelBufferLockBaseAddress(**this, 0); + CVPixelBufferCreateWithBytes( + NULL, CVPixelBufferGetWidth(**this), CVPixelBufferGetHeight(**this), + CVPixelBufferGetPixelFormatType(**this), + CVPixelBufferGetBaseAddress(**this), CVPixelBufferGetBytesPerRow(**this), + NULL, NULL, NULL, &simulator_pixel_buffer); + CVPixelBufferUnlockBaseAddress(**this, 0); + err = CVOpenGLESTextureCacheCreateTextureFromImage( + kCFAllocatorDefault, gl_context->cv_texture_cache(), + simulator_pixel_buffer, NULL, GL_TEXTURE_2D, info.gl_internal_format, + width() / info.downscale, height() / info.downscale, info.gl_format, + info.gl_type, plane, &cv_texture_temp); +#else err = CVOpenGLESTextureCacheCreateTextureFromImage( kCFAllocatorDefault, gl_context->cv_texture_cache(), **this, NULL, GL_TEXTURE_2D, info.gl_internal_format, width() / info.downscale, height() / info.downscale, info.gl_format, info.gl_type, plane, &cv_texture_temp); +#endif + ABSL_CHECK(cv_texture_temp && !err) << "CVOpenGLESTextureCacheCreateTextureFromImage failed: " << err; CFHolder cv_texture; @@ -102,6 +129,7 @@ static void ViewDoneWritingSimulatorWorkaround(CVPixelBufferRef pixel_buffer, std::vector contiguous_buffer(contiguous_bytes_per_row * view.height()); uint8_t* temp_ptr = contiguous_buffer.data(); + glPixelStorei(GL_PACK_ALIGNMENT, 4); glReadPixels(0, 0, view.width(), view.height(), GL_BGRA, GL_UNSIGNED_BYTE, temp_ptr); for (int i = 0; i < view.height(); ++i) { diff --git a/mediapipe/objc/util.cc b/mediapipe/objc/util.cc index e5908f25a..65ef6b88f 100644 --- a/mediapipe/objc/util.cc +++ b/mediapipe/objc/util.cc @@ -251,25 +251,9 @@ static void FreeRefConReleaseCallback(void* refCon, const void* baseAddress) { CVReturn CreateCVPixelBufferWithoutPool(int width, int height, OSType cv_format, CVPixelBufferRef* out_buffer) { -#if TARGET_IPHONE_SIMULATOR - // On the simulator, syncing the texture with the pixelbuffer does not work, - // and we have to use glReadPixels. Since GL_UNPACK_ROW_LENGTH is not - // available in OpenGL ES 2, we should create the buffer so the pixels are - // contiguous. - // - // TODO: verify if we can use kIOSurfaceBytesPerRow to force - // CoreVideo to give us contiguous data. - size_t bytes_per_row = width * 4; - void* data = malloc(bytes_per_row * height); - return CVPixelBufferCreateWithBytes( - kCFAllocatorDefault, width, height, cv_format, data, bytes_per_row, - FreeRefConReleaseCallback, data, - GetCVPixelBufferAttributesForGlCompatibility(), out_buffer); -#else return CVPixelBufferCreate(kCFAllocatorDefault, width, height, cv_format, GetCVPixelBufferAttributesForGlCompatibility(), out_buffer); -#endif } absl::StatusOr> CreateCVPixelBufferWithoutPool( @@ -655,13 +639,16 @@ CFDictionaryRef GetCVPixelBufferAttributesForGlCompatibility() { kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); - // To ensure compatibility with CVOpenGLESTextureCache, these attributes - // should be present. However, on simulator this IOSurface attribute - // actually causes CVOpenGLESTextureCache to fail. b/144850076 + // To ensure compatibility with CVMetalTextureCache + // kCVPixelBufferIOSurfacePropertiesKey must be present. To ensure + // compatibility with CVOpenGLESTextureCache all the listed property keys + // must be present. However, on simulator this IOSurface attribute actually + // causes CVOpenGLESTextureCache to fail. b/144850076 We will use the pixel + // buffer created using these attributes to create CVOpenGLESTextureCache + // only on the device. For simulator, a different pixel buffer will be + // created. const void* keys[] = { -#if !TARGET_IPHONE_SIMULATOR kCVPixelBufferIOSurfacePropertiesKey, -#endif // !TARGET_IPHONE_SIMULATOR #if TARGET_OS_OSX kCVPixelFormatOpenGLCompatibility, @@ -671,10 +658,8 @@ CFDictionaryRef GetCVPixelBufferAttributesForGlCompatibility() { }; const void* values[] = { -#if !TARGET_IPHONE_SIMULATOR - empty_dict, -#endif // !TARGET_IPHONE_SIMULATOR - kCFBooleanTrue + empty_dict, + kCFBooleanTrue, }; attrs = CFDictionaryCreate(