Fixed crash in creation of OpenGL ES texture from CVPixelBuffer on iOS simulator

mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc

@@ -50,11 +50,38 @@ GlTextureView GpuBufferStorageCvPixelBuffer::GetTexture(
   const GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
       format(), plane, gl_context->GetGlVersion());
   CVTextureType cv_texture_temp;
+
+// The current pixel buffer was created by `CVPixelBufferCreate` with attribute
+// `kCVPixelBufferIOSurfacePropertiesKey` to ensure that it can be used to
+// create a `CVMetalTextureCache`. But creating an OPENGL ES texture from a
+// pixel buffer with IOSurface crashes on the simulator. To workaround this, a
+// new pixel buffer sharing the same storage of the current pixel buffer eith
+// IOSurface is created using `CVPixelBufferCreateWithBytes`. This pixel buffer
+// is then used to create the OPENGL ES texture on the simulator. On the device
+// OPENGL ES texture creation requires a pixel buffer with IOSurface and hence
+// the current piel buffer can be used.
+#if TARGET_IPHONE_SIMULATOR
+  CVPixelBufferRef simulator_pixel_buffer;
+  CVPixelBufferLockBaseAddress(**this, 0);
+  CVPixelBufferCreateWithBytes(
+      NULL, CVPixelBufferGetWidth(**this), CVPixelBufferGetHeight(**this),
+      CVPixelBufferGetPixelFormatType(**this),
+      CVPixelBufferGetBaseAddress(**this), CVPixelBufferGetBytesPerRow(**this),
+      NULL, NULL, NULL, &simulator_pixel_buffer);
+  CVPixelBufferUnlockBaseAddress(**this, 0);
+  err = CVOpenGLESTextureCacheCreateTextureFromImage(
+      kCFAllocatorDefault, gl_context->cv_texture_cache(),
+      simulator_pixel_buffer, NULL, GL_TEXTURE_2D, info.gl_internal_format,
+      width() / info.downscale, height() / info.downscale, info.gl_format,
+      info.gl_type, plane, &cv_texture_temp);
+#else
   err = CVOpenGLESTextureCacheCreateTextureFromImage(
       kCFAllocatorDefault, gl_context->cv_texture_cache(), **this, NULL,
       GL_TEXTURE_2D, info.gl_internal_format, width() / info.downscale,
       height() / info.downscale, info.gl_format, info.gl_type, plane,
       &cv_texture_temp);
+#endif
+
   ABSL_CHECK(cv_texture_temp && !err)
       << "CVOpenGLESTextureCacheCreateTextureFromImage failed: " << err;
   CFHolder<CVTextureType> cv_texture;
@@ -102,6 +129,7 @@ static void ViewDoneWritingSimulatorWorkaround(CVPixelBufferRef pixel_buffer,
         std::vector<uint8_t> contiguous_buffer(contiguous_bytes_per_row *
                                                view.height());
         uint8_t* temp_ptr = contiguous_buffer.data();
+        glPixelStorei(GL_PACK_ALIGNMENT, 4);
         glReadPixels(0, 0, view.width(), view.height(), GL_BGRA,
                      GL_UNSIGNED_BYTE, temp_ptr);
         for (int i = 0; i < view.height(); ++i) {

mediapipe/objc/util.cc

@@ -251,25 +251,9 @@ static void FreeRefConReleaseCallback(void* refCon, const void* baseAddress) {
 
 CVReturn CreateCVPixelBufferWithoutPool(int width, int height, OSType cv_format,
                                         CVPixelBufferRef* out_buffer) {
-#if TARGET_IPHONE_SIMULATOR
-  // On the simulator, syncing the texture with the pixelbuffer does not work,
-  // and we have to use glReadPixels. Since GL_UNPACK_ROW_LENGTH is not
-  // available in OpenGL ES 2, we should create the buffer so the pixels are
-  // contiguous.
-  //
-  // TODO: verify if we can use kIOSurfaceBytesPerRow to force
-  // CoreVideo to give us contiguous data.
-  size_t bytes_per_row = width * 4;
-  void* data = malloc(bytes_per_row * height);
-  return CVPixelBufferCreateWithBytes(
-      kCFAllocatorDefault, width, height, cv_format, data, bytes_per_row,
-      FreeRefConReleaseCallback, data,
-      GetCVPixelBufferAttributesForGlCompatibility(), out_buffer);
-#else
   return CVPixelBufferCreate(kCFAllocatorDefault, width, height, cv_format,
                              GetCVPixelBufferAttributesForGlCompatibility(),
                              out_buffer);
-#endif
 }
 
 absl::StatusOr<CFHolder<CVPixelBufferRef>> CreateCVPixelBufferWithoutPool(
@@ -655,13 +639,16 @@ CFDictionaryRef GetCVPixelBufferAttributesForGlCompatibility() {
         kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks,
         &kCFTypeDictionaryValueCallBacks);
 
-    // To ensure compatibility with CVOpenGLESTextureCache, these attributes
-    // should be present. However, on simulator this IOSurface attribute
-    // actually causes CVOpenGLESTextureCache to fail. b/144850076
+    // To ensure compatibility with CVMetalTextureCache
+    // kCVPixelBufferIOSurfacePropertiesKey must be present. To ensure
+    // compatibility with CVOpenGLESTextureCache all the listed property keys
+    // must be present. However, on simulator this IOSurface attribute actually
+    // causes CVOpenGLESTextureCache to fail. b/144850076 We will use the pixel
+    // buffer created using these attributes to create CVOpenGLESTextureCache
+    // only on the device. For simulator, a different pixel buffer will be
+    // created.
     const void* keys[] = {
-#if !TARGET_IPHONE_SIMULATOR
       kCVPixelBufferIOSurfacePropertiesKey,
-#endif  // !TARGET_IPHONE_SIMULATOR
 
 #if TARGET_OS_OSX
       kCVPixelFormatOpenGLCompatibility,
@@ -671,10 +658,8 @@ CFDictionaryRef GetCVPixelBufferAttributesForGlCompatibility() {
     };
 
     const void* values[] = {
-#if !TARGET_IPHONE_SIMULATOR
-      empty_dict,
-#endif  // !TARGET_IPHONE_SIMULATOR
-      kCFBooleanTrue
+        empty_dict,
+        kCFBooleanTrue,
     };
 
     attrs = CFDictionaryCreate(