From 0fe677b78f3f775c6a7c866b174858ca66380102 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 19 Oct 2023 19:24:40 +0530 Subject: [PATCH 1/6] Updated supported pixel formats in iOS image classifier Documentation --- .../image_classifier/sources/MPPImageClassifier.h | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h index 5b9b24fb6..8245dcbdf 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.h @@ -82,10 +82,9 @@ NS_SWIFT_NAME(ImageClassifier) * `.image`. * * This method supports classification of RGBA images. If your `MPImage` has a source type - * ofm`.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following + * of `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following * pixel format types: * 1. kCVPixelFormatType_32BGRA - * 2. kCVPixelFormatType_32RGBA * * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha * channel. @@ -108,7 +107,6 @@ NS_SWIFT_NAME(ImageClassifier) * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following * pixel format types: * 1. kCVPixelFormatType_32BGRA - * 2. kCVPixelFormatType_32RGBA * * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha * channel. @@ -137,7 +135,6 @@ NS_SWIFT_NAME(ImageClassifier) * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following * pixel format types: * 1. kCVPixelFormatType_32BGRA - * 2. kCVPixelFormatType_32RGBA * * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha * channel. @@ -165,7 +162,6 @@ NS_SWIFT_NAME(ImageClassifier) * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following * pixel format types: * 1. kCVPixelFormatType_32BGRA - * 2. kCVPixelFormatType_32RGBA * * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha * channel. @@ -203,7 +199,6 @@ NS_SWIFT_NAME(ImageClassifier) * .pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following * pixel format types: * 1. kCVPixelFormatType_32BGRA - * 2. kCVPixelFormatType_32RGBA * * If the input `MPImage` has a source type of `.image` ensure that the color space is RGB with an * Alpha channel. @@ -242,13 +237,12 @@ NS_SWIFT_NAME(ImageClassifier) * `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of the following * pixel format types: * 1. kCVPixelFormatType_32BGRA - * 2. kCVPixelFormatType_32RGBA * * If the input `MPImage` has a source type of `.image` ensure that the color space is RGB with an * Alpha channel. * * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you - * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its + * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32BGRA` using its * `videoSettings` property. * * @param image A live stream image data of type `MPImage` on which image classification is to be From 032d7a5d22988b0f6b15f45dc587f15a9c387e3a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 19 Oct 2023 19:56:44 +0530 Subject: [PATCH 2/6] Removed support for CVPixelBuffer of type 32RGBA --- .../tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm index 440b321b9..df61d4c32 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm @@ -120,8 +120,7 @@ static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size default: { [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError - description:@"Invalid source pixel buffer format. Expecting one of " - @"kCVPixelFormatType_32RGBA, kCVPixelFormatType_32BGRA"]; + description:@"Some internal error occured."]; return nullptr; } } @@ -149,7 +148,6 @@ static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size std::unique_ptr imageFrame = nullptr; switch (pixelBufferFormat) { - case kCVPixelFormatType_32RGBA: case kCVPixelFormatType_32BGRA: { CVPixelBufferLockBaseAddress(pixelBuffer, 0); imageFrame = [MPPPixelDataUtils @@ -165,9 +163,7 @@ static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size default: { [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError - description:@"Unsupported pixel format for CVPixelBuffer. Supported " - @"pixel format types are kCVPixelFormatType_32BGRA and " - @"kCVPixelFormatType_32RGBA"]; + description:@"Unsupported pixel format for CVPixelBuffer. Supported pixel format is kCVPixelFormatType_32BGRA"]; } } From ad6812206919a01b24aef45cc80e73d7fc2745d0 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 19 Oct 2023 19:58:40 +0530 Subject: [PATCH 3/6] Added support for creating CVPixelBuffer from C++ Images to iOS MPPImage Utils --- .../core/utils/sources/MPPImage+Utils.mm | 139 +++++++++++++++++- 1 file changed, 137 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm index df61d4c32..45915f019 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm @@ -30,6 +30,20 @@ namespace { using ::mediapipe::ImageFormat; using ::mediapipe::ImageFrame; +vImage_Buffer EmptyVImageBufferFromImageFrame(ImageFrame &imageFrame, bool shouldAllocate) { + UInt8 *data = shouldAllocate ? new UInt8[imageFrame.Height() * imageFrame.WidthStep()] : NULL; + return {.data = data, + .height = static_cast(imageFrame.Height()), + .width = static_cast(imageFrame.Width()), + .rowBytes = static_cast(imageFrame.WidthStep())}; +} + +vImage_Buffer VImageBufferFromImageFrame(ImageFrame &imageFrame) { + vImage_Buffer imageBuffer = EmptyVImageBufferFromImageFrame(imageFrame, false); + imageBuffer.data = imageFrame.MutablePixelData(); + return imageBuffer; +} + vImage_Buffer allocatedVImageBuffer(vImagePixelCount width, vImagePixelCount height, size_t rowBytes) { UInt8 *data = new UInt8[height * rowBytes]; @@ -37,7 +51,11 @@ vImage_Buffer allocatedVImageBuffer(vImagePixelCount width, vImagePixelCount hei } static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size_t size) { - delete (vImage_Buffer *)buffer; + delete[] buffer; +} + +static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { + delete[] refCon; } } // namespace @@ -51,6 +69,10 @@ static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size pixelBufferFormat:(OSType)pixelBufferFormatType error:(NSError **)error; ++ (UInt8 *)pixelDataFromImageFrame:(ImageFrame &)imageFrame + shouldCopy:(BOOL)shouldCopy + error:(NSError **)error; + @end @interface MPPCVPixelBufferUtils : NSObject @@ -58,6 +80,9 @@ static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size + (std::unique_ptr)imageFrameFromCVPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error; + ++ (CVPixelBufferRef)cvPixelBufferFromImageFrame:(ImageFrame &)imageFrame + error:(NSError **)error; @end @interface MPPCGImageUtils : NSObject @@ -138,6 +163,42 @@ static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size static_cast(destBuffer.data)); } ++ (UInt8 *)pixelDataFromImageFrame:(ImageFrame &)imageFrame + shouldCopy:(BOOL)shouldCopy + error:(NSError **)error { + vImage_Buffer sourceBuffer = VImageBufferFromImageFrame(imageFrame); + + // Pre-multiply the raw pixels from a `mediapipe::Image` before creating a `CGImage` to ensure + // that pixels are displayed correctly irrespective of their alpha values. + vImage_Error premultiplyError; + vImage_Buffer destinationBuffer; + + switch (imageFrame.Format()) { + case ImageFormat::SRGBA: { + destinationBuffer = + shouldCopy ? EmptyVImageBufferFromImageFrame(imageFrame, true) : sourceBuffer; + premultiplyError = vImagePremultiplyData_RGBA8888(&sourceBuffer, &destinationBuffer, kvImageNoFlags); + break; + } + default: { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInternalError + description:@"An internal error occured"]; + return NULL; + } + } + + if (premultiplyError != kvImageNoError) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInternalError + description:@"An internal error occured."]; + + return NULL; + } + + return (UInt8 *)destinationBuffer.data; +} + @end @implementation MPPCVPixelBufferUtils @@ -170,6 +231,64 @@ static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size return imageFrame; } ++ (CVPixelBufferRef)cvPixelBufferFromImageFrame:(ImageFrame &)imageFrame + error:(NSError **)error { + + // Supporting only RGBA and BGRA since creation of CVPixelBuffers with RGB format + // is restrictred in iOS. Thus, the APIs will never receive an input pixel buffer in RGB format + // and in turn the resulting image frame will never be of the RGB format. Moreover, writing unit + // tests for RGB images will also be not possible. + switch (imageFrame.Format()) { + case ImageFormat::SRGBA: + break; + default: { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInternalError + description:@"An internal error occured."]; + return NULL; + } + } + + UInt8 *pixelData = [MPPPixelDataUtils pixelDataFromImageFrame:imageFrame + shouldCopy:YES + error:error]; + + if (!pixelData) { + return NULL; + } + + const uint8_t permute_map[4] = {2, 1, 0, 3}; + vImage_Buffer sourceBuffer = EmptyVImageBufferFromImageFrame(imageFrame, NO); + sourceBuffer.data = pixelData; + + if (vImagePermuteChannels_ARGB8888(&sourceBuffer, &sourceBuffer, permute_map, kvImageNoFlags) != kvImageNoError) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInternalError + description:@"An internal error occured."]; + return NULL; + } + + CVPixelBufferRef outputBuffer; + + OSType pixelBufferFormatType = kCVPixelFormatType_32BGRA; + + + // If pixel data is copied, then pass in a release callback that will be invoked when the + // pixel buffer is destroyed. If data is not copied, the responsibility of deletion is on the + // owner of the data (a.k.a C++ Image Frame). + if(CVPixelBufferCreateWithBytes(kCFAllocatorDefault, imageFrame.Width(), imageFrame.Height(), + pixelBufferFormatType, pixelData, imageFrame.WidthStep(), + FreeRefConReleaseCallback, + pixelData, NULL, &outputBuffer) == kCVReturnSuccess) { + return outputBuffer; + } + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInternalError + description:@"An internal error occured."]; + return NULL; +} + + @end @implementation MPPCGImageUtils @@ -343,8 +462,24 @@ static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size return [self initWithUIImage:image orientation:sourceImage.orientation error:nil]; } + case MPPImageSourceTypePixelBuffer: { + if (!shouldCopyPixelData) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"When the source type is pixel buffer, you cannot request uncopied data"]; + return nil; + } + CVPixelBufferRef pixelBuffer = + [MPPCVPixelBufferUtils cvPixelBufferFromImageFrame:*(image.GetImageFrameSharedPtr()) + error:error]; + MPPImage *image = [self initWithPixelBuffer:pixelBuffer + orientation:sourceImage.orientation + error:nil]; + CVPixelBufferRelease(pixelBuffer); + return image; + } default: - // TODO Implement Other Source Types. + // TODO Implement CMSampleBuffer. return nil; } } From 4668d683d566983dcdff11130b9d324510a4b819 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 19 Oct 2023 19:59:09 +0530 Subject: [PATCH 4/6] Updated implementation of MPPImage Utils to reduce lines of code --- .../core/utils/sources/MPPImage+Utils.mm | 63 ++++++++----------- 1 file changed, 27 insertions(+), 36 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm index 45915f019..eb7d02c05 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm @@ -347,7 +347,14 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { CGBitmapInfo bitmapInfo = kCGImageAlphaNoneSkipLast | kCGBitmapByteOrderDefault; ImageFrame *internalImageFrame = imageFrame.get(); - size_t channelCount = 4; + + UInt8 *pixelData = [MPPPixelDataUtils pixelDataFromImageFrame:*internalImageFrame + shouldCopy:shouldCopyPixelData + error:error]; + + if (!pixelData) { + return NULL; + } switch (internalImageFrame->Format()) { case ImageFormat::SRGBA: { @@ -358,54 +365,38 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInternalError description:@"An internal error occured."]; - return nullptr; + return NULL; } - size_t bitsPerComponent = 8; - - vImage_Buffer sourceBuffer = { - .data = (void *)internalImageFrame->MutablePixelData(), - .height = static_cast(internalImageFrame->Height()), - .width = static_cast(internalImageFrame->Width()), - .rowBytes = static_cast(internalImageFrame->WidthStep())}; - - vImage_Buffer destBuffer; - CGDataProviderReleaseDataCallback callback = nullptr; - if (shouldCopyPixelData) { - destBuffer = allocatedVImageBuffer(static_cast(internalImageFrame->Width()), - static_cast(internalImageFrame->Height()), - static_cast(internalImageFrame->WidthStep())); - callback = FreeDataProviderReleaseCallback; - } else { - destBuffer = sourceBuffer; - } - - // Pre-multiply the raw pixels from a `mediapipe::Image` before creating a `CGImage` to ensure - // that pixels are displayed correctly irrespective of their alpha values. - vImage_Error premultiplyError = - vImagePremultiplyData_RGBA8888(&sourceBuffer, &destBuffer, kvImageNoFlags); - - if (premultiplyError != kvImageNoError) { - [MPPCommonUtils createCustomError:error - withCode:MPPTasksErrorCodeInternalError - description:@"An internal error occured."]; - - return nullptr; - } - CGDataProviderRef provider = CGDataProviderCreateWithData( - destBuffer.data, destBuffer.data, + pixelData, pixelData, internalImageFrame->WidthStep() * internalImageFrame->Height(), callback); + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); - CGImageRef cgImageRef = + + CGImageRef cgImageRef = NULL; + + if (provider && colorSpace) { + size_t bitsPerComponent = 8; + size_t channelCount = 4; + + cgImageRef = CGImageCreate(internalImageFrame->Width(), internalImageFrame->Height(), bitsPerComponent, bitsPerComponent * channelCount, internalImageFrame->WidthStep(), colorSpace, bitmapInfo, provider, nullptr, YES, kCGRenderingIntentDefault); + } + // Can safely pass `NULL` to these functions according to iOS docs. CGDataProviderRelease(provider); CGColorSpaceRelease(colorSpace); + + if (!cgImageRef) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInternalError + description:@"An internal error occured."]; + } return cgImageRef; } From b9c869494d6b80fd2f4e21b466e0f6f2d4492347 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 19 Oct 2023 19:59:59 +0530 Subject: [PATCH 5/6] Fixed formatting of MPPImage+Utils.mm --- .../core/utils/sources/MPPImage+Utils.mm | 119 +++++++++--------- 1 file changed, 58 insertions(+), 61 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm index eb7d02c05..34cd50973 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm @@ -54,9 +54,7 @@ static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size delete[] buffer; } -static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { - delete[] refCon; -} +static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { delete[] refCon; } } // namespace @@ -70,8 +68,8 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { error:(NSError **)error; + (UInt8 *)pixelDataFromImageFrame:(ImageFrame &)imageFrame - shouldCopy:(BOOL)shouldCopy - error:(NSError **)error; + shouldCopy:(BOOL)shouldCopy + error:(NSError **)error; @end @@ -80,9 +78,7 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { + (std::unique_ptr)imageFrameFromCVPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error; - -+ (CVPixelBufferRef)cvPixelBufferFromImageFrame:(ImageFrame &)imageFrame - error:(NSError **)error; ++ (CVPixelBufferRef)cvPixelBufferFromImageFrame:(ImageFrame &)imageFrame error:(NSError **)error; @end @interface MPPCGImageUtils : NSObject @@ -160,12 +156,12 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { // Uses default deleter return std::make_unique(imageFormat, width, height, destinationBytesPerRow, - static_cast(destBuffer.data)); + static_cast(destBuffer.data)); } + (UInt8 *)pixelDataFromImageFrame:(ImageFrame &)imageFrame shouldCopy:(BOOL)shouldCopy - error:(NSError **)error { + error:(NSError **)error { vImage_Buffer sourceBuffer = VImageBufferFromImageFrame(imageFrame); // Pre-multiply the raw pixels from a `mediapipe::Image` before creating a `CGImage` to ensure @@ -177,7 +173,8 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { case ImageFormat::SRGBA: { destinationBuffer = shouldCopy ? EmptyVImageBufferFromImageFrame(imageFrame, true) : sourceBuffer; - premultiplyError = vImagePremultiplyData_RGBA8888(&sourceBuffer, &destinationBuffer, kvImageNoFlags); + premultiplyError = + vImagePremultiplyData_RGBA8888(&sourceBuffer, &destinationBuffer, kvImageNoFlags); break; } default: { @@ -224,16 +221,15 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { default: { [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError - description:@"Unsupported pixel format for CVPixelBuffer. Supported pixel format is kCVPixelFormatType_32BGRA"]; + description:@"Unsupported pixel format for CVPixelBuffer. Supported " + @"pixel format is kCVPixelFormatType_32BGRA"]; } } return imageFrame; } -+ (CVPixelBufferRef)cvPixelBufferFromImageFrame:(ImageFrame &)imageFrame - error:(NSError **)error { - ++ (CVPixelBufferRef)cvPixelBufferFromImageFrame:(ImageFrame &)imageFrame error:(NSError **)error { // Supporting only RGBA and BGRA since creation of CVPixelBuffers with RGB format // is restrictred in iOS. Thus, the APIs will never receive an input pixel buffer in RGB format // and in turn the resulting image frame will never be of the RGB format. Moreover, writing unit @@ -249,46 +245,45 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { } } - UInt8 *pixelData = [MPPPixelDataUtils pixelDataFromImageFrame:imageFrame - shouldCopy:YES - error:error]; + UInt8 *pixelData = [MPPPixelDataUtils pixelDataFromImageFrame:imageFrame + shouldCopy:YES + error:error]; - if (!pixelData) { - return NULL; - } + if (!pixelData) { + return NULL; + } - const uint8_t permute_map[4] = {2, 1, 0, 3}; - vImage_Buffer sourceBuffer = EmptyVImageBufferFromImageFrame(imageFrame, NO); - sourceBuffer.data = pixelData; + const uint8_t permute_map[4] = {2, 1, 0, 3}; + vImage_Buffer sourceBuffer = EmptyVImageBufferFromImageFrame(imageFrame, NO); + sourceBuffer.data = pixelData; - if (vImagePermuteChannels_ARGB8888(&sourceBuffer, &sourceBuffer, permute_map, kvImageNoFlags) != kvImageNoError) { - [MPPCommonUtils createCustomError:error - withCode:MPPTasksErrorCodeInternalError - description:@"An internal error occured."]; - return NULL; - } - - CVPixelBufferRef outputBuffer; + if (vImagePermuteChannels_ARGB8888(&sourceBuffer, &sourceBuffer, permute_map, kvImageNoFlags) != + kvImageNoError) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInternalError + description:@"An internal error occured."]; + return NULL; + } - OSType pixelBufferFormatType = kCVPixelFormatType_32BGRA; + CVPixelBufferRef outputBuffer; + OSType pixelBufferFormatType = kCVPixelFormatType_32BGRA; - // If pixel data is copied, then pass in a release callback that will be invoked when the - // pixel buffer is destroyed. If data is not copied, the responsibility of deletion is on the - // owner of the data (a.k.a C++ Image Frame). - if(CVPixelBufferCreateWithBytes(kCFAllocatorDefault, imageFrame.Width(), imageFrame.Height(), + // If pixel data is copied, then pass in a release callback that will be invoked when the + // pixel buffer is destroyed. If data is not copied, the responsibility of deletion is on the + // owner of the data (a.k.a C++ Image Frame). + if (CVPixelBufferCreateWithBytes(kCFAllocatorDefault, imageFrame.Width(), imageFrame.Height(), pixelBufferFormatType, pixelData, imageFrame.WidthStep(), - FreeRefConReleaseCallback, - pixelData, NULL, &outputBuffer) == kCVReturnSuccess) { - return outputBuffer; - } - [MPPCommonUtils createCustomError:error - withCode:MPPTasksErrorCodeInternalError - description:@"An internal error occured."]; - return NULL; + FreeRefConReleaseCallback, pixelData, NULL, + &outputBuffer) == kCVReturnSuccess) { + return outputBuffer; + } + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInternalError + description:@"An internal error occured."]; + return NULL; } - @end @implementation MPPCGImageUtils @@ -371,31 +366,31 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { CGDataProviderReleaseDataCallback callback = nullptr; CGDataProviderRef provider = CGDataProviderCreateWithData( - pixelData, pixelData, - internalImageFrame->WidthStep() * internalImageFrame->Height(), callback); + pixelData, pixelData, internalImageFrame->WidthStep() * internalImageFrame->Height(), + callback); CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); CGImageRef cgImageRef = NULL; if (provider && colorSpace) { - size_t bitsPerComponent = 8; - size_t channelCount = 4; + size_t bitsPerComponent = 8; + size_t channelCount = 4; - cgImageRef = - CGImageCreate(internalImageFrame->Width(), internalImageFrame->Height(), bitsPerComponent, - bitsPerComponent * channelCount, internalImageFrame->WidthStep(), colorSpace, - bitmapInfo, provider, nullptr, YES, kCGRenderingIntentDefault); + cgImageRef = + CGImageCreate(internalImageFrame->Width(), internalImageFrame->Height(), bitsPerComponent, + bitsPerComponent * channelCount, internalImageFrame->WidthStep(), colorSpace, + bitmapInfo, provider, nullptr, YES, kCGRenderingIntentDefault); } // Can safely pass `NULL` to these functions according to iOS docs. CGDataProviderRelease(provider); CGColorSpaceRelease(colorSpace); - + if (!cgImageRef) { [MPPCommonUtils createCustomError:error - withCode:MPPTasksErrorCodeInternalError - description:@"An internal error occured."]; + withCode:MPPTasksErrorCodeInternalError + description:@"An internal error occured."]; } return cgImageRef; @@ -455,10 +450,12 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { } case MPPImageSourceTypePixelBuffer: { if (!shouldCopyPixelData) { - [MPPCommonUtils createCustomError:error - withCode:MPPTasksErrorCodeInvalidArgumentError - description:@"When the source type is pixel buffer, you cannot request uncopied data"]; - return nil; + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"When the source type is pixel buffer, you cannot request uncopied data"]; + return nil; } CVPixelBufferRef pixelBuffer = [MPPCVPixelBufferUtils cvPixelBufferFromImageFrame:*(image.GetImageFrameSharedPtr()) From af9a7e7e404dad4c745dcd3454b86d1acf7af5b5 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 19 Oct 2023 20:27:51 +0530 Subject: [PATCH 6/6] Added documentation --- .../core/utils/sources/MPPImage+Utils.mm | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm index 34cd50973..d8156a671 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm @@ -78,6 +78,24 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { d + (std::unique_ptr)imageFrameFromCVPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error; +// Always copies the pixel data of the image frame to the created `CVPixelBuffer`. + +// This method is used to create CVPixelBuffer from output images of tasks like `FaceStylizer` only +// when the input `MPImage` source type is `pixelBuffer`. +// +// The only possible 32 RGBA pixel format of input `CVPixelBuffer` is `kCVPixelFormatType_32BGRA`. +// But Mediapipe does not support inference on images of format `BGRA`. Hence the channels of the +// underlying pixel data of `CVPixelBuffer` are permuted to the supported RGBA format before passing +// them to the task for inference. The pixel format of the output images of any MediaPipe task will +// be the same as the pixel format of the input image. (RGBA in this case). +// +// Since creation of `CVPixelBuffer` from the output image pixels with a format of +// `kCVPixelFormatType_32RGBA` is not possible, the channels of the output C++ image `RGBA` have to +// be permuted to the format `BGRA`. When the pixels are copied to create `CVPixelBuffer` this does +// not pose a challenge. +// +// TODO: Investigate if permuting channels of output `mediapipe::Image` in place is possible for +// creating `CVPixelBuffer`s without copying the underlying pixels. + (CVPixelBufferRef)cvPixelBufferFromImageFrame:(ImageFrame &)imageFrame error:(NSError **)error; @end @@ -120,6 +138,9 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { d // Convert the raw pixel data to RGBA format and un-premultiply the alpha from the R, G, B values // since MediaPipe C++ APIs only accept un pre-multiplied channels. + // + // This method is commonly used for `MPImage`s of all source types. Hence supporting BGRA and RGBA + // formats. Only `pixelBuffer` source type is restricted to `BGRA` format. switch (pixelBufferFormatType) { case kCVPixelFormatType_32RGBA: { destBuffer = allocatedVImageBuffer((vImagePixelCount)width, (vImagePixelCount)height, @@ -128,6 +149,8 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { d break; } case kCVPixelFormatType_32BGRA: { + // Permute channels to `RGBA` since MediaPipe tasks don't support inference on images of + // format `BGRA`. const uint8_t permute_map[4] = {2, 1, 0, 3}; destBuffer = allocatedVImageBuffer((vImagePixelCount)width, (vImagePixelCount)height, destinationBytesPerRow); @@ -206,6 +229,8 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { d std::unique_ptr imageFrame = nullptr; switch (pixelBufferFormat) { + // Core Video only supports pixel data of order BGRA for 32 bit RGBA images. + // Thus other formats like `kCVPixelFormatType_32BGRA` don't need to be accounted for. case kCVPixelFormatType_32BGRA: { CVPixelBufferLockBaseAddress(pixelBuffer, 0); imageFrame = [MPPPixelDataUtils @@ -230,10 +255,6 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { d } + (CVPixelBufferRef)cvPixelBufferFromImageFrame:(ImageFrame &)imageFrame error:(NSError **)error { - // Supporting only RGBA and BGRA since creation of CVPixelBuffers with RGB format - // is restrictred in iOS. Thus, the APIs will never receive an input pixel buffer in RGB format - // and in turn the resulting image frame will never be of the RGB format. Moreover, writing unit - // tests for RGB images will also be not possible. switch (imageFrame.Format()) { case ImageFormat::SRGBA: break; @@ -269,9 +290,8 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { d OSType pixelBufferFormatType = kCVPixelFormatType_32BGRA; - // If pixel data is copied, then pass in a release callback that will be invoked when the - // pixel buffer is destroyed. If data is not copied, the responsibility of deletion is on the - // owner of the data (a.k.a C++ Image Frame). + // Since data is copied, pass in a release callback that will be invoked when the + // pixel buffer is destroyed. if (CVPixelBufferCreateWithBytes(kCFAllocatorDefault, imageFrame.Width(), imageFrame.Height(), pixelBufferFormatType, pixelData, imageFrame.WidthStep(), FreeRefConReleaseCallback, pixelData, NULL, @@ -450,6 +470,9 @@ static void FreeRefConReleaseCallback(void *refCon, const void *baseAddress) { d } case MPPImageSourceTypePixelBuffer: { if (!shouldCopyPixelData) { + // TODO: Investigate possibility of permuting channels of `mediapipe::Image` returned by + // vision tasks in place to ensure that we can support creating `CVPixelBuffer`s without + // copying the pixel data. [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError