Added MPPTaskImage for iOS vision tasks

This commit is contained in:
Prianka Liz Kariat 2023-02-06 16:29:53 +05:30
parent 4d8af4315f
commit cbccd472ab
3 changed files with 316 additions and 0 deletions

View File

@ -0,0 +1,15 @@
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
objc_library(
name = "MPPTaskImage",
srcs = ["sources/MPPTaskImage.h"],
hdrs = ["sources/MPPTaskImage.m"],
module_name = "MPPTaskImage",
sdk_frameworks = [
"CoreMedia",
"CoreVideo",
"UIKit",
]
)

View File

@ -0,0 +1,178 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <CoreMedia/CoreMedia.h>
#import <CoreVideo/CoreVideo.h>
#import <UIKit/UIKit.h>
NS_ASSUME_NONNULL_BEGIN
/** Types of image sources. */
typedef NSInteger GMLImageSourceType NS_TYPED_ENUM NS_SWIFT_NAME(MLImageSourceType);
/** Image source is a `UIImage`. */
static const GMLImageSourceType MPPTaskImageSourceTypeImage = 0;
/** Image source is a `CVPixelBuffer`. */
static const GMLImageSourceType MPPTaskImageSourceTypePixelBuffer = 1;
/** Image source is a `CMSampleBuffer`. */
static const GMLImageSourceType MPPTaskImageSourceTypeSampleBuffer = 2;
/** An image used in on-device machine learning using MediaPipe Task library. */
NS_SWIFT_NAME(MPPTaskImage)
@interface MPPTaskImage : NSObject
/** Width of the image in pixels. */
@property(nonatomic, readonly) CGFloat width;
/** Height of the image in pixels. */
@property(nonatomic, readonly) CGFloat height;
/**
* The display orientation of the image. If `imageSourceType` is `MPPTaskImageSourceTypeImage`, the
* default value is `image.imageOrientation`; otherwise the default value is `UIImageOrientationUp`.
* If the `MPPTaskImage` is being used as input for any MediaPipe vision tasks and is set to any
* orientation other than `UIImageOrientationUp`, inference will be performed on a rotated copy of
* the image according to the orientation.
*/
@property(nonatomic, readonly) UIImageOrientation orientation;
/** The type of the image source. */
@property(nonatomic, readonly) GMLImageSourceType imageSourceType;
/** The source image. `nil` if `imageSourceType` is not `.image`. */
@property(nonatomic, readonly, nullable) UIImage *image;
/** The source pixel buffer. `nil` if `imageSourceType` is not `.pixelBuffer`. */
@property(nonatomic, readonly, nullable) CVPixelBufferRef pixelBuffer;
/** The source sample buffer. `nil` if `imageSourceType` is not `.sampleBuffer`. */
@property(nonatomic, readonly, nullable) CMSampleBufferRef sampleBuffer;
/**
* Initializes an `MPPTaskImage` object with the given `UIImage`.
* The orientation of the newly created `MPPTaskImage` will be `UIImageOrientationUp`.
* Hence, if this image is used as input for any MediaPipe vision tasks, inference will be performed
* on the it without any rotation. To create an `MPPTaskImage` with a different orientation, please
* use `[MPPTaskImage initWithImage:orientation:error:]`.
*
* @param image The image to use as the source. Its `CGImage` property must not be `NULL`.
* @param error An optional error parameter populated when there is an error in initializing the
* `MPPTaskImage`.
*
* @return A new `MPPTaskImage` instance with the given image as the source. `nil` if the given
* `image` is `nil` or invalid.
*/
- (nullable instancetype)initWithUIImage:(UIImage *)image
error:(NSError **)error NS_DESIGNATED_INITIALIZER;
/**
* Initializes an `MPPTaskImage` object with the given `UIImabe` and orientation.
*
* If the newly created `MPPTaskImage` is used as input for any MediaPipe vision tasks, inference
* will be performed on a copy of the image rotated according to the orientation.
*
* @param image The image to use as the source. Its `CGImage` property must not be `NULL`.
* @param orientation The display orientation of the image. This will be stored in the property
* `orientation`. `MPPTaskImage`.
* @param error An optional error parameter populated when there is an error in initializing the
* `MPPTaskImage`.
*
* @return A new `MPPTaskImage` instance with the given image as the source. `nil` if the given
* `image` is `nil` or invalid.
*/
- (nullable instancetype)initWithUIImage:(UIImage *)image
orientation:(UIImageOrientation)orientation
error:(NSError **)error NS_DESIGNATED_INITIALIZER;
/**
* Initializes an `MPPTaskImage` object with the given pixel buffer.
*
* The orientation of the newly created `MPPTaskImage` will be `UIImageOrientationUp`.
* Hence, if this image is used as input for any MediaPipe vision tasks, inference will be performed
* on the it without any rotation. To create an `MPPTaskImage` with a different orientation, please
* use `[MPPTaskImage initWithPixelBuffer:orientation:error:]`.
*
* @param pixelBuffer The pixel buffer to use as the source. It will be retained by the new
* `MPPTaskImage` instance for the duration of its lifecycle.
* @param error An optional error parameter populated when there is an error in initializing the
* `MPPTaskImage`.
*
* @return A new `MPPTaskImage` instance with the given pixel buffer as the source. `nil` if the
* given pixel buffer is `nil` or invalid.
*/
- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer
error:(NSError **)error NS_DESIGNATED_INITIALIZER;
/**
* Initializes an `MPPTaskImage` object with the given pixel buffer and orientation.
*
* If the newly created `MPPTaskImage` is used as input for any MediaPipe vision tasks, inference
* will be performed on a copy of the image rotated according to the orientation.
*
* @param pixelBuffer The pixel buffer to use as the source. It will be retained by the new
* `MPPTaskImage` instance for the duration of its lifecycle.
* @param orientation The display orientation of the image.
* @param error An optional error parameter populated when there is an error in initializing the
* `MPPTaskImage`.
*
* @return A new `MPPTaskImage` instance with the given orientation and pixel buffer as the source.
* `nil` if the given pixel buffer is `nil` or invalid.
*/
- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer
orientation:(UIImageOrientation)orientation
NS_DESIGNATED_INITIALIZER;
/**
* Initializes an `MPPTaskImage` object with the given sample buffer.
*
* The orientation of the newly created `MPPTaskImage` will be `UIImageOrientationUp`.
* Hence, if this image is used as input for any MediaPipe vision tasks, inference will be performed
* on the it without any rotation. To create an `MPPTaskImage` with a different orientation, please
* use `[MPPTaskImage initWithSampleBuffer:orientation:error:]`.
*
* @param sampleBuffer The sample buffer to use as the source. It will be retained by the new
* `MPPTaskImage` instance for the duration of its lifecycle. The sample buffer must be based on
* a pixel buffer (not compressed data). In practice, it should be the video output of the camera on
* an iOS device, not other arbitrary types of `CMSampleBuffer`s.
* @return A new `MPPTaskImage` instance with the given sample buffer as the source. `nil` if the
* given sample buffer is `nil` or invalid.
*/
- (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
error:(NSError **)error NS_DESIGNATED_INITIALIZER;
/**
* Initializes an `MPPTaskImage` object with the given sample buffer and orientation.
*
* If the newly created `MPPTaskImage` is used as input for any MediaPipe vision tasks, inference
* will be performed on a copy of the image rotated according to the orientation.
*
* @param sampleBuffer The sample buffer to use as the source. It will be retained by the new
* `MPPTaskImage` instance for the duration of its lifecycle. The sample buffer must be based on
* a pixel buffer (not compressed data). In practice, it should be the video output of the camera on
* an iOS device, not other arbitrary types of `CMSampleBuffer`s.
* @param orientation The display orientation of the image.
* @return A new `MPPTaskImage` instance with the given orientation and sample buffer as the source.
* `nil` if the given sample buffer is `nil` or invalid.
*/
- (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
orientation:(UIImageOrientation)orientation
NS_DESIGNATED_INITIALIZER;
/** Unavailable. */
- (instancetype)init NS_UNAVAILABLE;
+ (instancetype)new NS_UNAVAILABLE;
@end
NS_ASSUME_NONNULL_END

View File

@ -0,0 +1,123 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "mediapipe/tasks/ios/vision/core/sources/MPPTaskImage.h"
#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h"
NS_ASSUME_NONNULL_BEGIN
@implementation MPPTaskImage
- (nullable instancetype)initWithUIImage:(UIImage *)image error:(NSError **)error {
return [self initWithUIImage:image orientation:image.orientation error:error];
}
- (nullable instancetype)initWithUIImage:(UIImage *)image
orientation:(UIImageOrientation)orientation
error:(NSError **)error {
if (image == nil) {
[MPPCommonUtils createCustomError:error
withCode:MPPTasksErrorCodeInvalidArgumentError
description:@"Image cannot be nil."];
}
if (image.CGImage == NULL) {
[MPPCommonUtils createCustomError:error
withCode:MPPTasksErrorCodeInvalidArgumentError
description:@"Image does not have a valid underlying CGImage. "
@"image.CGImage must be non nil."];
return nil;
}
self = [super init];
if (self) {
_imageSourceType = MPPTaskImageSourceTypeImage;
_orientation = orientation;
_image = image;
_width = image.size.width * image.scale;
_height = image.size.height * image.scale;
}
return self;
}
- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error {
return [self initWithPixelBuffer:pixelBuffer orientation:UIImageOrientationUp error:error];
}
- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer
orientation:(UIImageOrientation)orientation
error:(NSError **)error {
if (pixelBuffer == NULL) {
[MPPCommonUtils createCustomError:error
withCode:MPPTasksErrorCodeInvalidArgumentError
description:@"Pixel Buffer cannot be nil."];
return nil;
}
self = [super init];
if (self != nil) {
_imageSourceType = MPPTaskImageSourceTypePixelBuffer;
_orientation = orientation;
CVPixelBufferRetain(pixelBuffer);
_pixelBuffer = pixelBuffer;
_width = CVPixelBufferGetWidth(pixelBuffer);
_height = CVPixelBufferGetHeight(pixelBuffer);
}
return self;
}
- (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
error:(NSError **)error {
return [self initWithSampleBuffer:sampleBuffer orientation:UIImageOrientation error:error];
}
- (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
orientation:(UIImageOrientation)orientation
error:(NSError **)error {
if (!CMSampleBufferIsValid(sampleBuffer)) {
[MPPCommonUtils createCustomError:error
withCode:MPPTasksErrorCodeInvalidArgumentError
description:@"Sample buffer is not valid. Invoking "
@"CMSampleBufferIsValid(sampleBuffer) must return true."];
return nil;
}
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (imageBuffer == NULL) {
return nil;
}
self = [super init];
if (self != nil) {
_imageSourceType = MPPTaskImageSourceTypeSampleBuffer;
_orientation = orientation;
CFRetain(sampleBuffer);
_sampleBuffer = sampleBuffer;
_width = CVPixelBufferGetWidth(imageBuffer);
_height = CVPixelBufferGetHeight(imageBuffer);
}
return self;
}
- (void)dealloc {
if (_sampleBuffer != NULL) {
CFRelease(_sampleBuffer);
}
if (_pixelBuffer != NULL) {
CVPixelBufferRelease(_pixelBuffer);
}
}
@end
NS_ASSUME_NONNULL_END