Merge pull request #4066 from priankakariatyml:ios-ml-image
PiperOrigin-RevId: 507844473
This commit is contained in:
Copybara-Service
commit
fd6dfcfcab
21
mediapipe/tasks/ios/vision/core/BUILD
Normal file
21
mediapipe/tasks/ios/vision/core/BUILD
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
package(default_visibility = ["//mediapipe/tasks:internal"])
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
objc_library(
|
||||
name = "MPPImage",
|
||||
srcs = ["sources/MPPImage.h"],
|
||||
hdrs = ["sources/MPPImage.m"],
|
||||
module_name = "MPPImage",
|
||||
sdk_frameworks = [
|
||||
"CoreMedia",
|
||||
"CoreVideo",
|
||||
"UIKit",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/tasks/ios/common/utils:MPPCommonUtils",
|
||||
"//third_party/apple_frameworks:CoreMedia",
|
||||
"//third_party/apple_frameworks:CoreVideo",
|
||||
"//third_party/apple_frameworks:UIKit",
|
||||
],
|
||||
)
|
||||
176
mediapipe/tasks/ios/vision/core/sources/MPPImage.h
Normal file
176
mediapipe/tasks/ios/vision/core/sources/MPPImage.h
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <CoreMedia/CoreMedia.h>
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
/** Types of image sources. */
|
||||
typedef NSInteger MPPImageSourceType NS_TYPED_ENUM NS_SWIFT_NAME(MPImageSourceType);
|
||||
/** Image source is a `UIImage`. */
|
||||
static const MPPImageSourceType MPPImageSourceTypeImage = 0;
|
||||
/** Image source is a `CVPixelBuffer`. */
|
||||
static const MPPImageSourceType MPPImageSourceTypePixelBuffer = 1;
|
||||
/** Image source is a `CMSampleBuffer`. */
|
||||
static const MPPImageSourceType MPPImageSourceTypeSampleBuffer = 2;
|
||||
|
||||
/** An image used in on-device machine learning using MediaPipe Task library. */
|
||||
NS_SWIFT_NAME(MPImage)
|
||||
@interface MPPImage : NSObject
|
||||
|
||||
/** Width of the image in pixels. */
|
||||
@property(nonatomic, readonly) CGFloat width;
|
||||
|
||||
/** Height of the image in pixels. */
|
||||
@property(nonatomic, readonly) CGFloat height;
|
||||
|
||||
/**
|
||||
* The display orientation of the image. If `imageSourceType` is `MPPImageSourceTypeImage`, the
|
||||
* default value is `image.imageOrientation`; otherwise the default value is
|
||||
* `UIImageOrientationUp`. If the `MPPImage` is being used as input for any MediaPipe vision tasks
|
||||
* and is set to any orientation other than `UIImageOrientationUp`, inference will be performed on
|
||||
* a rotated copy of the image according to the orientation.
|
||||
*/
|
||||
@property(nonatomic, readonly) UIImageOrientation orientation;
|
||||
|
||||
/** The type of the image source. */
|
||||
@property(nonatomic, readonly) MPPImageSourceType imageSourceType;
|
||||
|
||||
/** The source image. `nil` if `imageSourceType` is not `.image`. */
|
||||
@property(nonatomic, readonly, nullable) UIImage *image;
|
||||
|
||||
/** The source pixel buffer. `nil` if `imageSourceType` is not `.pixelBuffer`. */
|
||||
@property(nonatomic, readonly, nullable) CVPixelBufferRef pixelBuffer;
|
||||
|
||||
/** The source sample buffer. `nil` if `imageSourceType` is not `.sampleBuffer`. */
|
||||
@property(nonatomic, readonly, nullable) CMSampleBufferRef sampleBuffer;
|
||||
|
||||
/**
|
||||
* Initializes an `MPPImage` object with the given `UIImage`.
|
||||
* The orientation of the newly created `MPPImage` will be `UIImageOrientationUp`.
|
||||
* Hence, if this image is used as input for any MediaPipe vision tasks, inference will be
|
||||
* performed on the it without any rotation. To create an `MPPImage` with a different orientation,
|
||||
* please use `[MPPImage initWithImage:orientation:error:]`.
|
||||
*
|
||||
* @param image The image to use as the source. Its `CGImage` property must not be `NULL`.
|
||||
* @param error An optional error parameter populated when there is an error in initializing the
|
||||
* `MPPImage`.
|
||||
*
|
||||
* @return A new `MPPImage` instance with the given image as the source. `nil` if the given
|
||||
* `image` is `nil` or invalid.
|
||||
*/
|
||||
- (nullable instancetype)initWithUIImage:(UIImage *)image error:(NSError **)error;
|
||||
|
||||
/**
|
||||
* Initializes an `MPPImage` object with the given `UIImabe` and orientation.
|
||||
*
|
||||
* If the newly created `MPPImage` is used as input for any MediaPipe vision tasks, inference
|
||||
* will be performed on a copy of the image rotated according to the orientation.
|
||||
*
|
||||
* @param image The image to use as the source. Its `CGImage` property must not be `NULL`.
|
||||
* @param orientation The display orientation of the image. This will be stored in the property
|
||||
* `orientation`. `MPPImage`.
|
||||
* @param error An optional error parameter populated when there is an error in initializing the
|
||||
* `MPPImage`.
|
||||
*
|
||||
* @return A new `MPPImage` instance with the given image as the source. `nil` if the given
|
||||
* `image` is `nil` or invalid.
|
||||
*/
|
||||
- (nullable instancetype)initWithUIImage:(UIImage *)image
|
||||
orientation:(UIImageOrientation)orientation
|
||||
error:(NSError **)error NS_DESIGNATED_INITIALIZER;
|
||||
|
||||
/**
|
||||
* Initializes an `MPPImage` object with the given pixel buffer.
|
||||
*
|
||||
* The orientation of the newly created `MPPImage` will be `UIImageOrientationUp`.
|
||||
* Hence, if this image is used as input for any MediaPipe vision tasks, inference will be
|
||||
* performed on the it without any rotation. To create an `MPPImage` with a different
|
||||
* orientation, please use `[MPPImage initWithPixelBuffer:orientation:error:]`.
|
||||
*
|
||||
* @param pixelBuffer The pixel buffer to use as the source. It will be retained by the new
|
||||
* `MPPImage` instance for the duration of its lifecycle.
|
||||
* @param error An optional error parameter populated when there is an error in initializing the
|
||||
* `MPPImage`.
|
||||
*
|
||||
* @return A new `MPPImage` instance with the given pixel buffer as the source. `nil` if the
|
||||
* given pixel buffer is `nil` or invalid.
|
||||
*/
|
||||
- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error;
|
||||
|
||||
/**
|
||||
* Initializes an `MPPImage` object with the given pixel buffer and orientation.
|
||||
*
|
||||
* If the newly created `MPPImage` is used as input for any MediaPipe vision tasks, inference
|
||||
* will be performed on a copy of the image rotated according to the orientation.
|
||||
*
|
||||
* @param pixelBuffer The pixel buffer to use as the source. It will be retained by the new
|
||||
* `MPPImage` instance for the duration of its lifecycle.
|
||||
* @param orientation The display orientation of the image.
|
||||
* @param error An optional error parameter populated when there is an error in initializing the
|
||||
* `MPPImage`.
|
||||
*
|
||||
* @return A new `MPPImage` instance with the given orientation and pixel buffer as the source.
|
||||
* `nil` if the given pixel buffer is `nil` or invalid.
|
||||
*/
|
||||
- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
||||
orientation:(UIImageOrientation)orientation
|
||||
NS_DESIGNATED_INITIALIZER;
|
||||
|
||||
/**
|
||||
* Initializes an `MPPImage` object with the given sample buffer.
|
||||
*
|
||||
* The orientation of the newly created `MPPImage` will be `UIImageOrientationUp`.
|
||||
* Hence, if this image is used as input for any MediaPipe vision tasks, inference will be
|
||||
* performed on the it without any rotation. To create an `MPPImage` with a different orientation,
|
||||
* please use `[MPPImage initWithSampleBuffer:orientation:error:]`.
|
||||
*
|
||||
* @param sampleBuffer The sample buffer to use as the source. It will be retained by the new
|
||||
* `MPPImage` instance for the duration of its lifecycle. The sample buffer must be based on
|
||||
* a pixel buffer (not compressed data). In practice, it should be the video output of the
|
||||
* camera on an iOS device, not other arbitrary types of `CMSampleBuffer`s.
|
||||
* @return A new `MPPImage` instance with the given sample buffer as the source. `nil` if the
|
||||
* given sample buffer is `nil` or invalid.
|
||||
*/
|
||||
- (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
error:(NSError **)error;
|
||||
|
||||
/**
|
||||
* Initializes an `MPPImage` object with the given sample buffer and orientation.
|
||||
*
|
||||
* If the newly created `MPPImage` is used as input for any MediaPipe vision tasks, inference
|
||||
* will be performed on a copy of the image rotated according to the orientation.
|
||||
*
|
||||
* @param sampleBuffer The sample buffer to use as the source. It will be retained by the new
|
||||
* `MPPImage` instance for the duration of its lifecycle. The sample buffer must be based on
|
||||
* a pixel buffer (not compressed data). In practice, it should be the video output of the
|
||||
* camera on an iOS device, not other arbitrary types of `CMSampleBuffer`s.
|
||||
* @param orientation The display orientation of the image.
|
||||
* @return A new `MPPImage` instance with the given orientation and sample buffer as the source.
|
||||
* `nil` if the given sample buffer is `nil` or invalid.
|
||||
*/
|
||||
- (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
orientation:(UIImageOrientation)orientation
|
||||
NS_DESIGNATED_INITIALIZER;
|
||||
|
||||
/** Unavailable. */
|
||||
- (instancetype)init NS_UNAVAILABLE;
|
||||
|
||||
+ (instancetype)new NS_UNAVAILABLE;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
123
mediapipe/tasks/ios/vision/core/sources/MPPImage.m
Normal file
123
mediapipe/tasks/ios/vision/core/sources/MPPImage.m
Normal file
|
|
@ -0,0 +1,123 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h"
|
||||
#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h"
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
||||
@implementation MPPImage
|
||||
|
||||
- (nullable instancetype)initWithUIImage:(UIImage *)image error:(NSError **)error {
|
||||
return [self initWithUIImage:image orientation:image.orientation error:error];
|
||||
}
|
||||
|
||||
- (nullable instancetype)initWithUIImage:(UIImage *)image
|
||||
orientation:(UIImageOrientation)orientation
|
||||
error:(NSError **)error {
|
||||
if (image == nil) {
|
||||
[MPPCommonUtils createCustomError:error
|
||||
withCode:MPPTasksErrorCodeInvalidArgumentError
|
||||
description:@"Image cannot be nil."];
|
||||
}
|
||||
if (image.CGImage == NULL) {
|
||||
[MPPCommonUtils createCustomError:error
|
||||
withCode:MPPTasksErrorCodeInvalidArgumentError
|
||||
description:@"Image does not have a valid underlying CGImage. "
|
||||
@"image.CGImage must be non nil."];
|
||||
return nil;
|
||||
}
|
||||
|
||||
self = [super init];
|
||||
if (self) {
|
||||
_imageSourceType = MPPTaskImageSourceTypeImage;
|
||||
_orientation = orientation;
|
||||
_image = image;
|
||||
_width = image.size.width * image.scale;
|
||||
_height = image.size.height * image.scale;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error {
|
||||
return [self initWithPixelBuffer:pixelBuffer orientation:UIImageOrientationUp error:error];
|
||||
}
|
||||
|
||||
- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer
|
||||
orientation:(UIImageOrientation)orientation
|
||||
error:(NSError **)error {
|
||||
if (pixelBuffer == NULL) {
|
||||
[MPPCommonUtils createCustomError:error
|
||||
withCode:MPPTasksErrorCodeInvalidArgumentError
|
||||
description:@"Pixel Buffer cannot be nil."];
|
||||
return nil;
|
||||
}
|
||||
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_imageSourceType = MPPTaskImageSourceTypePixelBuffer;
|
||||
_orientation = orientation;
|
||||
CVPixelBufferRetain(pixelBuffer);
|
||||
_pixelBuffer = pixelBuffer;
|
||||
_width = CVPixelBufferGetWidth(pixelBuffer);
|
||||
_height = CVPixelBufferGetHeight(pixelBuffer);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
error:(NSError **)error {
|
||||
return [self initWithSampleBuffer:sampleBuffer orientation:UIImageOrientation error:error];
|
||||
}
|
||||
|
||||
- (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer
|
||||
orientation:(UIImageOrientation)orientation
|
||||
error:(NSError **)error {
|
||||
if (!CMSampleBufferIsValid(sampleBuffer)) {
|
||||
[MPPCommonUtils createCustomError:error
|
||||
withCode:MPPTasksErrorCodeInvalidArgumentError
|
||||
description:@"Sample buffer is not valid. Invoking "
|
||||
@"CMSampleBufferIsValid(sampleBuffer) must return true."];
|
||||
return nil;
|
||||
}
|
||||
|
||||
CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
|
||||
if (imageBuffer == NULL) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
self = [super init];
|
||||
if (self != nil) {
|
||||
_imageSourceType = MPPTaskImageSourceTypeSampleBuffer;
|
||||
_orientation = orientation;
|
||||
CFRetain(sampleBuffer);
|
||||
_sampleBuffer = sampleBuffer;
|
||||
_width = CVPixelBufferGetWidth(imageBuffer);
|
||||
_height = CVPixelBufferGetHeight(imageBuffer);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
if (_sampleBuffer != NULL) {
|
||||
CFRelease(_sampleBuffer);
|
||||
}
|
||||
if (_pixelBuffer != NULL) {
|
||||
CVPixelBufferRelease(_pixelBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
Loading…
Reference in New Issue
Block a user