Merge pull request #4268 from priankakariatyml:object-detector-objc-tests
PiperOrigin-RevId: 527991967
This commit is contained in:
Copybara-Service
commit
bbbc0f98c5
|
|
@ -387,17 +387,14 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks";
|
||||||
|
|
||||||
NSArray<MPPCategory *> *expectedCategories = @[
|
NSArray<MPPCategory *> *expectedCategories = @[
|
||||||
[[MPPCategory alloc] initWithIndex:934
|
[[MPPCategory alloc] initWithIndex:934
|
||||||
score:0.622074f
|
score:0.753852f
|
||||||
categoryName:@"cheeseburger"
|
categoryName:@"cheeseburger"
|
||||||
displayName:nil],
|
displayName:nil],
|
||||||
[[MPPCategory alloc] initWithIndex:963
|
|
||||||
score:0.051214f
|
|
||||||
categoryName:@"meat loaf"
|
|
||||||
displayName:nil],
|
|
||||||
[[MPPCategory alloc] initWithIndex:925
|
[[MPPCategory alloc] initWithIndex:925
|
||||||
score:0.048719f
|
score:0.028609f
|
||||||
categoryName:@"guacamole"
|
categoryName:@"guacamole"
|
||||||
displayName:nil]
|
displayName:nil],
|
||||||
|
[[MPPCategory alloc] initWithIndex:932 score:0.027782f categoryName:@"bagel" displayName:nil]
|
||||||
|
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|
@ -420,7 +417,7 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks";
|
||||||
|
|
||||||
NSArray<MPPCategory *> *expectedCategories =
|
NSArray<MPPCategory *> *expectedCategories =
|
||||||
@[ [[MPPCategory alloc] initWithIndex:560
|
@[ [[MPPCategory alloc] initWithIndex:560
|
||||||
score:0.682305f
|
score:0.604605f
|
||||||
categoryName:@"folding chair"
|
categoryName:@"folding chair"
|
||||||
displayName:nil] ];
|
displayName:nil] ];
|
||||||
|
|
||||||
|
|
|
||||||
55
mediapipe/tasks/ios/test/vision/object_detector/BUILD
Normal file
55
mediapipe/tasks/ios/test/vision/object_detector/BUILD
Normal file
|
|
@ -0,0 +1,55 @@
|
||||||
|
load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test")
|
||||||
|
load(
|
||||||
|
"//mediapipe/framework/tool:ios.bzl",
|
||||||
|
"MPP_TASK_MINIMUM_OS_VERSION",
|
||||||
|
)
|
||||||
|
load(
|
||||||
|
"@org_tensorflow//tensorflow/lite:special_rules.bzl",
|
||||||
|
"tflite_ios_lab_runner",
|
||||||
|
)
|
||||||
|
|
||||||
|
package(default_visibility = ["//mediapipe/tasks:internal"])
|
||||||
|
|
||||||
|
licenses(["notice"])
|
||||||
|
|
||||||
|
# Default tags for filtering iOS targets. Targets are restricted to Apple platforms.
|
||||||
|
TFL_DEFAULT_TAGS = [
|
||||||
|
"apple",
|
||||||
|
]
|
||||||
|
|
||||||
|
# Following sanitizer tests are not supported by iOS test targets.
|
||||||
|
TFL_DISABLED_SANITIZER_TAGS = [
|
||||||
|
"noasan",
|
||||||
|
"nomsan",
|
||||||
|
"notsan",
|
||||||
|
]
|
||||||
|
|
||||||
|
objc_library(
|
||||||
|
name = "MPPObjectDetectorObjcTestLibrary",
|
||||||
|
testonly = 1,
|
||||||
|
srcs = ["MPPObjectDetectorTests.m"],
|
||||||
|
copts = [
|
||||||
|
"-ObjC++",
|
||||||
|
"-std=c++17",
|
||||||
|
"-x objective-c++",
|
||||||
|
],
|
||||||
|
data = [
|
||||||
|
"//mediapipe/tasks/testdata/vision:test_images",
|
||||||
|
"//mediapipe/tasks/testdata/vision:test_models",
|
||||||
|
],
|
||||||
|
deps = [
|
||||||
|
"//mediapipe/tasks/ios/common:MPPCommon",
|
||||||
|
"//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils",
|
||||||
|
"//mediapipe/tasks/ios/vision/object_detector:MPPObjectDetector",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
ios_unit_test(
|
||||||
|
name = "MPPObjectDetectorObjcTest",
|
||||||
|
minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION,
|
||||||
|
runner = tflite_ios_lab_runner("IOS_LATEST"),
|
||||||
|
tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS,
|
||||||
|
deps = [
|
||||||
|
":MPPObjectDetectorObjcTestLibrary",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
@ -0,0 +1,721 @@
|
||||||
|
// Copyright 2023 The MediaPipe Authors.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
#import <XCTest/XCTest.h>
|
||||||
|
|
||||||
|
#import "mediapipe/tasks/ios/common/sources/MPPCommon.h"
|
||||||
|
#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h"
|
||||||
|
#import "mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h"
|
||||||
|
|
||||||
|
static NSString *const kModelName = @"coco_ssd_mobilenet_v1_1.0_quant_2018_06_29";
|
||||||
|
static NSDictionary *const kCatsAndDogsImage = @{@"name" : @"cats_and_dogs", @"type" : @"jpg"};
|
||||||
|
static NSDictionary *const kCatsAndDogsRotatedImage =
|
||||||
|
@{@"name" : @"cats_and_dogs_rotated", @"type" : @"jpg"};
|
||||||
|
static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks";
|
||||||
|
static const float pixelDifferenceTolerance = 10.0f;
|
||||||
|
static const float scoreDifferenceTolerance = 0.02f;
|
||||||
|
|
||||||
|
#define AssertEqualErrors(error, expectedError) \
|
||||||
|
XCTAssertNotNil(error); \
|
||||||
|
XCTAssertEqualObjects(error.domain, expectedError.domain); \
|
||||||
|
XCTAssertEqual(error.code, expectedError.code); \
|
||||||
|
XCTAssertNotEqual( \
|
||||||
|
[error.localizedDescription rangeOfString:expectedError.localizedDescription].location, \
|
||||||
|
NSNotFound)
|
||||||
|
|
||||||
|
#define AssertEqualCategories(category, expectedCategory, detectionIndex, categoryIndex) \
|
||||||
|
XCTAssertEqual(category.index, expectedCategory.index, \
|
||||||
|
@"detection Index = %d category array index j = %d", detectionIndex, \
|
||||||
|
categoryIndex); \
|
||||||
|
XCTAssertEqualWithAccuracy(category.score, expectedCategory.score, scoreDifferenceTolerance, \
|
||||||
|
@"detection Index = %d, category array index j = %d", detectionIndex, \
|
||||||
|
categoryIndex); \
|
||||||
|
XCTAssertEqualObjects(category.categoryName, expectedCategory.categoryName, \
|
||||||
|
@"detection Index = %d, category array index j = %d", detectionIndex, \
|
||||||
|
categoryIndex); \
|
||||||
|
XCTAssertEqualObjects(category.displayName, expectedCategory.displayName, \
|
||||||
|
@"detection Index = %d, category array index j = %d", detectionIndex, \
|
||||||
|
categoryIndex);
|
||||||
|
|
||||||
|
#define AssertApproximatelyEqualBoundingBoxes(boundingBox, expectedBoundingBox, idx) \
|
||||||
|
XCTAssertEqualWithAccuracy(boundingBox.origin.x, expectedBoundingBox.origin.x, \
|
||||||
|
pixelDifferenceTolerance, @"index i = %d", idx); \
|
||||||
|
XCTAssertEqualWithAccuracy(boundingBox.origin.y, expectedBoundingBox.origin.y, \
|
||||||
|
pixelDifferenceTolerance, @"index i = %d", idx); \
|
||||||
|
XCTAssertEqualWithAccuracy(boundingBox.size.width, expectedBoundingBox.size.width, \
|
||||||
|
pixelDifferenceTolerance, @"index i = %d", idx); \
|
||||||
|
XCTAssertEqualWithAccuracy(boundingBox.size.height, expectedBoundingBox.size.height, \
|
||||||
|
pixelDifferenceTolerance, @"index i = %d", idx);
|
||||||
|
|
||||||
|
@interface MPPObjectDetectorTests : XCTestCase
|
||||||
|
@end
|
||||||
|
|
||||||
|
@implementation MPPObjectDetectorTests
|
||||||
|
|
||||||
|
#pragma mark Results
|
||||||
|
|
||||||
|
+ (MPPObjectDetectionResult *)expectedDetectionResultForCatsAndDogsImageWithTimestampInMilliseconds:
|
||||||
|
(NSInteger)timestampInMilliseconds {
|
||||||
|
NSArray<MPPDetection *> *detections = @[
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.69921875f categoryName:@"cat" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(608, 161, 381, 439)
|
||||||
|
keypoints:nil],
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.656250f categoryName:@"cat" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(57, 398, 392, 196)
|
||||||
|
keypoints:nil],
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.51171875f categoryName:@"cat" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(257, 395, 173, 202)
|
||||||
|
keypoints:nil],
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.48828125f categoryName:@"cat" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(363, 195, 330, 412)
|
||||||
|
keypoints:nil],
|
||||||
|
];
|
||||||
|
|
||||||
|
return [[MPPObjectDetectionResult alloc] initWithDetections:detections
|
||||||
|
timestampInMilliseconds:timestampInMilliseconds];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)assertDetections:(NSArray<MPPDetection *> *)detections
|
||||||
|
isEqualToExpectedDetections:(NSArray<MPPDetection *> *)expectedDetections {
|
||||||
|
for (int i = 0; i < detections.count; i++) {
|
||||||
|
MPPDetection *detection = detections[i];
|
||||||
|
XCTAssertNotNil(detection);
|
||||||
|
for (int j = 0; j < detection.categories.count; j++) {
|
||||||
|
AssertEqualCategories(detection.categories[j], expectedDetections[i].categories[j], i, j);
|
||||||
|
}
|
||||||
|
AssertApproximatelyEqualBoundingBoxes(detection.boundingBox, expectedDetections[i].boundingBox,
|
||||||
|
i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)assertObjectDetectionResult:(MPPObjectDetectionResult *)objectDetectionResult
|
||||||
|
isEqualToExpectedResult:(MPPObjectDetectionResult *)expectedObjectDetectionResult
|
||||||
|
expectedDetectionsCount:(NSInteger)expectedDetectionsCount {
|
||||||
|
XCTAssertNotNil(objectDetectionResult);
|
||||||
|
|
||||||
|
NSArray<MPPDetection *> *detectionsSubsetToCompare;
|
||||||
|
XCTAssertEqual(objectDetectionResult.detections.count, expectedDetectionsCount);
|
||||||
|
if (objectDetectionResult.detections.count > expectedObjectDetectionResult.detections.count) {
|
||||||
|
detectionsSubsetToCompare = [objectDetectionResult.detections
|
||||||
|
subarrayWithRange:NSMakeRange(0, expectedObjectDetectionResult.detections.count)];
|
||||||
|
} else {
|
||||||
|
detectionsSubsetToCompare = objectDetectionResult.detections;
|
||||||
|
}
|
||||||
|
|
||||||
|
[self assertDetections:detectionsSubsetToCompare
|
||||||
|
isEqualToExpectedDetections:expectedObjectDetectionResult.detections];
|
||||||
|
|
||||||
|
XCTAssertEqual(objectDetectionResult.timestampInMilliseconds,
|
||||||
|
expectedObjectDetectionResult.timestampInMilliseconds);
|
||||||
|
}
|
||||||
|
|
||||||
|
#pragma mark File
|
||||||
|
|
||||||
|
- (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension {
|
||||||
|
NSString *filePath = [[NSBundle bundleForClass:self.class] pathForResource:fileName
|
||||||
|
ofType:extension];
|
||||||
|
return filePath;
|
||||||
|
}
|
||||||
|
|
||||||
|
#pragma mark Object Detector Initializers
|
||||||
|
|
||||||
|
- (MPPObjectDetectorOptions *)objectDetectorOptionsWithModelName:(NSString *)modelName {
|
||||||
|
NSString *modelPath = [self filePathWithName:modelName extension:@"tflite"];
|
||||||
|
MPPObjectDetectorOptions *objectDetectorOptions = [[MPPObjectDetectorOptions alloc] init];
|
||||||
|
objectDetectorOptions.baseOptions.modelAssetPath = modelPath;
|
||||||
|
|
||||||
|
return objectDetectorOptions;
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)assertCreateObjectDetectorWithOptions:(MPPObjectDetectorOptions *)objectDetectorOptions
|
||||||
|
failsWithExpectedError:(NSError *)expectedError {
|
||||||
|
NSError *error = nil;
|
||||||
|
MPPObjectDetector *objectDetector =
|
||||||
|
[[MPPObjectDetector alloc] initWithOptions:objectDetectorOptions error:&error];
|
||||||
|
|
||||||
|
XCTAssertNil(objectDetector);
|
||||||
|
AssertEqualErrors(error, expectedError);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (MPPObjectDetector *)objectDetectorWithOptionsSucceeds:
|
||||||
|
(MPPObjectDetectorOptions *)objectDetectorOptions {
|
||||||
|
MPPObjectDetector *objectDetector =
|
||||||
|
[[MPPObjectDetector alloc] initWithOptions:objectDetectorOptions error:nil];
|
||||||
|
XCTAssertNotNil(objectDetector);
|
||||||
|
|
||||||
|
return objectDetector;
|
||||||
|
}
|
||||||
|
|
||||||
|
#pragma mark Assert Detection Results
|
||||||
|
|
||||||
|
- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo {
|
||||||
|
MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPObjectDetectorTests class]
|
||||||
|
fileName:fileInfo[@"name"]
|
||||||
|
ofType:fileInfo[@"type"]];
|
||||||
|
XCTAssertNotNil(image);
|
||||||
|
|
||||||
|
return image;
|
||||||
|
}
|
||||||
|
|
||||||
|
- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo
|
||||||
|
orientation:(UIImageOrientation)orientation {
|
||||||
|
MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPObjectDetectorTests class]
|
||||||
|
fileName:fileInfo[@"name"]
|
||||||
|
ofType:fileInfo[@"type"]
|
||||||
|
orientation:orientation];
|
||||||
|
XCTAssertNotNil(image);
|
||||||
|
|
||||||
|
return image;
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)assertResultsOfDetectInImage:(MPPImage *)mppImage
|
||||||
|
usingObjectDetector:(MPPObjectDetector *)objectDetector
|
||||||
|
maxResults:(NSInteger)maxResults
|
||||||
|
equalsObjectDetectionResult:(MPPObjectDetectionResult *)expectedObjectDetectionResult {
|
||||||
|
MPPObjectDetectionResult *objectDetectionResult = [objectDetector detectInImage:mppImage
|
||||||
|
error:nil];
|
||||||
|
|
||||||
|
[self assertObjectDetectionResult:objectDetectionResult
|
||||||
|
isEqualToExpectedResult:expectedObjectDetectionResult
|
||||||
|
expectedDetectionsCount:maxResults > 0 ? maxResults
|
||||||
|
: objectDetectionResult.detections.count];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)assertResultsOfDetectInImageWithFileInfo:(NSDictionary *)fileInfo
|
||||||
|
usingObjectDetector:(MPPObjectDetector *)objectDetector
|
||||||
|
maxResults:(NSInteger)maxResults
|
||||||
|
|
||||||
|
equalsObjectDetectionResult:
|
||||||
|
(MPPObjectDetectionResult *)expectedObjectDetectionResult {
|
||||||
|
MPPImage *mppImage = [self imageWithFileInfo:fileInfo];
|
||||||
|
|
||||||
|
[self assertResultsOfDetectInImage:mppImage
|
||||||
|
usingObjectDetector:objectDetector
|
||||||
|
maxResults:maxResults
|
||||||
|
equalsObjectDetectionResult:expectedObjectDetectionResult];
|
||||||
|
}
|
||||||
|
|
||||||
|
#pragma mark General Tests
|
||||||
|
|
||||||
|
- (void)testCreateObjectDetectorWithMissingModelPathFails {
|
||||||
|
NSString *modelPath = [self filePathWithName:@"" extension:@""];
|
||||||
|
|
||||||
|
NSError *error = nil;
|
||||||
|
MPPObjectDetector *objectDetector = [[MPPObjectDetector alloc] initWithModelPath:modelPath
|
||||||
|
error:&error];
|
||||||
|
XCTAssertNil(objectDetector);
|
||||||
|
|
||||||
|
NSError *expectedError = [NSError
|
||||||
|
errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey :
|
||||||
|
@"INVALID_ARGUMENT: ExternalFile must specify at least one of 'file_content', "
|
||||||
|
@"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'."
|
||||||
|
}];
|
||||||
|
AssertEqualErrors(error, expectedError);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testCreateObjectDetectorAllowlistAndDenylistFails {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
options.categoryAllowlist = @[ @"cat" ];
|
||||||
|
options.categoryDenylist = @[ @"dog" ];
|
||||||
|
|
||||||
|
[self assertCreateObjectDetectorWithOptions:options
|
||||||
|
failsWithExpectedError:
|
||||||
|
[NSError
|
||||||
|
errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey :
|
||||||
|
@"INVALID_ARGUMENT: `category_allowlist` and "
|
||||||
|
@"`category_denylist` are mutually exclusive options."
|
||||||
|
}]];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithModelPathSucceeds {
|
||||||
|
NSString *modelPath = [self filePathWithName:kModelName extension:@"tflite"];
|
||||||
|
MPPObjectDetector *objectDetector = [[MPPObjectDetector alloc] initWithModelPath:modelPath
|
||||||
|
error:nil];
|
||||||
|
XCTAssertNotNil(objectDetector);
|
||||||
|
|
||||||
|
[self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage
|
||||||
|
usingObjectDetector:objectDetector
|
||||||
|
maxResults:-1
|
||||||
|
equalsObjectDetectionResult:
|
||||||
|
[MPPObjectDetectorTests
|
||||||
|
expectedDetectionResultForCatsAndDogsImageWithTimestampInMilliseconds:
|
||||||
|
0]];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithOptionsSucceeds {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
[self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage
|
||||||
|
usingObjectDetector:objectDetector
|
||||||
|
maxResults:-1
|
||||||
|
equalsObjectDetectionResult:
|
||||||
|
[MPPObjectDetectorTests
|
||||||
|
expectedDetectionResultForCatsAndDogsImageWithTimestampInMilliseconds:
|
||||||
|
0]];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithMaxResultsSucceeds {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
|
||||||
|
const NSInteger maxResults = 4;
|
||||||
|
options.maxResults = maxResults;
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
[self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage
|
||||||
|
usingObjectDetector:objectDetector
|
||||||
|
maxResults:maxResults
|
||||||
|
equalsObjectDetectionResult:
|
||||||
|
[MPPObjectDetectorTests
|
||||||
|
expectedDetectionResultForCatsAndDogsImageWithTimestampInMilliseconds:
|
||||||
|
0]];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithScoreThresholdSucceeds {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
options.scoreThreshold = 0.68f;
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
NSArray<MPPDetection *> *detections = @[
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.69921875f categoryName:@"cat" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(608, 161, 381, 439)
|
||||||
|
keypoints:nil],
|
||||||
|
];
|
||||||
|
MPPObjectDetectionResult *expectedObjectDetectionResult =
|
||||||
|
[[MPPObjectDetectionResult alloc] initWithDetections:detections timestampInMilliseconds:0];
|
||||||
|
|
||||||
|
[self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage
|
||||||
|
usingObjectDetector:objectDetector
|
||||||
|
maxResults:-1
|
||||||
|
equalsObjectDetectionResult:expectedObjectDetectionResult];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithCategoryAllowlistSucceeds {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
options.categoryAllowlist = @[ @"cat" ];
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
NSArray<MPPDetection *> *detections = @[
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.69921875f categoryName:@"cat" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(608, 161, 381, 439)
|
||||||
|
keypoints:nil],
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.656250f categoryName:@"cat" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(57, 398, 392, 196)
|
||||||
|
keypoints:nil],
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.51171875f categoryName:@"cat" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(257, 395, 173, 202)
|
||||||
|
keypoints:nil],
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.48828125f categoryName:@"cat" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(363, 195, 330, 412)
|
||||||
|
keypoints:nil],
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.355469f categoryName:@"cat" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(275, 216, 610, 386)
|
||||||
|
keypoints:nil],
|
||||||
|
];
|
||||||
|
|
||||||
|
MPPObjectDetectionResult *expectedDetectionResult =
|
||||||
|
[[MPPObjectDetectionResult alloc] initWithDetections:detections timestampInMilliseconds:0];
|
||||||
|
|
||||||
|
[self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage
|
||||||
|
usingObjectDetector:objectDetector
|
||||||
|
maxResults:-1
|
||||||
|
equalsObjectDetectionResult:expectedDetectionResult];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithCategoryDenylistSucceeds {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
options.categoryDenylist = @[ @"cat" ];
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
NSArray<MPPDetection *> *detections = @[
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1
|
||||||
|
score:0.476562f
|
||||||
|
categoryName:@"teddy bear"
|
||||||
|
displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(780, 407, 314, 190)
|
||||||
|
keypoints:nil],
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1
|
||||||
|
score:0.390625f
|
||||||
|
categoryName:@"teddy bear"
|
||||||
|
displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(90, 225, 568, 366)
|
||||||
|
keypoints:nil],
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1
|
||||||
|
score:0.367188f
|
||||||
|
categoryName:@"teddy bear"
|
||||||
|
displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(888, 434, 187, 167)
|
||||||
|
keypoints:nil],
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.332031f categoryName:@"bed" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(79, 364, 1097, 224)
|
||||||
|
keypoints:nil],
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1
|
||||||
|
score:0.289062f
|
||||||
|
categoryName:@"teddy bear"
|
||||||
|
displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(605, 398, 445, 199)
|
||||||
|
keypoints:nil],
|
||||||
|
];
|
||||||
|
|
||||||
|
MPPObjectDetectionResult *expectedDetectionResult =
|
||||||
|
[[MPPObjectDetectionResult alloc] initWithDetections:detections timestampInMilliseconds:0];
|
||||||
|
|
||||||
|
[self assertResultsOfDetectInImageWithFileInfo:kCatsAndDogsImage
|
||||||
|
usingObjectDetector:objectDetector
|
||||||
|
maxResults:-1
|
||||||
|
equalsObjectDetectionResult:expectedDetectionResult];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithOrientationSucceeds {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
options.maxResults = 1;
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
NSArray<MPPDetection *> *detections = @[
|
||||||
|
[[MPPDetection alloc] initWithCategories:@[
|
||||||
|
[[MPPCategory alloc] initWithIndex:-1 score:0.699219f categoryName:@"cat" displayName:nil],
|
||||||
|
]
|
||||||
|
boundingBox:CGRectMake(0, 608, 439, 387)
|
||||||
|
keypoints:nil],
|
||||||
|
];
|
||||||
|
|
||||||
|
MPPObjectDetectionResult *expectedDetectionResult =
|
||||||
|
[[MPPObjectDetectionResult alloc] initWithDetections:detections timestampInMilliseconds:0];
|
||||||
|
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kCatsAndDogsRotatedImage
|
||||||
|
orientation:UIImageOrientationRight];
|
||||||
|
|
||||||
|
[self assertResultsOfDetectInImage:image
|
||||||
|
usingObjectDetector:objectDetector
|
||||||
|
maxResults:1
|
||||||
|
equalsObjectDetectionResult:expectedDetectionResult];
|
||||||
|
}
|
||||||
|
|
||||||
|
#pragma mark Running Mode Tests
|
||||||
|
|
||||||
|
- (void)testCreateObjectDetectorFailsWithResultListenerInNonLiveStreamMode {
|
||||||
|
MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo};
|
||||||
|
for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
|
||||||
|
options.runningMode = runningModesToTest[i];
|
||||||
|
options.completion =
|
||||||
|
^(MPPObjectDetectionResult *result, NSInteger timestampInMilliseconds, NSError *error) {
|
||||||
|
};
|
||||||
|
|
||||||
|
[self
|
||||||
|
assertCreateObjectDetectorWithOptions:options
|
||||||
|
failsWithExpectedError:
|
||||||
|
[NSError
|
||||||
|
errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey :
|
||||||
|
@"The vision task is in image or video mode, a "
|
||||||
|
@"user-defined result callback should not be provided."
|
||||||
|
}]];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testCreateObjectDetectorFailsWithMissingResultListenerInLiveStreamMode {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
|
||||||
|
options.runningMode = MPPRunningModeLiveStream;
|
||||||
|
|
||||||
|
[self assertCreateObjectDetectorWithOptions:options
|
||||||
|
failsWithExpectedError:
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey :
|
||||||
|
@"The vision task is in live stream mode, a "
|
||||||
|
@"user-defined result callback must be provided."
|
||||||
|
}]];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectFailsWithCallingWrongApiInImageMode {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage];
|
||||||
|
|
||||||
|
NSError *liveStreamApiCallError;
|
||||||
|
XCTAssertFalse([objectDetector detectAsyncInImage:image
|
||||||
|
timestampInMilliseconds:0
|
||||||
|
error:&liveStreamApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedLiveStreamApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with live "
|
||||||
|
@"stream mode. Current Running Mode: Image"
|
||||||
|
}];
|
||||||
|
|
||||||
|
AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError);
|
||||||
|
|
||||||
|
NSError *videoApiCallError;
|
||||||
|
XCTAssertFalse([objectDetector detectInVideoFrame:image
|
||||||
|
timestampInMilliseconds:0
|
||||||
|
error:&videoApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedVideoApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with "
|
||||||
|
@"video mode. Current Running Mode: Image"
|
||||||
|
}];
|
||||||
|
AssertEqualErrors(videoApiCallError, expectedVideoApiCallError);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectFailsWithCallingWrongApiInVideoMode {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
options.runningMode = MPPRunningModeVideo;
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage];
|
||||||
|
|
||||||
|
NSError *liveStreamApiCallError;
|
||||||
|
XCTAssertFalse([objectDetector detectAsyncInImage:image
|
||||||
|
timestampInMilliseconds:0
|
||||||
|
error:&liveStreamApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedLiveStreamApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with live "
|
||||||
|
@"stream mode. Current Running Mode: Video"
|
||||||
|
}];
|
||||||
|
|
||||||
|
AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError);
|
||||||
|
|
||||||
|
NSError *imageApiCallError;
|
||||||
|
XCTAssertFalse([objectDetector detectInImage:image error:&imageApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedImageApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with "
|
||||||
|
@"image mode. Current Running Mode: Video"
|
||||||
|
}];
|
||||||
|
AssertEqualErrors(imageApiCallError, expectedImageApiCallError);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectFailsWithCallingWrongApiInLiveStreamMode {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
|
||||||
|
options.runningMode = MPPRunningModeLiveStream;
|
||||||
|
options.completion =
|
||||||
|
^(MPPObjectDetectionResult *result, NSInteger timestampInMilliseconds, NSError *error) {
|
||||||
|
|
||||||
|
};
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage];
|
||||||
|
|
||||||
|
NSError *imageApiCallError;
|
||||||
|
XCTAssertFalse([objectDetector detectInImage:image error:&imageApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedImageApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with "
|
||||||
|
@"image mode. Current Running Mode: Live Stream"
|
||||||
|
}];
|
||||||
|
AssertEqualErrors(imageApiCallError, expectedImageApiCallError);
|
||||||
|
|
||||||
|
NSError *videoApiCallError;
|
||||||
|
XCTAssertFalse([objectDetector detectInVideoFrame:image
|
||||||
|
timestampInMilliseconds:0
|
||||||
|
error:&videoApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedVideoApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with "
|
||||||
|
@"video mode. Current Running Mode: Live Stream"
|
||||||
|
}];
|
||||||
|
AssertEqualErrors(videoApiCallError, expectedVideoApiCallError);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testClassifyWithVideoModeSucceeds {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
|
||||||
|
options.runningMode = MPPRunningModeVideo;
|
||||||
|
|
||||||
|
NSInteger maxResults = 4;
|
||||||
|
options.maxResults = maxResults;
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage];
|
||||||
|
|
||||||
|
for (int i = 0; i < 3; i++) {
|
||||||
|
MPPObjectDetectionResult *objectDetectionResult = [objectDetector detectInVideoFrame:image
|
||||||
|
timestampInMilliseconds:i
|
||||||
|
error:nil];
|
||||||
|
|
||||||
|
[self assertObjectDetectionResult:objectDetectionResult
|
||||||
|
isEqualToExpectedResult:
|
||||||
|
[MPPObjectDetectorTests
|
||||||
|
expectedDetectionResultForCatsAndDogsImageWithTimestampInMilliseconds:i]
|
||||||
|
expectedDetectionsCount:maxResults];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithOutOfOrderTimestampsAndLiveStreamModeFails {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
|
||||||
|
NSInteger maxResults = 4;
|
||||||
|
options.maxResults = maxResults;
|
||||||
|
|
||||||
|
options.runningMode = MPPRunningModeLiveStream;
|
||||||
|
|
||||||
|
XCTestExpectation *expectation = [[XCTestExpectation alloc]
|
||||||
|
initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"];
|
||||||
|
expectation.expectedFulfillmentCount = 1;
|
||||||
|
|
||||||
|
options.completion =
|
||||||
|
^(MPPObjectDetectionResult *result, NSInteger timestampInMilliseconds, NSError *error) {
|
||||||
|
[self assertObjectDetectionResult:result
|
||||||
|
isEqualToExpectedResult:
|
||||||
|
[MPPObjectDetectorTests
|
||||||
|
expectedDetectionResultForCatsAndDogsImageWithTimestampInMilliseconds:
|
||||||
|
timestampInMilliseconds]
|
||||||
|
expectedDetectionsCount:maxResults];
|
||||||
|
[expectation fulfill];
|
||||||
|
};
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage];
|
||||||
|
|
||||||
|
XCTAssertTrue([objectDetector detectAsyncInImage:image timestampInMilliseconds:1 error:nil]);
|
||||||
|
|
||||||
|
NSError *error;
|
||||||
|
XCTAssertFalse([objectDetector detectAsyncInImage:image timestampInMilliseconds:0 error:&error]);
|
||||||
|
|
||||||
|
NSError *expectedError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey :
|
||||||
|
@"INVALID_ARGUMENT: Input timestamp must be monotonically increasing."
|
||||||
|
}];
|
||||||
|
AssertEqualErrors(error, expectedError);
|
||||||
|
[self waitForExpectations:@[ expectation ] timeout:1.0];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithLiveStreamModeSucceeds {
|
||||||
|
MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName];
|
||||||
|
|
||||||
|
NSInteger maxResults = 4;
|
||||||
|
options.maxResults = maxResults;
|
||||||
|
|
||||||
|
options.runningMode = MPPRunningModeLiveStream;
|
||||||
|
|
||||||
|
NSInteger iterationCount = 100;
|
||||||
|
|
||||||
|
// Because of flow limiting, we cannot ensure that the callback will be
|
||||||
|
// invoked `iterationCount` times.
|
||||||
|
// An normal expectation will fail if expectation.fullfill() is not called
|
||||||
|
// `expectation.expectedFulfillmentCount` times.
|
||||||
|
// If `expectation.isInverted = true`, the test will only succeed if
|
||||||
|
// expectation is not fullfilled for the specified `expectedFulfillmentCount`.
|
||||||
|
// Since in our case we cannot predict how many times the expectation is
|
||||||
|
// supposed to be fullfilled setting,
|
||||||
|
// `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and
|
||||||
|
// `expectation.isInverted = true` ensures that test succeeds if
|
||||||
|
// expectation is fullfilled <= `iterationCount` times.
|
||||||
|
XCTestExpectation *expectation = [[XCTestExpectation alloc]
|
||||||
|
initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"];
|
||||||
|
expectation.expectedFulfillmentCount = iterationCount + 1;
|
||||||
|
expectation.inverted = YES;
|
||||||
|
|
||||||
|
options.completion =
|
||||||
|
^(MPPObjectDetectionResult *result, NSInteger timestampInMilliseconds, NSError *error) {
|
||||||
|
[self assertObjectDetectionResult:result
|
||||||
|
isEqualToExpectedResult:
|
||||||
|
[MPPObjectDetectorTests
|
||||||
|
expectedDetectionResultForCatsAndDogsImageWithTimestampInMilliseconds:
|
||||||
|
timestampInMilliseconds]
|
||||||
|
expectedDetectionsCount:maxResults];
|
||||||
|
[expectation fulfill];
|
||||||
|
};
|
||||||
|
|
||||||
|
MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
// TODO: Mimic initialization from CMSampleBuffer as live stream mode is most likely to be used
|
||||||
|
// with the iOS camera. AVCaptureVideoDataOutput sample buffer delegates provide frames of type
|
||||||
|
// `CMSampleBuffer`.
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage];
|
||||||
|
|
||||||
|
for (int i = 0; i < iterationCount; i++) {
|
||||||
|
XCTAssertTrue([objectDetector detectAsyncInImage:image timestampInMilliseconds:i error:nil]);
|
||||||
|
}
|
||||||
|
|
||||||
|
[self waitForExpectations:@[ expectation ] timeout:0.5];
|
||||||
|
}
|
||||||
|
|
||||||
|
@end
|
||||||
|
|
@ -70,6 +70,8 @@ NS_ASSUME_NONNULL_BEGIN
|
||||||
* @param roi A `CGRect` specifying the region of interest. If the input region of interest equals
|
* @param roi A `CGRect` specifying the region of interest. If the input region of interest equals
|
||||||
* `CGRectZero`, the returned `NormalizedRect` covers the whole image. Make sure that `roi` equals
|
* `CGRectZero`, the returned `NormalizedRect` covers the whole image. Make sure that `roi` equals
|
||||||
* `CGRectZero` if `ROIAllowed` is NO. Otherwise, an error will be returned.
|
* `CGRectZero` if `ROIAllowed` is NO. Otherwise, an error will be returned.
|
||||||
|
* @param imageSize A `CGSize` specifying the size of the image within which normalized rect is
|
||||||
|
* calculated.
|
||||||
* @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the
|
* @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the
|
||||||
* image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise.
|
* image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise.
|
||||||
* Mirrored orientations (`UIImageOrientationUpMirrored`, `UIImageOrientationDownMirrored`,
|
* Mirrored orientations (`UIImageOrientationUpMirrored`, `UIImageOrientationDownMirrored`,
|
||||||
|
|
@ -83,6 +85,7 @@ NS_ASSUME_NONNULL_BEGIN
|
||||||
*/
|
*/
|
||||||
- (std::optional<mediapipe::NormalizedRect>)
|
- (std::optional<mediapipe::NormalizedRect>)
|
||||||
normalizedRectFromRegionOfInterest:(CGRect)roi
|
normalizedRectFromRegionOfInterest:(CGRect)roi
|
||||||
|
imageSize:(CGSize)imageSize
|
||||||
imageOrientation:(UIImageOrientation)imageOrientation
|
imageOrientation:(UIImageOrientation)imageOrientation
|
||||||
ROIAllowed:(BOOL)ROIAllowed
|
ROIAllowed:(BOOL)ROIAllowed
|
||||||
error:(NSError **)error;
|
error:(NSError **)error;
|
||||||
|
|
|
||||||
|
|
@ -86,11 +86,12 @@ static const NSInteger kMPPOrientationDegreesLeft = -270;
|
||||||
}
|
}
|
||||||
|
|
||||||
- (std::optional<NormalizedRect>)normalizedRectFromRegionOfInterest:(CGRect)roi
|
- (std::optional<NormalizedRect>)normalizedRectFromRegionOfInterest:(CGRect)roi
|
||||||
|
imageSize:(CGSize)imageSize
|
||||||
imageOrientation:
|
imageOrientation:
|
||||||
(UIImageOrientation)imageOrientation
|
(UIImageOrientation)imageOrientation
|
||||||
ROIAllowed:(BOOL)ROIAllowed
|
ROIAllowed:(BOOL)ROIAllowed
|
||||||
error:(NSError **)error {
|
error:(NSError **)error {
|
||||||
if (CGRectEqualToRect(roi, CGRectZero) && !ROIAllowed) {
|
if (!CGRectEqualToRect(roi, CGRectZero) && !ROIAllowed) {
|
||||||
[MPPCommonUtils createCustomError:error
|
[MPPCommonUtils createCustomError:error
|
||||||
withCode:MPPTasksErrorCodeInvalidArgumentError
|
withCode:MPPTasksErrorCodeInvalidArgumentError
|
||||||
description:@"This task doesn't support region-of-interest."];
|
description:@"This task doesn't support region-of-interest."];
|
||||||
|
|
@ -102,8 +103,6 @@ static const NSInteger kMPPOrientationDegreesLeft = -270;
|
||||||
NormalizedRect normalizedRect;
|
NormalizedRect normalizedRect;
|
||||||
normalizedRect.set_x_center(CGRectGetMidX(calculatedRoi));
|
normalizedRect.set_x_center(CGRectGetMidX(calculatedRoi));
|
||||||
normalizedRect.set_y_center(CGRectGetMidY(calculatedRoi));
|
normalizedRect.set_y_center(CGRectGetMidY(calculatedRoi));
|
||||||
normalizedRect.set_width(CGRectGetWidth(calculatedRoi));
|
|
||||||
normalizedRect.set_height(CGRectGetHeight(calculatedRoi));
|
|
||||||
|
|
||||||
int rotationDegrees = 0;
|
int rotationDegrees = 0;
|
||||||
switch (imageOrientation) {
|
switch (imageOrientation) {
|
||||||
|
|
@ -134,6 +133,23 @@ static const NSInteger kMPPOrientationDegreesLeft = -270;
|
||||||
|
|
||||||
normalizedRect.set_rotation(rotationDegrees * M_PI / kMPPOrientationDegreesDown);
|
normalizedRect.set_rotation(rotationDegrees * M_PI / kMPPOrientationDegreesDown);
|
||||||
|
|
||||||
|
// For 90° and 270° rotations, we need to swap width and height.
|
||||||
|
// This is due to the internal behavior of ImageToTensorCalculator, which:
|
||||||
|
// - first denormalizes the provided rect by multiplying the rect width or height by the image
|
||||||
|
// width or height, repectively.
|
||||||
|
// - then rotates this by denormalized rect by the provided rotation, and uses this for cropping,
|
||||||
|
// - then finally rotates this back.
|
||||||
|
if (rotationDegrees % 180 == 0) {
|
||||||
|
normalizedRect.set_width(CGRectGetWidth(calculatedRoi));
|
||||||
|
normalizedRect.set_height(CGRectGetHeight(calculatedRoi));
|
||||||
|
} else {
|
||||||
|
const float width = CGRectGetHeight(calculatedRoi) * imageSize.height / imageSize.width;
|
||||||
|
const float height = CGRectGetWidth(calculatedRoi) * imageSize.width / imageSize.height;
|
||||||
|
|
||||||
|
normalizedRect.set_width(width);
|
||||||
|
normalizedRect.set_height(height);
|
||||||
|
}
|
||||||
|
|
||||||
return normalizedRect;
|
return normalizedRect;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -120,6 +120,7 @@ static NSString *const kTaskGraphName =
|
||||||
error:(NSError **)error {
|
error:(NSError **)error {
|
||||||
std::optional<NormalizedRect> rect =
|
std::optional<NormalizedRect> rect =
|
||||||
[_visionTaskRunner normalizedRectFromRegionOfInterest:roi
|
[_visionTaskRunner normalizedRectFromRegionOfInterest:roi
|
||||||
|
imageSize:CGSizeMake(image.width, image.height)
|
||||||
imageOrientation:image.orientation
|
imageOrientation:image.orientation
|
||||||
ROIAllowed:YES
|
ROIAllowed:YES
|
||||||
error:error];
|
error:error];
|
||||||
|
|
@ -154,6 +155,7 @@ static NSString *const kTaskGraphName =
|
||||||
error:(NSError **)error {
|
error:(NSError **)error {
|
||||||
std::optional<NormalizedRect> rect =
|
std::optional<NormalizedRect> rect =
|
||||||
[_visionTaskRunner normalizedRectFromRegionOfInterest:roi
|
[_visionTaskRunner normalizedRectFromRegionOfInterest:roi
|
||||||
|
imageSize:CGSizeMake(image.width, image.height)
|
||||||
imageOrientation:image.orientation
|
imageOrientation:image.orientation
|
||||||
ROIAllowed:YES
|
ROIAllowed:YES
|
||||||
error:error];
|
error:error];
|
||||||
|
|
|
||||||
|
|
@ -58,6 +58,5 @@ objc_library(
|
||||||
"//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner",
|
"//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner",
|
||||||
"//mediapipe/tasks/ios/vision/object_detector/utils:MPPObjectDetectionResultHelpers",
|
"//mediapipe/tasks/ios/vision/object_detector/utils:MPPObjectDetectionResultHelpers",
|
||||||
"//mediapipe/tasks/ios/vision/object_detector/utils:MPPObjectDetectorOptionsHelpers",
|
"//mediapipe/tasks/ios/vision/object_detector/utils:MPPObjectDetectorOptionsHelpers",
|
||||||
"//third_party/apple_frameworks:UIKit",
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -109,28 +109,6 @@ NS_SWIFT_NAME(ObjectDetector)
|
||||||
error:(NSError **)error
|
error:(NSError **)error
|
||||||
NS_SWIFT_NAME(detect(image:));
|
NS_SWIFT_NAME(detect(image:));
|
||||||
|
|
||||||
/**
|
|
||||||
* Performs object detectionon the provided `MPPImage` cropped to the specified region of
|
|
||||||
* interest. Rotation will be applied on the cropped image according to the `orientation` property
|
|
||||||
* of the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with
|
|
||||||
* `MPPRunningModeImage`.
|
|
||||||
*
|
|
||||||
* @param image The `MPPImage` on which object detection is to be performed.
|
|
||||||
* @param roi A `CGRect` specifying the region of interest within the given `MPPImage`, on which
|
|
||||||
* object detection should be performed.
|
|
||||||
* @param error An optional error parameter populated when there is an error in performing object
|
|
||||||
* detection on the input image.
|
|
||||||
*
|
|
||||||
* @return An `MPPObjectDetectionResult` object that contains a list of detections, each detection
|
|
||||||
* has a bounding box that is expressed in the unrotated input frame of reference coordinates
|
|
||||||
* system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying
|
|
||||||
* image data.
|
|
||||||
*/
|
|
||||||
- (nullable MPPObjectDetectionResult *)detectInImage:(MPPImage *)image
|
|
||||||
regionOfInterest:(CGRect)roi
|
|
||||||
error:(NSError **)error
|
|
||||||
NS_SWIFT_NAME(detect(image:regionOfInterest:));
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Performs object detection on the provided video frame of type `MPPImage` using the whole
|
* Performs object detection on the provided video frame of type `MPPImage` using the whole
|
||||||
* image as region of interest. Rotation will be applied according to the `orientation` property of
|
* image as region of interest. Rotation will be applied according to the `orientation` property of
|
||||||
|
|
@ -153,36 +131,6 @@ NS_SWIFT_NAME(ObjectDetector)
|
||||||
error:(NSError **)error
|
error:(NSError **)error
|
||||||
NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:));
|
NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:));
|
||||||
|
|
||||||
/**
|
|
||||||
* Performs object detection on the provided video frame of type `MPPImage` cropped to the
|
|
||||||
* specified region of interest. Rotation will be applied according to the `orientation` property of
|
|
||||||
* the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with
|
|
||||||
* `MPPRunningModeVideo`.
|
|
||||||
*
|
|
||||||
* It's required to provide the video frame's timestamp (in milliseconds). The input timestamps must
|
|
||||||
* be monotonically increasing.
|
|
||||||
*
|
|
||||||
* @param image A live stream image data of type `MPPImage` on which object detection is to be
|
|
||||||
* performed.
|
|
||||||
* @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input
|
|
||||||
* timestamps must be monotonically increasing.
|
|
||||||
* @param roi A `CGRect` specifying the region of interest within the given `MPPImage`, on which
|
|
||||||
* object detection should be performed.
|
|
||||||
*
|
|
||||||
* @param error An optional error parameter populated when there is an error in performing object
|
|
||||||
* detection on the input image.
|
|
||||||
*
|
|
||||||
* @return An `MPPObjectDetectionResult` object that contains a list of detections, each detection
|
|
||||||
* has a bounding box that is expressed in the unrotated input frame of reference coordinates
|
|
||||||
* system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying
|
|
||||||
* image data.
|
|
||||||
*/
|
|
||||||
- (nullable MPPObjectDetectionResult *)detectInVideoFrame:(MPPImage *)image
|
|
||||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
|
||||||
regionOfInterest:(CGRect)roi
|
|
||||||
error:(NSError **)error
|
|
||||||
NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:regionOfInterest:));
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sends live stream image data of type `MPPImage` to perform object detection using the whole
|
* Sends live stream image data of type `MPPImage` to perform object detection using the whole
|
||||||
* image as region of interest. Rotation will be applied according to the `orientation` property of
|
* image as region of interest. Rotation will be applied according to the `orientation` property of
|
||||||
|
|
@ -207,33 +155,6 @@ NS_SWIFT_NAME(ObjectDetector)
|
||||||
error:(NSError **)error
|
error:(NSError **)error
|
||||||
NS_SWIFT_NAME(detectAsync(image:timestampInMilliseconds:));
|
NS_SWIFT_NAME(detectAsync(image:timestampInMilliseconds:));
|
||||||
|
|
||||||
/**
|
|
||||||
* Sends live stream image data of type `MPPImage` to perform object detection, cropped to the
|
|
||||||
* specified region of interest. Rotation will be applied according to the `orientation` property
|
|
||||||
* of the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with
|
|
||||||
* `MPPRunningModeLiveStream`. Results are provided asynchronously via the `completion` callback
|
|
||||||
* provided in the `MPPObjectDetectorOptions`.
|
|
||||||
*
|
|
||||||
* It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent
|
|
||||||
* to the object detector. The input timestamps must be monotonically increasing.
|
|
||||||
*
|
|
||||||
* @param image A live stream image data of type `MPPImage` on which object detection is to be
|
|
||||||
* performed.
|
|
||||||
* @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input
|
|
||||||
* image is sent to the object detector. The input timestamps must be monotonically increasing.
|
|
||||||
* @param roi A `CGRect` specifying the region of interest within the given live stream image data
|
|
||||||
* of type `MPPImage`, on which iobject detection should be performed.
|
|
||||||
* @param error An optional error parameter populated when there is an error in performing object
|
|
||||||
* detection on the input live stream image data.
|
|
||||||
*
|
|
||||||
* @return `YES` if the image was sent to the task successfully, otherwise `NO`.
|
|
||||||
*/
|
|
||||||
- (BOOL)detectAsyncInImage:(MPPImage *)image
|
|
||||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
|
||||||
regionOfInterest:(CGRect)roi
|
|
||||||
error:(NSError **)error
|
|
||||||
NS_SWIFT_NAME(detectAsync(image:timestampInMilliseconds:regionOfInterest:));
|
|
||||||
|
|
||||||
- (instancetype)init NS_UNAVAILABLE;
|
- (instancetype)init NS_UNAVAILABLE;
|
||||||
|
|
||||||
+ (instancetype)new NS_UNAVAILABLE;
|
+ (instancetype)new NS_UNAVAILABLE;
|
||||||
|
|
|
||||||
|
|
@ -123,13 +123,42 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG
|
||||||
return [self initWithOptions:options error:error];
|
return [self initWithOptions:options error:error];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
- (std::optional<PacketMap>)inputPacketMapWithMPPImage:(MPPImage *)image
|
||||||
|
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
||||||
|
error:(NSError **)error {
|
||||||
|
std::optional<NormalizedRect> rect =
|
||||||
|
[_visionTaskRunner normalizedRectFromRegionOfInterest:CGRectZero
|
||||||
|
imageSize:CGSizeMake(image.width, image.height)
|
||||||
|
imageOrientation:image.orientation
|
||||||
|
ROIAllowed:NO
|
||||||
|
error:error];
|
||||||
|
if (!rect.has_value()) {
|
||||||
|
return std::nullopt;
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image
|
||||||
|
timestampInMilliseconds:timestampInMilliseconds
|
||||||
|
error:error];
|
||||||
|
if (imagePacket.IsEmpty()) {
|
||||||
|
return std::nullopt;
|
||||||
|
}
|
||||||
|
|
||||||
|
Packet normalizedRectPacket =
|
||||||
|
[MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()
|
||||||
|
timestampInMilliseconds:timestampInMilliseconds];
|
||||||
|
|
||||||
|
PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket);
|
||||||
|
return inputPacketMap;
|
||||||
|
}
|
||||||
|
|
||||||
- (nullable MPPObjectDetectionResult *)detectInImage:(MPPImage *)image
|
- (nullable MPPObjectDetectionResult *)detectInImage:(MPPImage *)image
|
||||||
regionOfInterest:(CGRect)roi
|
regionOfInterest:(CGRect)roi
|
||||||
error:(NSError **)error {
|
error:(NSError **)error {
|
||||||
std::optional<NormalizedRect> rect =
|
std::optional<NormalizedRect> rect =
|
||||||
[_visionTaskRunner normalizedRectFromRegionOfInterest:roi
|
[_visionTaskRunner normalizedRectFromRegionOfInterest:roi
|
||||||
|
imageSize:CGSizeMake(image.width, image.height)
|
||||||
imageOrientation:image.orientation
|
imageOrientation:image.orientation
|
||||||
ROIAllowed:YES
|
ROIAllowed:NO
|
||||||
error:error];
|
error:error];
|
||||||
if (!rect.has_value()) {
|
if (!rect.has_value()) {
|
||||||
return nil;
|
return nil;
|
||||||
|
|
@ -156,45 +185,15 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG
|
||||||
.value()[kDetectionsStreamName.cppString]];
|
.value()[kDetectionsStreamName.cppString]];
|
||||||
}
|
}
|
||||||
|
|
||||||
- (std::optional<PacketMap>)inputPacketMapWithMPPImage:(MPPImage *)image
|
|
||||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
|
||||||
regionOfInterest:(CGRect)roi
|
|
||||||
error:(NSError **)error {
|
|
||||||
std::optional<NormalizedRect> rect =
|
|
||||||
[_visionTaskRunner normalizedRectFromRegionOfInterest:roi
|
|
||||||
imageOrientation:image.orientation
|
|
||||||
ROIAllowed:YES
|
|
||||||
error:error];
|
|
||||||
if (!rect.has_value()) {
|
|
||||||
return std::nullopt;
|
|
||||||
}
|
|
||||||
|
|
||||||
Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image
|
|
||||||
timestampInMilliseconds:timestampInMilliseconds
|
|
||||||
error:error];
|
|
||||||
if (imagePacket.IsEmpty()) {
|
|
||||||
return std::nullopt;
|
|
||||||
}
|
|
||||||
|
|
||||||
Packet normalizedRectPacket =
|
|
||||||
[MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()
|
|
||||||
timestampInMilliseconds:timestampInMilliseconds];
|
|
||||||
|
|
||||||
PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket);
|
|
||||||
return inputPacketMap;
|
|
||||||
}
|
|
||||||
|
|
||||||
- (nullable MPPObjectDetectionResult *)detectInImage:(MPPImage *)image error:(NSError **)error {
|
- (nullable MPPObjectDetectionResult *)detectInImage:(MPPImage *)image error:(NSError **)error {
|
||||||
return [self detectInImage:image regionOfInterest:CGRectZero error:error];
|
return [self detectInImage:image regionOfInterest:CGRectZero error:error];
|
||||||
}
|
}
|
||||||
|
|
||||||
- (nullable MPPObjectDetectionResult *)detectInVideoFrame:(MPPImage *)image
|
- (nullable MPPObjectDetectionResult *)detectInVideoFrame:(MPPImage *)image
|
||||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
||||||
regionOfInterest:(CGRect)roi
|
|
||||||
error:(NSError **)error {
|
error:(NSError **)error {
|
||||||
std::optional<PacketMap> inputPacketMap = [self inputPacketMapWithMPPImage:image
|
std::optional<PacketMap> inputPacketMap = [self inputPacketMapWithMPPImage:image
|
||||||
timestampInMilliseconds:timestampInMilliseconds
|
timestampInMilliseconds:timestampInMilliseconds
|
||||||
regionOfInterest:roi
|
|
||||||
error:error];
|
error:error];
|
||||||
if (!inputPacketMap.has_value()) {
|
if (!inputPacketMap.has_value()) {
|
||||||
return nil;
|
return nil;
|
||||||
|
|
@ -212,22 +211,11 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG
|
||||||
.value()[kDetectionsStreamName.cppString]];
|
.value()[kDetectionsStreamName.cppString]];
|
||||||
}
|
}
|
||||||
|
|
||||||
- (nullable MPPObjectDetectionResult *)detectInVideoFrame:(MPPImage *)image
|
|
||||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
|
||||||
error:(NSError **)error {
|
|
||||||
return [self detectInVideoFrame:image
|
|
||||||
timestampInMilliseconds:timestampInMilliseconds
|
|
||||||
regionOfInterest:CGRectZero
|
|
||||||
error:error];
|
|
||||||
}
|
|
||||||
|
|
||||||
- (BOOL)detectAsyncInImage:(MPPImage *)image
|
- (BOOL)detectAsyncInImage:(MPPImage *)image
|
||||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
||||||
regionOfInterest:(CGRect)roi
|
|
||||||
error:(NSError **)error {
|
error:(NSError **)error {
|
||||||
std::optional<PacketMap> inputPacketMap = [self inputPacketMapWithMPPImage:image
|
std::optional<PacketMap> inputPacketMap = [self inputPacketMapWithMPPImage:image
|
||||||
timestampInMilliseconds:timestampInMilliseconds
|
timestampInMilliseconds:timestampInMilliseconds
|
||||||
regionOfInterest:roi
|
|
||||||
error:error];
|
error:error];
|
||||||
if (!inputPacketMap.has_value()) {
|
if (!inputPacketMap.has_value()) {
|
||||||
return NO;
|
return NO;
|
||||||
|
|
@ -236,13 +224,5 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG
|
||||||
return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error];
|
return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error];
|
||||||
}
|
}
|
||||||
|
|
||||||
- (BOOL)detectAsyncInImage:(MPPImage *)image
|
|
||||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
|
||||||
error:(NSError **)error {
|
|
||||||
return [self detectAsyncInImage:image
|
|
||||||
timestampInMilliseconds:timestampInMilliseconds
|
|
||||||
regionOfInterest:CGRectZero
|
|
||||||
error:error];
|
|
||||||
}
|
|
||||||
|
|
||||||
@end
|
@end
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user