Added live stream mode tests for iOS Hand Landmarker
This commit is contained in:
parent
baa79046b9
commit
43e51c1094
|
@ -58,7 +58,10 @@ static const float kLandmarksErrorTolerance = 0.03f;
|
||||||
XCTAssertTrue(handLandmarkerResult.landmarks.count == 0); \
|
XCTAssertTrue(handLandmarkerResult.landmarks.count == 0); \
|
||||||
XCTAssertTrue(handLandmarkerResult.worldLandmarks.count == 0);
|
XCTAssertTrue(handLandmarkerResult.worldLandmarks.count == 0);
|
||||||
|
|
||||||
@interface MPPHandLandmarkerTests : XCTestCase
|
@interface MPPHandLandmarkerTests : XCTestCase {
|
||||||
|
NSDictionary<NSString *, id> *_liveStreamSucceedsTestDict;
|
||||||
|
NSDictionary<NSString *, id> *_outOfOrderTimestampTestDict;
|
||||||
|
}
|
||||||
@end
|
@end
|
||||||
|
|
||||||
@implementation MPPHandLandmarkerTests
|
@implementation MPPHandLandmarkerTests
|
||||||
|
@ -153,7 +156,7 @@ static const float kLandmarksErrorTolerance = 0.03f;
|
||||||
return filePath;
|
return filePath;
|
||||||
}
|
}
|
||||||
|
|
||||||
#pragma mark Gesture Recognizer Initializers
|
#pragma mark Hand Landmarker Initializers
|
||||||
|
|
||||||
- (MPPHandLandmarkerOptions *)handLandmarkerOptionsWithModelFileInfo:
|
- (MPPHandLandmarkerOptions *)handLandmarkerOptionsWithModelFileInfo:
|
||||||
(ResourceFileInfo *)modelFileInfo {
|
(ResourceFileInfo *)modelFileInfo {
|
||||||
|
@ -185,7 +188,7 @@ static const float kLandmarksErrorTolerance = 0.03f;
|
||||||
AssertEqualErrors(error, expectedError);
|
AssertEqualErrors(error, expectedError);
|
||||||
}
|
}
|
||||||
|
|
||||||
#pragma mark Assert Gesture Recognizer Results
|
#pragma mark Assert Hand Landmarker Results
|
||||||
|
|
||||||
- (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo {
|
- (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo {
|
||||||
MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPHandLandmarkerTests class]
|
MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPHandLandmarkerTests class]
|
||||||
|
@ -289,4 +292,285 @@ static const float kLandmarksErrorTolerance = 0.03f;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#pragma mark Running Mode Tests
|
||||||
|
|
||||||
|
- (void)testCreateHandLandmarkerFailsWithDelegateInNonLiveStreamMode {
|
||||||
|
MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo};
|
||||||
|
for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) {
|
||||||
|
MPPHandLandmarkerOptions *options =
|
||||||
|
[self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
|
||||||
|
|
||||||
|
options.runningMode = runningModesToTest[i];
|
||||||
|
options.handLandmarkerLiveStreamDelegate = self;
|
||||||
|
|
||||||
|
[self assertCreateHandLandmarkerWithOptions:options
|
||||||
|
failsWithExpectedError:
|
||||||
|
[NSError
|
||||||
|
errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey :
|
||||||
|
@"The vision task is in image or video mode. The "
|
||||||
|
@"delegate must not be set in the task's options."
|
||||||
|
}]];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testCreateHandLandmarkerFailsWithMissingDelegateInLiveStreamMode {
|
||||||
|
MPPHandLandmarkerOptions *options =
|
||||||
|
[self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
|
||||||
|
|
||||||
|
options.runningMode = MPPRunningModeLiveStream;
|
||||||
|
|
||||||
|
[self
|
||||||
|
assertCreateHandLandmarkerWithOptions:options
|
||||||
|
failsWithExpectedError:
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey :
|
||||||
|
@"The vision task is in live stream mode. An "
|
||||||
|
@"object must be set as the delegate of the task "
|
||||||
|
@"in its options to ensure asynchronous delivery "
|
||||||
|
@"of results."
|
||||||
|
}]];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectFailsWithCallingWrongApiInImageMode {
|
||||||
|
MPPHandLandmarkerOptions *options =
|
||||||
|
[self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
|
||||||
|
|
||||||
|
MPPHandLandmarker *handLandmarker =
|
||||||
|
[self createHandLandmarkerWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kFistImage];
|
||||||
|
|
||||||
|
NSError *liveStreamApiCallError;
|
||||||
|
XCTAssertFalse([handLandmarker detectAsyncInImage:image
|
||||||
|
timestampInMilliseconds:0
|
||||||
|
error:&liveStreamApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedLiveStreamApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with live "
|
||||||
|
@"stream mode. Current Running Mode: Image"
|
||||||
|
}];
|
||||||
|
|
||||||
|
AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError);
|
||||||
|
|
||||||
|
NSError *videoApiCallError;
|
||||||
|
XCTAssertFalse([handLandmarker detectInVideoFrame:image
|
||||||
|
timestampInMilliseconds:0
|
||||||
|
error:&videoApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedVideoApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with "
|
||||||
|
@"video mode. Current Running Mode: Image"
|
||||||
|
}];
|
||||||
|
AssertEqualErrors(videoApiCallError, expectedVideoApiCallError);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectFailsWithCallingWrongApiInVideoMode {
|
||||||
|
MPPHandLandmarkerOptions *options =
|
||||||
|
[self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
|
||||||
|
options.runningMode = MPPRunningModeVideo;
|
||||||
|
|
||||||
|
MPPHandLandmarker *handLandmarker =
|
||||||
|
[self createHandLandmarkerWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kFistImage];
|
||||||
|
|
||||||
|
NSError *liveStreamApiCallError;
|
||||||
|
XCTAssertFalse([handLandmarker detectAsyncInImage:image
|
||||||
|
timestampInMilliseconds:0
|
||||||
|
error:&liveStreamApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedLiveStreamApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with live "
|
||||||
|
@"stream mode. Current Running Mode: Video"
|
||||||
|
}];
|
||||||
|
|
||||||
|
AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError);
|
||||||
|
|
||||||
|
NSError *imageApiCallError;
|
||||||
|
XCTAssertFalse([handLandmarker detectInImage:image error:&imageApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedImageApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with "
|
||||||
|
@"image mode. Current Running Mode: Video"
|
||||||
|
}];
|
||||||
|
AssertEqualErrors(imageApiCallError, expectedImageApiCallError);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectFailsWithCallingWrongApiInLiveStreamMode {
|
||||||
|
MPPHandLandmarkerOptions *options =
|
||||||
|
[self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
|
||||||
|
options.runningMode = MPPRunningModeLiveStream;
|
||||||
|
options.handLandmarkerLiveStreamDelegate = self;
|
||||||
|
|
||||||
|
MPPHandLandmarker *handLandmarker =
|
||||||
|
[self createHandLandmarkerWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kFistImage];
|
||||||
|
|
||||||
|
NSError *imageApiCallError;
|
||||||
|
XCTAssertFalse([handLandmarker detectInImage:image error:&imageApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedImageApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with "
|
||||||
|
@"image mode. Current Running Mode: Live Stream"
|
||||||
|
}];
|
||||||
|
AssertEqualErrors(imageApiCallError, expectedImageApiCallError);
|
||||||
|
|
||||||
|
NSError *videoApiCallError;
|
||||||
|
XCTAssertFalse([handLandmarker detectInVideoFrame:image
|
||||||
|
timestampInMilliseconds:0
|
||||||
|
error:&videoApiCallError]);
|
||||||
|
|
||||||
|
NSError *expectedVideoApiCallError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey : @"The vision task is not initialized with "
|
||||||
|
@"video mode. Current Running Mode: Live Stream"
|
||||||
|
}];
|
||||||
|
AssertEqualErrors(videoApiCallError, expectedVideoApiCallError);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithVideoModeSucceeds {
|
||||||
|
MPPHandLandmarkerOptions *options =
|
||||||
|
[self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
|
||||||
|
options.runningMode = MPPRunningModeVideo;
|
||||||
|
|
||||||
|
MPPHandLandmarker *handLandmarker =
|
||||||
|
[self createHandLandmarkerWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kThumbUpImage];
|
||||||
|
|
||||||
|
for (int i = 0; i < 3; i++) {
|
||||||
|
MPPHandLandmarkerResult *handLandmarkerResult =
|
||||||
|
[handLandmarker detectInVideoFrame:image timestampInMilliseconds:i error:nil];
|
||||||
|
[self assertHandLandmarkerResult:handLandmarkerResult
|
||||||
|
isApproximatelyEqualToExpectedResult:[MPPHandLandmarkerTests
|
||||||
|
thumbUpHandLandmarkerResult]];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithOutOfOrderTimestampsAndLiveStreamModeFails {
|
||||||
|
MPPHandLandmarkerOptions *options =
|
||||||
|
[self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
|
||||||
|
options.runningMode = MPPRunningModeLiveStream;
|
||||||
|
options.handLandmarkerLiveStreamDelegate = self;
|
||||||
|
|
||||||
|
XCTestExpectation *expectation = [[XCTestExpectation alloc]
|
||||||
|
initWithDescription:@"detectWiththOutOfOrderTimestampsAndLiveStream"];
|
||||||
|
|
||||||
|
expectation.expectedFulfillmentCount = 1;
|
||||||
|
|
||||||
|
MPPHandLandmarker *handLandmarker =
|
||||||
|
[self createHandLandmarkerWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
_outOfOrderTimestampTestDict = @{
|
||||||
|
kLiveStreamTestsDictHandLandmarkerKey : handLandmarker,
|
||||||
|
kLiveStreamTestsDictExpectationKey : expectation
|
||||||
|
};
|
||||||
|
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kThumbUpImage];
|
||||||
|
|
||||||
|
XCTAssertTrue([handLandmarker detectAsyncInImage:image timestampInMilliseconds:1 error:nil]);
|
||||||
|
|
||||||
|
NSError *error;
|
||||||
|
XCTAssertFalse([handLandmarker detectAsyncInImage:image
|
||||||
|
timestampInMilliseconds:0
|
||||||
|
error:&error]);
|
||||||
|
|
||||||
|
NSError *expectedError =
|
||||||
|
[NSError errorWithDomain:kExpectedErrorDomain
|
||||||
|
code:MPPTasksErrorCodeInvalidArgumentError
|
||||||
|
userInfo:@{
|
||||||
|
NSLocalizedDescriptionKey :
|
||||||
|
@"INVALID_ARGUMENT: Input timestamp must be monotonically increasing."
|
||||||
|
}];
|
||||||
|
AssertEqualErrors(error, expectedError);
|
||||||
|
|
||||||
|
NSTimeInterval timeout = 0.5f;
|
||||||
|
[self waitForExpectations:@[ expectation ] timeout:timeout];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)testDetectWithLiveStreamModeSucceeds {
|
||||||
|
MPPHandLandmarkerOptions *options =
|
||||||
|
[self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
|
||||||
|
options.runningMode = MPPRunningModeLiveStream;
|
||||||
|
options.handLandmarkerLiveStreamDelegate = self;
|
||||||
|
|
||||||
|
NSInteger iterationCount = 100;
|
||||||
|
|
||||||
|
// Because of flow limiting, we cannot ensure that the callback will be invoked `iterationCount`
|
||||||
|
// times. An normal expectation will fail if expectation.fulfill() is not called
|
||||||
|
// `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will
|
||||||
|
// only succeed if expectation is not fulfilled for the specified `expectedFulfillmentCount`.
|
||||||
|
// Since in our case we cannot predict how many times the expectation is supposed to be fullfilled
|
||||||
|
// setting, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and
|
||||||
|
// `expectation.isInverted = true` ensures that test succeeds ifexpectation is fullfilled <=
|
||||||
|
// `iterationCount` times.
|
||||||
|
XCTestExpectation *expectation =
|
||||||
|
[[XCTestExpectation alloc] initWithDescription:@"detectWithLiveStream"];
|
||||||
|
|
||||||
|
expectation.expectedFulfillmentCount = iterationCount + 1;
|
||||||
|
expectation.inverted = YES;
|
||||||
|
|
||||||
|
MPPHandLandmarker *handLandmarker =
|
||||||
|
[self createHandLandmarkerWithOptionsSucceeds:options];
|
||||||
|
|
||||||
|
_liveStreamSucceedsTestDict = @{
|
||||||
|
kLiveStreamTestsDictHandLandmarkerKey : handLandmarker,
|
||||||
|
kLiveStreamTestsDictExpectationKey : expectation
|
||||||
|
};
|
||||||
|
|
||||||
|
// TODO: Mimic initialization from CMSampleBuffer as live stream mode is most likely to be used
|
||||||
|
// with the iOS camera. AVCaptureVideoDataOutput sample buffer delegates provide frames of type
|
||||||
|
// `CMSampleBuffer`.
|
||||||
|
MPPImage *image = [self imageWithFileInfo:kThumbUpImage];
|
||||||
|
|
||||||
|
for (int i = 0; i < iterationCount; i++) {
|
||||||
|
XCTAssertTrue([handLandmarker detectAsyncInImage:image
|
||||||
|
timestampInMilliseconds:i
|
||||||
|
error:nil]);
|
||||||
|
}
|
||||||
|
|
||||||
|
NSTimeInterval timeout = 0.5f;
|
||||||
|
[self waitForExpectations:@[ expectation ] timeout:timeout];
|
||||||
|
}
|
||||||
|
|
||||||
|
- (void)handLandmarker:(MPPHandLandmarker *)handLandmarker
|
||||||
|
didFinishRecognitionWithResult:(MPPHandLandmarkerResult *)handLandmarkerResult
|
||||||
|
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
||||||
|
error:(NSError *)error {
|
||||||
|
[self assertHandLandmarkerResult:handLandmarkerResult
|
||||||
|
isApproximatelyEqualToExpectedResult:[MPPHandLandmarkerTests
|
||||||
|
thumbUpHandLandmarkerResult]];
|
||||||
|
|
||||||
|
if (handLandmarker == _outOfOrderTimestampTestDict[kLiveStreamTestsDictHandLandmarkerKey]) {
|
||||||
|
[_outOfOrderTimestampTestDict[kLiveStreamTestsDictExpectationKey] fulfill];
|
||||||
|
} else if (handLandmarker ==
|
||||||
|
_liveStreamSucceedsTestDict[kLiveStreamTestsDictHandLandmarkerKey]) {
|
||||||
|
[_liveStreamSucceedsTestDict[kLiveStreamTestsDictExpectationKey] fulfill];
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
@end
|
@end
|
||||||
|
|
Loading…
Reference in New Issue
Block a user