diff --git a/mediapipe/tasks/ios/test/vision/image_segmenter/BUILD b/mediapipe/tasks/ios/test/vision/image_segmenter/BUILD new file mode 100644 index 000000000..03c515529 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/image_segmenter/BUILD @@ -0,0 +1,70 @@ +load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "//mediapipe/framework/tool:ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPImageSegmenterObjcTestLibrary", + testonly = 1, + srcs = ["MPPImageSegmenterTests.mm"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + "//mediapipe/tasks/testdata/vision:test_protos", + ], + deps = [ + # "//mediapipe/framework/formats:classification_cc_proto", + # "//mediapipe/framework/formats:landmark_cc_proto", + # "//mediapipe/framework/formats:matrix_data_cc_proto", + # "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto", + # "//mediapipe/tasks/ios/common:MPPCommon", + # "//mediapipe/tasks/ios/components/containers/utils:MPPClassificationResultHelpers", + # "//mediapipe/tasks/ios/components/containers/utils:MPPDetectionHelpers", + # "//mediapipe/tasks/ios/components/containers/utils:MPPLandmarkHelpers", + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + "//mediapipe/tasks/ios/vision/image_segmenter:MPPImageSegmenter", + "//mediapipe/tasks/ios/vision/image_segmenter:MPPImageSegmenterResult", + "//third_party/apple_frameworks:UIKit", + ] + select({ + "//third_party:opencv_ios_sim_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_x86_64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//conditions:default": ["@ios_opencv//:OpencvFramework"], + }), +) + +ios_unit_test( + name = "MPPImageSegmenterObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPImageSegmenterObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm new file mode 100644 index 000000000..1a1dcfc8a --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm @@ -0,0 +1,575 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import +#import + +// #include "mediapipe/framework/formats/classification.pb.h" +// #include "mediapipe/framework/formats/landmark.pb.h" +// #include "mediapipe/framework/formats/matrix_data.pb.h" +// #include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h" +// #import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +// #import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" +// #import "mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.h" +// #import "mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +// #include "mediapipe/tasks/ios/test/vision/utils/sources/parse_proto_utils.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h" + +// using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList; +// using ClassificationListProto = ::mediapipe::ClassificationList; +// using FaceGeometryProto = ::mediapipe::tasks::vision::face_geometry::proto::FaceGeometry; +// using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt; + +static NSString *const kPbFileExtension = @"pbtxt"; + +typedef NSDictionary ResourceFileInfo; + +// static ResourceFileInfo *const kPortraitImage = +// @{@"name" : @"portrait", @"type" : @"jpg", @"orientation" : @(UIImageOrientationUp)}; +// static ResourceFileInfo *const kPortraitRotatedImage = +// @{@"name" : @"portrait_rotated", @"type" : @"jpg", @"orientation" : @(UIImageOrientationRight)}; +static ResourceFileInfo *const kCatImage = @{@"name" : @"cat", @"type" : @"jpg"}; +// static ResourceFileInfo *const kPortraitExpectedLandmarksName = +// @{@"name" : @"portrait_expected_face_landmarks", @"type" : kPbFileExtension}; +// static ResourceFileInfo *const kPortraitExpectedBlendshapesName = +// @{@"name" : @"portrait_expected_blendshapes", @"type" : kPbFileExtension}; +// static ResourceFileInfo *const kPortraitExpectedGeometryName = +// @{@"name" : @"portrait_expected_face_geometry", @"type" : kPbFileExtension}; +static NSString *const kImageSegmenterModelName = @"deeplabv3"; +// static NSString *const kFaceLandmarkerWithBlendshapesModelName = +// @"face_landmarker_v2_with_blendshapes"; +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; +static NSString *const kLiveStreamTestsDictFaceLandmarkerKey = @"image_segmenter"; +static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; + +constexpr float kLandmarkErrorThreshold = 0.03f; +constexpr float kBlendshapesErrorThreshold = 0.1f; +constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertEqualObjects(error.localizedDescription, expectedError.localizedDescription) + +@interface MPPImageSegmenterTests : XCTestCase { + NSDictionary *_liveStreamSucceedsTestDict; + NSDictionary *_outOfOrderTimestampTestDict; +} +@end + +@implementation MPPImageSegmenterTests + +#pragma mark General Tests + +- (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo { + UIImageOrientation orientation = (UIImageOrientation)[fileInfo[@"orientation"] intValue]; + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPImageSegmenterTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"] + orientation:orientation]; + XCTAssertNotNil(image); + return image; +} + ++ (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension { + NSString *filePath = + [[NSBundle bundleForClass:[MPPImageSegmenterTests class]] pathForResource:fileName + ofType:extension]; + return filePath; +} + +// - (void)testCreateImageSegmenterWithMissingModelPathFails { +// NSString *modelPath = [MPPFaceLandmarkerTests filePathWithName:@"" extension:@""]; + +// NSError *error = nil; +// MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithModelPath:modelPath +// error:&error]; +// XCTAssertNil(faceLandmarker); + +// NSError *expectedError = [NSError +// errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : +// @"INVALID_ARGUMENT: ExternalFile must specify at least one of 'file_content', " +// @"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'." +// }]; +// AssertEqualErrors(error, expectedError); +// } + +#pragma mark Image Mode Tests + +- (void)testDetectWithImageModeAndPotraitSucceeds { + NSString *modelPath = [MPPImageSegmenterTests filePathWithName:kImageSegmenterModelName + extension:@"tflite"]; + MPPImageSegmenter *imageSegmenter = [[MPPImageSegmenter alloc] initWithModelPath:modelPath + error:nil]; + + MPPImage *image = [self imageWithFileInfo:kCatImage]; + MPPImageSegmenterResult *result = [imageSegmenter segmentImage:image error:nil]; + // NSArray *expectedLandmarks = + // [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; + // [self assertResultsOfDetectInImageWithFileInfo:kPortraitImage + // usingFaceLandmarker:faceLandmarker + // containsExpectedLandmarks:expectedLandmarks + // expectedBlendshapes:NULL + // expectedTransformationMatrix:NULL]; +} + +@end + +// - (void)testDetectWithImageModeAndPotraitAndFacialTransformationMatrixesSucceeds { +// MPPFaceLandmarkerOptions *options = +// [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; +// options.outputFacialTransformationMatrixes = YES; +// MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + +// NSArray *expectedLandmarks = +// [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; +// MPPTransformMatrix *expectedTransformationMatrix = [MPPFaceLandmarkerTests +// expectedTransformationMatrixFromFileInfo:kPortraitExpectedGeometryName]; +// [self assertResultsOfDetectInImageWithFileInfo:kPortraitImage +// usingFaceLandmarker:faceLandmarker +// containsExpectedLandmarks:expectedLandmarks +// expectedBlendshapes:NULL +// expectedTransformationMatrix:expectedTransformationMatrix]; +// } + +// - (void)testDetectWithImageModeAndNoFaceSucceeds { +// NSString *modelPath = [MPPFaceLandmarkerTests filePathWithName:kFaceLandmarkerModelName +// extension:@"task"]; +// MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithModelPath:modelPath +// error:nil]; +// XCTAssertNotNil(faceLandmarker); + +// NSError *error; +// MPPImage *mppImage = [self imageWithFileInfo:kCatImage]; +// MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInImage:mppImage +// error:&error]; +// XCTAssertNil(error); +// XCTAssertNotNil(faceLandmarkerResult); +// XCTAssertEqualObjects(faceLandmarkerResult.faceLandmarks, [NSArray array]); +// XCTAssertEqualObjects(faceLandmarkerResult.faceBlendshapes, [NSArray array]); +// XCTAssertEqualObjects(faceLandmarkerResult.facialTransformationMatrixes, [NSArray array]); +// } + +// #pragma mark Video Mode Tests + +// - (void)testDetectWithVideoModeAndPotraitSucceeds { +// MPPFaceLandmarkerOptions *options = +// [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; +// options.runningMode = MPPRunningModeVideo; +// MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + +// MPPImage *image = [self imageWithFileInfo:kPortraitImage]; +// NSArray *expectedLandmarks = +// [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; +// for (int i = 0; i < 3; i++) { +// MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInVideoFrame:image +// timestampInMilliseconds:i +// error:nil]; +// [self assertFaceLandmarkerResult:faceLandmarkerResult +// containsExpectedLandmarks:expectedLandmarks +// expectedBlendshapes:NULL +// expectedTransformationMatrix:NULL]; +// } +// } + +// #pragma mark Live Stream Mode Tests + +// - (void)testDetectWithLiveStreamModeAndPotraitSucceeds { +// NSInteger iterationCount = 100; + +// // Because of flow limiting, the callback might be invoked fewer than `iterationCount` times. An +// // normal expectation will fail if expectation.fulfill() is not called +// // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will +// // only succeed if expectation is not fulfilled for the specified `expectedFulfillmentCount`. +// // Since it is not possible to predict how many times the expectation is supposed to be +// // fulfilled, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and +// // `expectation.isInverted = true` ensures that test succeeds if expectation is fulfilled <= +// // `iterationCount` times. +// XCTestExpectation *expectation = [[XCTestExpectation alloc] +// initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; +// expectation.expectedFulfillmentCount = iterationCount + 1; +// expectation.inverted = YES; + +// MPPFaceLandmarkerOptions *options = +// [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; +// options.runningMode = MPPRunningModeLiveStream; +// options.faceLandmarkerLiveStreamDelegate = self; + +// MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; +// MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + +// _liveStreamSucceedsTestDict = @{ +// kLiveStreamTestsDictFaceLandmarkerKey : faceLandmarker, +// kLiveStreamTestsDictExpectationKey : expectation +// }; + +// for (int i = 0; i < iterationCount; i++) { +// XCTAssertTrue([faceLandmarker detectAsyncInImage:image timestampInMilliseconds:i error:nil]); +// } + +// NSTimeInterval timeout = 0.5f; +// [self waitForExpectations:@[ expectation ] timeout:timeout]; +// } + +// - (void)testDetectWithOutOfOrderTimestampsAndLiveStreamModeFails { +// MPPFaceLandmarkerOptions *options = +// [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; +// options.runningMode = MPPRunningModeLiveStream; +// options.faceLandmarkerLiveStreamDelegate = self; + +// XCTestExpectation *expectation = [[XCTestExpectation alloc] +// initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; +// expectation.expectedFulfillmentCount = 1; + +// MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; +// _liveStreamSucceedsTestDict = @{ +// kLiveStreamTestsDictFaceLandmarkerKey : faceLandmarker, +// kLiveStreamTestsDictExpectationKey : expectation +// }; + +// MPPImage *image = [self imageWithFileInfo:kPortraitImage]; +// XCTAssertTrue([faceLandmarker detectAsyncInImage:image timestampInMilliseconds:1 error:nil]); + +// NSError *error; +// XCTAssertFalse([faceLandmarker detectAsyncInImage:image timestampInMilliseconds:0 error:&error]); + +// NSError *expectedError = +// [NSError errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : +// @"INVALID_ARGUMENT: Input timestamp must be monotonically increasing." +// }]; +// AssertEqualErrors(error, expectedError); + +// NSTimeInterval timeout = 0.5f; +// [self waitForExpectations:@[ expectation ] timeout:timeout]; +// } + +// #pragma mark Running Mode Tests + +// - (void)testCreateFaceLandmarkerFailsWithDelegateInNonLiveStreamMode { +// MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo}; +// for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) { +// MPPFaceLandmarkerOptions *options = +// [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; + +// options.runningMode = runningModesToTest[i]; +// options.faceLandmarkerLiveStreamDelegate = self; + +// [self +// assertCreateFaceLandmarkerWithOptions:options +// failsWithExpectedError: +// [NSError errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : +// @"The vision task is in image or video mode. The " +// @"delegate must not be set in the task's options." +// }]]; +// } +// } + +// - (void)testCreateFaceLandmarkerFailsWithMissingDelegateInLiveStreamMode { +// MPPFaceLandmarkerOptions *options = +// [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; +// options.runningMode = MPPRunningModeLiveStream; + +// [self assertCreateFaceLandmarkerWithOptions:options +// failsWithExpectedError: +// [NSError errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : +// @"The vision task is in live stream mode. An " +// @"object must be set as the delegate of the task " +// @"in its options to ensure asynchronous delivery " +// @"of results." +// }]]; +// } + +// - (void)testDetectFailsWithCallingWrongAPIInImageMode { +// MPPFaceLandmarkerOptions *options = +// [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; +// MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + +// MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + +// NSError *liveStreamAPICallError; +// XCTAssertFalse([faceLandmarker detectAsyncInImage:image +// timestampInMilliseconds:0 +// error:&liveStreamAPICallError]); + +// NSError *expectedLiveStreamAPICallError = +// [NSError errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : @"The vision task is not initialized with live " +// @"stream mode. Current Running Mode: Image" +// }]; +// AssertEqualErrors(liveStreamAPICallError, expectedLiveStreamAPICallError); + +// NSError *videoAPICallError; +// XCTAssertFalse([faceLandmarker detectInVideoFrame:image +// timestampInMilliseconds:0 +// error:&videoAPICallError]); + +// NSError *expectedVideoAPICallError = +// [NSError errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : @"The vision task is not initialized with " +// @"video mode. Current Running Mode: Image" +// }]; +// AssertEqualErrors(videoAPICallError, expectedVideoAPICallError); +// } + +// - (void)testDetectFailsWithCallingWrongAPIInVideoMode { +// MPPFaceLandmarkerOptions *options = +// [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; +// options.runningMode = MPPRunningModeVideo; + +// MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + +// MPPImage *image = [self imageWithFileInfo:kPortraitImage]; +// NSError *liveStreamAPICallError; +// XCTAssertFalse([faceLandmarker detectAsyncInImage:image +// timestampInMilliseconds:0 +// error:&liveStreamAPICallError]); + +// NSError *expectedLiveStreamAPICallError = +// [NSError errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : @"The vision task is not initialized with live " +// @"stream mode. Current Running Mode: Video" +// }]; +// AssertEqualErrors(liveStreamAPICallError, expectedLiveStreamAPICallError); + +// NSError *imageAPICallError; +// XCTAssertFalse([faceLandmarker detectInImage:image error:&imageAPICallError]); + +// NSError *expectedImageAPICallError = +// [NSError errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : @"The vision task is not initialized with " +// @"image mode. Current Running Mode: Video" +// }]; +// AssertEqualErrors(imageAPICallError, expectedImageAPICallError); +// } + +// - (void)testDetectFailsWithCallingWrongAPIInLiveStreamMode { +// MPPFaceLandmarkerOptions *options = +// [self faceLandmarkerOptionsWithModelName:kFaceLandmarkerModelName]; +// options.runningMode = MPPRunningModeLiveStream; +// options.faceLandmarkerLiveStreamDelegate = self; +// MPPFaceLandmarker *faceLandmarker = [[MPPFaceLandmarker alloc] initWithOptions:options error:nil]; + +// MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + +// NSError *imageAPICallError; +// XCTAssertFalse([faceLandmarker detectInImage:image error:&imageAPICallError]); + +// NSError *expectedImageAPICallError = +// [NSError errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : @"The vision task is not initialized with " +// @"image mode. Current Running Mode: Live Stream" +// }]; +// AssertEqualErrors(imageAPICallError, expectedImageAPICallError); + +// NSError *videoAPICallError; +// XCTAssertFalse([faceLandmarker detectInVideoFrame:image +// timestampInMilliseconds:0 +// error:&videoAPICallError]); + +// NSError *expectedVideoAPICallError = +// [NSError errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : @"The vision task is not initialized with " +// @"video mode. Current Running Mode: Live Stream" +// }]; +// AssertEqualErrors(videoAPICallError, expectedVideoAPICallError); +// } + +// #pragma mark MPPFaceLandmarkerLiveStreamDelegate Methods +// - (void)faceLandmarker:(MPPFaceLandmarker *)faceLandmarker +// didFinishDetectionWithResult:(MPPFaceLandmarkerResult *)faceLandmarkerResult +// timestampInMilliseconds:(NSInteger)timestampInMilliseconds +// error:(NSError *)error { +// NSArray *expectedLandmarks = +// [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; +// [self assertFaceLandmarkerResult:faceLandmarkerResult +// containsExpectedLandmarks:expectedLandmarks +// expectedBlendshapes:NULL +// expectedTransformationMatrix:NULL]; + +// if (faceLandmarker == _outOfOrderTimestampTestDict[kLiveStreamTestsDictFaceLandmarkerKey]) { +// [_outOfOrderTimestampTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; +// } else if (faceLandmarker == _liveStreamSucceedsTestDict[kLiveStreamTestsDictFaceLandmarkerKey]) { +// [_liveStreamSucceedsTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; +// } +// } + +// + (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension { +// NSString *filePath = +// [[NSBundle bundleForClass:[MPPFaceLandmarkerTests class]] pathForResource:fileName +// ofType:extension]; +// return filePath; +// } + +// + (NSArray *)expectedLandmarksFromFileInfo:(NSDictionary *)fileInfo { +// NSString *filePath = [self filePathWithName:fileInfo[@"name"] extension:fileInfo[@"type"]]; +// NormalizedLandmarkListProto proto; +// if (!get_proto_from_pbtxt([filePath UTF8String], proto).ok()) { +// return nil; +// } +// NSMutableArray *landmarks = +// [NSMutableArray arrayWithCapacity:(NSUInteger)proto.landmark_size()]; +// for (const auto &landmarkProto : proto.landmark()) { +// [landmarks addObject:[MPPNormalizedLandmark normalizedLandmarkWithProto:landmarkProto]]; +// } +// return landmarks; +// } + +// + (MPPClassifications *)expectedBlendshapesFromFileInfo:(NSDictionary *)fileInfo { +// NSString *filePath = [self filePathWithName:fileInfo[@"name"] extension:fileInfo[@"type"]]; +// ClassificationListProto proto; +// if (!get_proto_from_pbtxt([filePath UTF8String], proto).ok()) { +// return nil; +// } +// return [MPPClassifications classificationsWithClassificationListProto:proto +// headIndex:0 +// headName:[NSString string]]; +// } + +// + (MPPTransformMatrix *)expectedTransformationMatrixFromFileInfo:(NSDictionary *)fileInfo { +// NSString *filePath = [self filePathWithName:fileInfo[@"name"] extension:fileInfo[@"type"]]; +// FaceGeometryProto proto; +// if (!get_proto_from_pbtxt([filePath UTF8String], proto).ok()) { +// return nil; +// } +// return [[MPPTransformMatrix alloc] initWithData:proto.pose_transform_matrix().packed_data().data() +// rows:proto.pose_transform_matrix().rows() +// columns:proto.pose_transform_matrix().cols()]; +// } + +// - (void)assertFaceLandmarkerResult:(MPPFaceLandmarkerResult *)faceLandmarkerResult +// containsExpectedLandmarks:(NSArray *)expectedLandmarks +// expectedBlendshapes:(nullable MPPClassifications *)expectedBlendshapes +// expectedTransformationMatrix:(nullable MPPTransformMatrix *)expectedTransformationMatrix { +// NSArray *landmarks = faceLandmarkerResult.faceLandmarks[0]; +// XCTAssertEqual(landmarks.count, expectedLandmarks.count); +// for (int i = 0; i < landmarks.count; ++i) { +// XCTAssertEqualWithAccuracy(landmarks[i].x, expectedLandmarks[i].x, kLandmarkErrorThreshold, +// @"index i = %d", i); +// XCTAssertEqualWithAccuracy(landmarks[i].y, expectedLandmarks[i].y, kLandmarkErrorThreshold, +// @"index i = %d", i); +// } + +// if (expectedBlendshapes == NULL) { +// XCTAssertEqualObjects(faceLandmarkerResult.faceBlendshapes, [NSArray array]); +// } else { +// MPPClassifications *blendshapes = faceLandmarkerResult.faceBlendshapes[0]; +// NSArray *actualCategories = blendshapes.categories; +// NSArray *expectedCategories = expectedBlendshapes.categories; +// XCTAssertEqual(actualCategories.count, expectedCategories.count); +// for (int i = 0; i < actualCategories.count; ++i) { +// XCTAssertEqual(actualCategories[i].index, expectedCategories[i].index, @"index i = %d", i); +// XCTAssertEqualWithAccuracy(actualCategories[i].score, expectedCategories[i].score, +// kBlendshapesErrorThreshold, @"index i = %d", i); +// XCTAssertEqualObjects(actualCategories[i].categoryName, expectedCategories[i].categoryName, +// @"index i = %d", i); +// XCTAssertEqualObjects(actualCategories[i].displayName, expectedCategories[i].displayName, +// @"index i = %d", i); +// } +// } + +// if (expectedTransformationMatrix == NULL) { +// XCTAssertEqualObjects(faceLandmarkerResult.facialTransformationMatrixes, [NSArray array]); +// } else { +// MPPTransformMatrix *actualTransformationMatrix = +// faceLandmarkerResult.facialTransformationMatrixes[0]; +// XCTAssertEqual(actualTransformationMatrix.rows, expectedTransformationMatrix.rows); +// XCTAssertEqual(actualTransformationMatrix.columns, expectedTransformationMatrix.columns); +// for (int i = 0; i < actualTransformationMatrix.rows * actualTransformationMatrix.columns; ++i) { +// XCTAssertEqualWithAccuracy(actualTransformationMatrix.data[i], +// expectedTransformationMatrix.data[i], +// kFacialTransformationMatrixErrorThreshold, @"index i = %d", i); +// } +// } +// } + +// #pragma mark Face Landmarker Initializers + +// - (MPPFaceLandmarkerOptions *)faceLandmarkerOptionsWithModelName:(NSString *)modelName { +// NSString *modelPath = [MPPFaceLandmarkerTests filePathWithName:modelName extension:@"task"]; +// MPPFaceLandmarkerOptions *faceLandmarkerOptions = [[MPPFaceLandmarkerOptions alloc] init]; +// faceLandmarkerOptions.baseOptions.modelAssetPath = modelPath; +// return faceLandmarkerOptions; +// } + +// - (void)assertCreateFaceLandmarkerWithOptions:(MPPFaceLandmarkerOptions *)faceLandmarkerOptions +// failsWithExpectedError:(NSError *)expectedError { +// NSError *error = nil; +// MPPFaceLandmarker *faceLandmarker = +// [[MPPFaceLandmarker alloc] initWithOptions:faceLandmarkerOptions error:&error]; +// XCTAssertNil(faceLandmarker); +// AssertEqualErrors(error, expectedError); +// } + +// #pragma mark Assert Detection Results + +// - (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo { +// UIImageOrientation orientation = (UIImageOrientation)[fileInfo[@"orientation"] intValue]; +// MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPFaceLandmarkerTests class] +// fileName:fileInfo[@"name"] +// ofType:fileInfo[@"type"] +// orientation:orientation]; +// XCTAssertNotNil(image); +// return image; +// } + +// - (void)assertResultsOfDetectInImageWithFileInfo:(ResourceFileInfo *)fileInfo +// usingFaceLandmarker:(MPPFaceLandmarker *)faceLandmarker +// containsExpectedLandmarks: +// (NSArray *)expectedLandmarks +// expectedBlendshapes:(nullable MPPClassifications *)expectedBlendshapes +// expectedTransformationMatrix: +// (nullable MPPTransformMatrix *)expectedTransformationMatrix { +// MPPImage *mppImage = [self imageWithFileInfo:fileInfo]; + +// NSError *error; +// MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInImage:mppImage +// error:&error]; +// XCTAssertNil(error); +// XCTAssertNotNil(faceLandmarkerResult); + +// [self assertFaceLandmarkerResult:faceLandmarkerResult +// containsExpectedLandmarks:expectedLandmarks +// expectedBlendshapes:expectedBlendshapes +// expectedTransformationMatrix:expectedTransformationMatrix]; +// } + +// @end diff --git a/mediapipe/tasks/ios/test/vision/image_segmenter/utils/BUILD b/mediapipe/tasks/ios/test/vision/image_segmenter/utils/BUILD new file mode 100644 index 000000000..74f2bd11a --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/image_segmenter/utils/BUILD @@ -0,0 +1,55 @@ +load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "//mediapipe/framework/tool:ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPFaceLandmarkeResultHelpersTestLibary", + testonly = 1, + srcs = ["sources/MPPFaceLandmarkerResult+HelpersTests.mm"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:matrix_data_cc_proto", + "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto", + "//mediapipe/tasks/ios/vision/face_landmarker:MPPFaceLandmarkerResult", + "//mediapipe/tasks/ios/vision/face_landmarker/utils:MPPFaceLandmarkerResultHelpers", + ], +) + +ios_unit_test( + name = "MPPFaceLandmarkeResultHelpersTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPFaceLandmarkeResultHelpersTestLibary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/image_segmenter/utils/sources/MPPFaceLandmarkerResult+HelpersTests.mm b/mediapipe/tasks/ios/test/vision/image_segmenter/utils/sources/MPPFaceLandmarkerResult+HelpersTests.mm new file mode 100644 index 000000000..3572aa47e --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/image_segmenter/utils/sources/MPPFaceLandmarkerResult+HelpersTests.mm @@ -0,0 +1,112 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/matrix_data.pb.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarkerResult.h" +#import "mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerResult+Helpers.h" + +using ::mediapipe::MakePacket; +using ::mediapipe::Packet; +using ::mediapipe::Timestamp; +using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList; +using ClassificationListProto = ::mediapipe::ClassificationList; +using FaceGeometryProto = ::mediapipe::tasks::vision::face_geometry::proto::FaceGeometry; + +static constexpr int kMicrosecondsPerMillisecond = 1000; + +@interface MPPLandmarkerResultHelpersTests : XCTestCase { +} +@end + +@implementation MPPLandmarkerResultHelpersTests + +- (void)testCreatesResultFromLandmarkerPackets { + const std::vector normalizedLandmarkProtos({{}}); + const std::vector classificationProtos({{}}); + const std::vector faceGeometryProto({{}}); + + const auto landmarksPacket = + MakePacket>(normalizedLandmarkProtos) + .At(Timestamp(42 * kMicrosecondsPerMillisecond)); + const auto classificationsPacket = + MakePacket>(classificationProtos) + .At(Timestamp(42 * kMicrosecondsPerMillisecond)); + const auto faceGeometryPacket = MakePacket>(faceGeometryProto) + .At(Timestamp(42 * kMicrosecondsPerMillisecond)); + + MPPFaceLandmarkerResult *results = + [MPPFaceLandmarkerResult faceLandmarkerResultWithLandmarksPacket:landmarksPacket + blendshapesPacket:classificationsPacket + transformationMatrixesPacket:faceGeometryPacket]; + + XCTAssertEqual(results.faceLandmarks.count, 1); + XCTAssertEqual(results.faceBlendshapes.count, 1); + XCTAssertEqual(results.facialTransformationMatrixes.count, 1); + XCTAssertEqual(results.timestampInMilliseconds, 42); +} + +- (void)testCreatesCreatesCopyOfFacialTransformationMatrix { + MPPFaceLandmarkerResult *results; + + { + // Create scope so that FaceGeometryProto gets deallocated before we access the + // MPPFaceLandmarkerResult. + FaceGeometryProto faceGeometryProto{}; + auto *matrixData = faceGeometryProto.mutable_pose_transform_matrix(); + matrixData->set_cols(4); + matrixData->set_rows(4); + for (size_t i = 0; i < 4 * 4; ++i) { + matrixData->add_packed_data(0.1f * i); + } + + const std::vector faceGeometryProtos({faceGeometryProto}); + const auto faceGeometryPacket = MakePacket>(faceGeometryProtos); + results = [MPPFaceLandmarkerResult faceLandmarkerResultWithLandmarksPacket:{} + blendshapesPacket:{} + transformationMatrixesPacket:faceGeometryPacket]; + } + + XCTAssertEqual(results.facialTransformationMatrixes.count, 1); + XCTAssertEqual(results.facialTransformationMatrixes[0].rows, 4); + XCTAssertEqual(results.facialTransformationMatrixes[0].columns, 4); + for (size_t column = 0; column < 4; ++column) { + for (size_t row = 0; row < 4; ++row) { + XCTAssertEqualWithAccuracy( + [results.facialTransformationMatrixes[0] valueAtRow:row column:column], + 0.4f * row + 0.1f * column, /* accuracy= */ 0.0001f, @"at [%zu,%zu]", column, row); + } + } +} + +- (void)testCreatesResultFromEmptyPackets { + const Packet emptyPacket = Packet{}.At(Timestamp(0)); + MPPFaceLandmarkerResult *results = + [MPPFaceLandmarkerResult faceLandmarkerResultWithLandmarksPacket:emptyPacket + blendshapesPacket:emptyPacket + transformationMatrixesPacket:emptyPacket]; + + NSArray *emptyArray = [NSArray array]; + XCTAssertEqualObjects(results.faceLandmarks, emptyArray); + XCTAssertEqualObjects(results.faceBlendshapes, emptyArray); + XCTAssertEqualObjects(results.facialTransformationMatrixes, emptyArray); + XCTAssertEqual(results.timestampInMilliseconds, 0); +} + +@end