From 6e80941215f0839f41fdaab0696694d9c2df653f Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 19 Sep 2023 19:59:07 +0530 Subject: [PATCH 1/5] Fixed premature deallocation of C++ masks in iOS Image Segmenter --- .../image_segmenter/sources/MPPImageSegmenter.mm | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm index 410eb4e2e..fa46b3387 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm @@ -243,16 +243,18 @@ using ::mediapipe::tasks::core::PacketsCallback; return; } - PacketMap &outputPacketMap = liveStreamResult.value(); + // Output packet map is moved to a block variable that will not be deallocated for the lifetime of the `dispatch_async` call. Eventhough masks are not copied, we have to ensure that they are not de allocated before the delegate call completes. + __block PacketMap outputPacketMap = std::move(liveStreamResult.value()); if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { return; } - - MPPImageSegmenterResult *result = + + dispatch_async(_callbackQueue, ^{ + + MPPImageSegmenterResult *result = [MPPImageSegmenter imageSegmenterResultWithOutputPacketMap:outputPacketMap shouldCopyMaskPacketData:NO]; - dispatch_async(_callbackQueue, ^{ [self.imageSegmenterLiveStreamDelegate imageSegmenter:self didFinishSegmentationWithResult:result timestampInMilliseconds:result.timestampInMilliseconds From 08a5d55ac1785134d46f7d6a736a46f840ae086b Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 19 Sep 2023 19:59:33 +0530 Subject: [PATCH 2/5] Updated interface of iOS image segmenter --- .../ios/vision/image_segmenter/sources/MPPImageSegmenter.h | 2 +- .../ios/vision/image_segmenter/sources/MPPImageSegmenter.mm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h index f4f2e96cc..f75e7575e 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h @@ -203,7 +203,7 @@ NS_SWIFT_NAME(ImageSegmenter) * * @return `YES` if the image was sent to the task successfully, otherwise `NO`. */ -- (BOOL)segmentAsyncInImage:(MPPImage *)image +- (BOOL)segmentAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error NS_SWIFT_NAME(segmentAsync(image:timestampInMilliseconds:)); diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm index fa46b3387..dfa8845f2 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm @@ -187,7 +187,7 @@ using ::mediapipe::tasks::core::PacketsCallback; shouldCopyMaskPacketData:NO]; completionHandler(result, error); } -- (BOOL)segmentAsyncInImage:(MPPImage *)image +- (BOOL)segmentAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { return [_visionTaskRunner processLiveStreamImage:image From bac60548dc658fbb19e7603b2815de057d770539 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 19 Sep 2023 20:00:41 +0530 Subject: [PATCH 3/5] Added selfie segmentation and running mode tests to image segmenter --- .../ios/test/vision/image_segmenter/BUILD | 1 + .../image_segmenter/MPPImageSegmenterTests.mm | 387 +++++++++++++++++- 2 files changed, 382 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/image_segmenter/BUILD b/mediapipe/tasks/ios/test/vision/image_segmenter/BUILD index 65143e69b..98e5749e9 100644 --- a/mediapipe/tasks/ios/test/vision/image_segmenter/BUILD +++ b/mediapipe/tasks/ios/test/vision/image_segmenter/BUILD @@ -53,6 +53,7 @@ objc_library( "//mediapipe/tasks/testdata/vision:test_protos", ], deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", "//mediapipe/tasks/ios/test/vision/utils:MPPMaskTestUtils", "//mediapipe/tasks/ios/vision/image_segmenter:MPPImageSegmenter", diff --git a/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm index cd1f70eb7..739871ffd 100644 --- a/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm +++ b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm @@ -15,6 +15,7 @@ #import #import +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" #import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" #import "mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.h" #import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h" @@ -30,11 +31,36 @@ static MPPFileInfo *const kSegmentationImageFileInfo = [[MPPFileInfo alloc] initWithName:@"segmentation_input_rotation0" type:@"jpg"]; static MPPFileInfo *const kSegmentationGoldenImageFileInfo = [[MPPFileInfo alloc] initWithName:@"segmentation_golden_rotation0" type:@"png"]; -static MPPFileInfo *const kImageSegmenterModel = [[MPPFileInfo alloc] initWithName:@"deeplabv3" - type:@"tflite"]; + +static MPPFileInfo *const kMozartImageFileInfo = [[MPPFileInfo alloc] initWithName:@"mozart_square" + type:@"jpg"]; +static MPPFileInfo *const kMozart128x128SegmentationGoldenImageFileInfo = + [[MPPFileInfo alloc] initWithName:@"selfie_segm_128_128_3_expected_mask" type:@"jpg"]; +static MPPFileInfo *const kMozart144x256SegmentationGoldenImageFileInfo = + [[MPPFileInfo alloc] initWithName:@"selfie_segm_144_256_3_expected_mask" type:@"jpg"]; + +static MPPFileInfo *const kImageSegmenterModelFileInfo = + [[MPPFileInfo alloc] initWithName:@"deeplabv3" type:@"tflite"]; +static MPPFileInfo *const kSelfie128x128ModelFileInfo = + [[MPPFileInfo alloc] initWithName:@"selfie_segm_128_128_3" type:@"tflite"]; +static MPPFileInfo *const kSelfie144x256ModelFileInfo = + [[MPPFileInfo alloc] initWithName:@"selfie_segm_144_256_3" type:@"tflite"]; + static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; +static NSString *const kLiveStreamTestsDictImageSegmenterKey = @"image_segmenter"; +static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; + constexpr float kSimilarityThreshold = 0.96f; constexpr NSInteger kMagnificationFactor = 10; +constexpr NSInteger kExpectedDeeplabV3ConfidenceMaskCount = 21; +constexpr NSInteger kExpected128x128SelfieSegmentationConfidenceMaskCount = 2; +constexpr NSInteger kExpected144x256SelfieSegmentationConfidenceMaskCount = 1; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertEqualObjects(error.localizedDescription, expectedError.localizedDescription) namespace { double sum(const std::vector &mask) { @@ -70,9 +96,12 @@ double softIOU(const float *mask1, const float *mask2, size_t size) { return unionSum > 0.0 ? interSectionSum / unionSum : 0.0; } -} // namespace +} // namespace -@interface MPPImageSegmenterTests : XCTestCase +@interface MPPImageSegmenterTests : XCTestCase { + NSDictionary *_liveStreamSucceedsTestDict; + NSDictionary *_outOfOrderTimestampTestDict; +} @end @@ -99,7 +128,7 @@ double softIOU(const float *mask1, const float *mask2, size_t size) { - (void)testSegmentWithCategoryMaskSucceeds { MPPImageSegmenterOptions *options = - [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModel]; + [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModelFileInfo]; options.shouldOutputConfidenceMasks = NO; options.shouldOutputCategoryMask = YES; @@ -113,17 +142,335 @@ double softIOU(const float *mask1, const float *mask2, size_t size) { - (void)testSegmentWithConfidenceMaskSucceeds { MPPImageSegmenterOptions *options = - [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModel]; + [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModelFileInfo]; MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; [self assertResultsOfSegmentImageWithFileInfo:kCatImageFileInfo usingImageSegmenter:imageSegmenter + hasConfidenceMasksCount: + kExpectedDeeplabV3ConfidenceMaskCount approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo:kCatGoldenImageFileInfo atIndex:8 shouldHaveCategoryMask:NO]; } +- (void)testSegmentWith128x128SegmentationSucceeds { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kSelfie128x128ModelFileInfo]; + + MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + + [self assertResultsOfSegmentImageWithFileInfo:kMozartImageFileInfo + usingImageSegmenter:imageSegmenter + hasConfidenceMasksCount: + kExpected128x128SelfieSegmentationConfidenceMaskCount + approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo: + kMozart128x128SegmentationGoldenImageFileInfo + atIndex:1 + shouldHaveCategoryMask:NO]; +} + +- (void)testSegmentWith144x256SegmentationSucceeds { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kSelfie144x256ModelFileInfo]; + + MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + + [self assertResultsOfSegmentImageWithFileInfo:kMozartImageFileInfo + usingImageSegmenter:imageSegmenter + hasConfidenceMasksCount: + kExpected144x256SelfieSegmentationConfidenceMaskCount + approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo: + kMozart144x256SegmentationGoldenImageFileInfo + atIndex:0 + shouldHaveCategoryMask:NO]; +} + +#pragma mark Running Mode Tests + +- (void)testCreateImageSegmenterFailsWithDelegateInNonLiveStreamMode { + MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo}; + for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kSelfie128x128ModelFileInfo]; + + options.runningMode = runningModesToTest[i]; + options.imageSegmenterLiveStreamDelegate = self; + + [self + assertCreateImageSegmenterWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in image or video mode. The " + @"delegate must not be set in the task's options." + }]]; + } +} + +- (void)testCreateImageSegmenterFailsWithMissingDelegateInLiveStreamMode { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kSelfie128x128ModelFileInfo]; + + options.runningMode = MPPRunningModeLiveStream; + + [self assertCreateImageSegmenterWithOptions:options + failsWithExpectedError: + [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in live stream mode. An object " + @"must be set as the delegate of the task in its " + @"options to ensure asynchronous delivery of results." + }]]; +} + +- (void)testSegmentFailsWithCallingWrongApiInImageMode { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModelFileInfo]; + + MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + + MPPImage *image = [MPPImage imageWithFileInfo:kCatImageFileInfo]; + XCTAssertNotNil(image); + + NSError *liveStreamApiCallError; + XCTAssertFalse([imageSegmenter segmentAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Image" + }]; + + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([imageSegmenter segmentVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Image" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testSegmentFailsWithCallingWrongApiInVideoMode { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModelFileInfo]; + options.runningMode = MPPRunningModeVideo; + + MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + + MPPImage *image = [MPPImage imageWithFileInfo:kCatImageFileInfo]; + XCTAssertNotNil(image); + + NSError *liveStreamApiCallError; + XCTAssertFalse([imageSegmenter segmentAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Video" + }]; + + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *imageApiCallError; + XCTAssertFalse([imageSegmenter segmentImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Video" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); +} + +- (void)testSegmentFailsWithCallingWrongApiInLiveStreamMode { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModelFileInfo]; + options.runningMode = MPPRunningModeLiveStream; + options.imageSegmenterLiveStreamDelegate = self; + + MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + + MPPImage *image = [MPPImage imageWithFileInfo:kCatImageFileInfo]; + XCTAssertNotNil(image); + + NSError *imageApiCallError; + XCTAssertFalse([imageSegmenter segmentImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([imageSegmenter segmentVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testSegmentWithVideoModeSucceeds { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModelFileInfo]; + options.runningMode = MPPRunningModeVideo; + + MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + + MPPImage *image = [MPPImage imageWithFileInfo:kCatImageFileInfo]; + XCTAssertNotNil(image); + + for (int i = 0; i < 3; i++) { + MPPImageSegmenterResult *result = [imageSegmenter segmentVideoFrame:image + timestampInMilliseconds:i + error:nil]; + [self assertImageSegmenterResult:result + hasConfidenceMasksCount: + kExpectedDeeplabV3ConfidenceMaskCount + approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo:kCatGoldenImageFileInfo + atIndex:8 + shouldHaveCategoryMask:NO]; + } +} + +// - (void)testSegmentWithOutOfOrderTimestampsAndLiveStreamModeFails { +// MPPImageSegmenterOptions *options = +// [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModelFileInfo]; +// options.runningMode = MPPRunningModeLiveStream; +// options.imageSegmenterLiveStreamDelegate = self; + +// XCTestExpectation *expectation = [[XCTestExpectation alloc] +// initWithDescription:@"segmentWithOutOfOrderTimestampsAndLiveStream"]; + +// expectation.expectedFulfillmentCount = 1; + +// MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + +// _outOfOrderTimestampTestDict = @{ +// kLiveStreamTestsDictImageSegmenterKey : imageSegmenter, +// kLiveStreamTestsDictExpectationKey : expectation +// }; + +// MPPImage *image = [MPPImage imageWithFileInfo:kCatImageFileInfo]; +// XCTAssertNotNil(image); + +// XCTAssertTrue([imageSegmenter segmentAsyncImage:image timestampInMilliseconds:1 error:nil]); + +// NSError *error; +// XCTAssertFalse([imageSegmenter segmentAsyncImage:image +// timestampInMilliseconds:0 +// error:&error]); + +// NSError *expectedError = +// [NSError errorWithDomain:kExpectedErrorDomain +// code:MPPTasksErrorCodeInvalidArgumentError +// userInfo:@{ +// NSLocalizedDescriptionKey : +// @"INVALID_ARGUMENT: Input timestamp must be monotonically +// increasing." +// }]; +// AssertEqualErrors(error, expectedError); + +// NSTimeInterval timeout = 0.5f; +// [self waitForExpectations:@[ expectation ] timeout:timeout]; +// } + +- (void)testSegmentWithLiveStreamModeSucceeds { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModelFileInfo]; + options.runningMode = MPPRunningModeLiveStream; + options.imageSegmenterLiveStreamDelegate = self; + + NSInteger iterationCount = 100; + + // Because of flow limiting, we cannot ensure that the callback will be invoked `iterationCount` + // times. An normal expectation will fail if expectation.fulfill() is not called + // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will + // only succeed if expectation is not fulfilled for the specified `expectedFulfillmentCount`. + // Since in our case we cannot predict how many times the expectation is supposed to be fulfilled + // setting, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and + // `expectation.isInverted = true` ensures that test succeeds ifexpectation is fulfilled <= + // `iterationCount` times. + XCTestExpectation *expectation = + [[XCTestExpectation alloc] initWithDescription:@"segmentWithLiveStream"]; + + expectation.expectedFulfillmentCount = iterationCount + 1; + expectation.inverted = YES; + + MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + + _outOfOrderTimestampTestDict = @{ + kLiveStreamTestsDictImageSegmenterKey : imageSegmenter, + kLiveStreamTestsDictExpectationKey : expectation + }; + + // TODO: Mimic initialization from CMSampleBuffer as live stream mode is most likely to be used + // with the iOS camera. AVCaptureVideoDataOutput sample buffer delegates provide frames of type + // `CMSampleBuffer`. + MPPImage *image = [MPPImage imageWithFileInfo:kCatImageFileInfo]; + XCTAssertNotNil(image); + + for (int i = 0; i < iterationCount; i++) { + XCTAssertTrue([imageSegmenter segmentAsyncImage:image timestampInMilliseconds:i error:nil]); + } + + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; +} + +- (void)imageSegmenter:(MPPImageSegmenter *)imageSegmenter + didFinishSegmentationWithResult:(MPPImageSegmenterResult *)imageSegmenterResult + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError *)error { + [self assertImageSegmenterResult:imageSegmenterResult + hasConfidenceMasksCount: + kExpectedDeeplabV3ConfidenceMaskCount + approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo:kCatGoldenImageFileInfo + atIndex:8 + shouldHaveCategoryMask:NO]; + + if (imageSegmenter == _outOfOrderTimestampTestDict[kLiveStreamTestsDictImageSegmenterKey]) { + [_outOfOrderTimestampTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } else if (imageSegmenter == _liveStreamSucceedsTestDict[kLiveStreamTestsDictImageSegmenterKey]) { + [_liveStreamSucceedsTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } +} + #pragma mark - Image Segmenter Initializers - (MPPImageSegmenterOptions *)imageSegmenterOptionsWithModelFileInfo:(MPPFileInfo *)fileInfo { @@ -142,6 +489,16 @@ double softIOU(const float *mask1, const float *mask2, size_t size) { return imageSegmenter; } +- (void)assertCreateImageSegmenterWithOptions:(MPPImageSegmenterOptions *)options + failsWithExpectedError:(NSError *)expectedError { + NSError *error = nil; + MPPImageSegmenter *imageSegmenter = [[MPPImageSegmenter alloc] initWithOptions:options + error:&error]; + + XCTAssertNil(imageSegmenter); + AssertEqualErrors(error, expectedError); +} + #pragma mark Assert Segmenter Results - (void)assertResultsOfSegmentImageWithFileInfo:(MPPFileInfo *)imageFileInfo usingImageSegmenter:(MPPImageSegmenter *)imageSegmenter @@ -165,6 +522,8 @@ double softIOU(const float *mask1, const float *mask2, size_t size) { - (void)assertResultsOfSegmentImageWithFileInfo:(MPPFileInfo *)imageFileInfo usingImageSegmenter:(MPPImageSegmenter *)imageSegmenter + hasConfidenceMasksCount: + (NSUInteger)expectedConfidenceMasksCount approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo: (MPPFileInfo *)expectedConfidenceMaskFileInfo atIndex:(NSInteger)index @@ -172,8 +531,24 @@ double softIOU(const float *mask1, const float *mask2, size_t size) { MPPImageSegmenterResult *result = [self segmentImageWithFileInfo:imageFileInfo usingImageSegmenter:imageSegmenter]; + [self assertImageSegmenterResult:result + hasConfidenceMasksCount:expectedConfidenceMasksCount + approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo:expectedConfidenceMaskFileInfo + atIndex:index + shouldHaveCategoryMask:shouldHaveCategoryMask]; +} + +- (void)assertImageSegmenterResult:(MPPImageSegmenterResult *)result + hasConfidenceMasksCount: + (NSUInteger)expectedConfidenceMasksCount + approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo: + (MPPFileInfo *)expectedConfidenceMaskFileInfo + atIndex:(NSInteger)index + shouldHaveCategoryMask:(BOOL)shouldHaveCategoryMask { XCTAssertNotNil(result.confidenceMasks); + XCTAssertEqual(result.confidenceMasks.count, expectedConfidenceMasksCount); + if (shouldHaveCategoryMask) { XCTAssertNotNil(result.categoryMask); } else { From d9d40163342c1721ecfdde79e9ce36168dfaa921 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 19 Sep 2023 20:06:25 +0530 Subject: [PATCH 4/5] Uncommented live stream test in iOS image segmenter tests --- .../image_segmenter/MPPImageSegmenterTests.mm | 61 +++++++++---------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm index 739871ffd..e400d66a0 100644 --- a/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm +++ b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm @@ -368,47 +368,44 @@ double softIOU(const float *mask1, const float *mask2, size_t size) { } } -// - (void)testSegmentWithOutOfOrderTimestampsAndLiveStreamModeFails { -// MPPImageSegmenterOptions *options = -// [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModelFileInfo]; -// options.runningMode = MPPRunningModeLiveStream; -// options.imageSegmenterLiveStreamDelegate = self; +- (void)testSegmentWithOutOfOrderTimestampsAndLiveStreamModeFails { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModelFileInfo]; + options.runningMode = MPPRunningModeLiveStream; + options.imageSegmenterLiveStreamDelegate = self; -// XCTestExpectation *expectation = [[XCTestExpectation alloc] -// initWithDescription:@"segmentWithOutOfOrderTimestampsAndLiveStream"]; + XCTestExpectation *expectation = [[XCTestExpectation alloc] + initWithDescription:@"segmentWithOutOfOrderTimestampsAndLiveStream"]; -// expectation.expectedFulfillmentCount = 1; + expectation.expectedFulfillmentCount = 1; -// MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; -// _outOfOrderTimestampTestDict = @{ -// kLiveStreamTestsDictImageSegmenterKey : imageSegmenter, -// kLiveStreamTestsDictExpectationKey : expectation -// }; + _outOfOrderTimestampTestDict = @{ + kLiveStreamTestsDictImageSegmenterKey : imageSegmenter, + kLiveStreamTestsDictExpectationKey : expectation + }; -// MPPImage *image = [MPPImage imageWithFileInfo:kCatImageFileInfo]; -// XCTAssertNotNil(image); + MPPImage *image = [MPPImage imageWithFileInfo:kCatImageFileInfo]; + XCTAssertNotNil(image); -// XCTAssertTrue([imageSegmenter segmentAsyncImage:image timestampInMilliseconds:1 error:nil]); + XCTAssertTrue([imageSegmenter segmentAsyncImage:image timestampInMilliseconds:1 error:nil]); -// NSError *error; -// XCTAssertFalse([imageSegmenter segmentAsyncImage:image -// timestampInMilliseconds:0 -// error:&error]); + NSError *error; + XCTAssertFalse([imageSegmenter segmentAsyncImage:image timestampInMilliseconds:0 error:&error]); -// NSError *expectedError = -// [NSError errorWithDomain:kExpectedErrorDomain -// code:MPPTasksErrorCodeInvalidArgumentError -// userInfo:@{ -// NSLocalizedDescriptionKey : -// @"INVALID_ARGUMENT: Input timestamp must be monotonically -// increasing." -// }]; -// AssertEqualErrors(error, expectedError); + NSError *expectedError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: Input timestamp must be monotonically increasing." + }]; + AssertEqualErrors(error, expectedError); -// NSTimeInterval timeout = 0.5f; -// [self waitForExpectations:@[ expectation ] timeout:timeout]; -// } + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; +} - (void)testSegmentWithLiveStreamModeSucceeds { MPPImageSegmenterOptions *options = From f1a5c8d549f8a215f9b37ac46a40c477004b5731 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 19 Sep 2023 20:10:23 +0530 Subject: [PATCH 5/5] Fixed formatting in MPPImageSegmenter.mm --- .../image_segmenter/sources/MPPImageSegmenter.mm | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm index dfa8845f2..8fad36671 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm @@ -243,17 +243,18 @@ using ::mediapipe::tasks::core::PacketsCallback; return; } - // Output packet map is moved to a block variable that will not be deallocated for the lifetime of the `dispatch_async` call. Eventhough masks are not copied, we have to ensure that they are not de allocated before the delegate call completes. + // Output packet map is moved to a block variable that will not be deallocated for the lifetime of + // the `dispatch_async` call. Since masks are not copied, this ensures that they are only + // deallocated after the delegate call completes. __block PacketMap outputPacketMap = std::move(liveStreamResult.value()); if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { return; } - + dispatch_async(_callbackQueue, ^{ - - MPPImageSegmenterResult *result = - [MPPImageSegmenter imageSegmenterResultWithOutputPacketMap:outputPacketMap - shouldCopyMaskPacketData:NO]; + MPPImageSegmenterResult *result = + [MPPImageSegmenter imageSegmenterResultWithOutputPacketMap:outputPacketMap + shouldCopyMaskPacketData:NO]; [self.imageSegmenterLiveStreamDelegate imageSegmenter:self didFinishSegmentationWithResult:result