From e0b059da586f258c400975cc8891f96ad109e7b2 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 15 Sep 2023 14:16:50 +0530 Subject: [PATCH 1/6] Added iOS MPPFileInfo for tests --- mediapipe/tasks/ios/test/utils/BUILD | 10 +++++ .../ios/test/utils/sources/MPPFileInfo.h | 42 +++++++++++++++++++ .../ios/test/utils/sources/MPPFileInfo.m | 33 +++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 mediapipe/tasks/ios/test/utils/BUILD create mode 100644 mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.h create mode 100644 mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.m diff --git a/mediapipe/tasks/ios/test/utils/BUILD b/mediapipe/tasks/ios/test/utils/BUILD new file mode 100644 index 000000000..3d9e1b20f --- /dev/null +++ b/mediapipe/tasks/ios/test/utils/BUILD @@ -0,0 +1,10 @@ +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPFileInfo", + srcs = ["sources/MPPFileInfo.m"], + hdrs = ["sources/MPPFileInfo.h"], + module_name = "MPPFileInfo", +) diff --git a/mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.h b/mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.h new file mode 100644 index 000000000..666ed0ace --- /dev/null +++ b/mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.h @@ -0,0 +1,42 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPFileInfo : NSObject + +/** The name of the file. */ +@property(nonatomic, readonly) NSString *name; + +/** The type of the file. */ +@property(nonatomic, readonly) NSString *type; + +/** The path to file in the app bundle. */ +@property(nonatomic, readonly, nullable) NSString *path; + +/** + * Initializes an `MPPFileInfo` using the given name and type of file. + * + * @param name The name of the file. + * @param type The type of the file. + * + * @return The `MPPFileInfo` with the given name and type of file. + */ +- (instancetype)initWithName:(NSString *)name type:(NSString *)type; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.m b/mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.m new file mode 100644 index 000000000..ef35f9bb8 --- /dev/null +++ b/mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.m @@ -0,0 +1,33 @@ +// Copyright 2023 The TensorFlow Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.h" + +@implementation MPPFileInfo + +- (instancetype)initWithName:(NSString *)name type:(NSString *)type { + self = [super init]; + if (self) { + _name = name; + _type = type; + } + + return self; +} + +- (NSString *)path { + return [[NSBundle bundleForClass:self.class] pathForResource:self.name ofType:self.type]; +} + +@end From 81ec5801ea34598d93a3e57bc17c2f7807ddbd43 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 15 Sep 2023 14:18:10 +0530 Subject: [PATCH 2/6] Added new initializers for iOS MPPImage in test utils --- mediapipe/tasks/ios/test/vision/utils/BUILD | 6 +++- .../vision/utils/sources/MPPImage+TestUtils.h | 30 +++++++++++++++++++ .../vision/utils/sources/MPPImage+TestUtils.m | 24 +++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/test/vision/utils/BUILD b/mediapipe/tasks/ios/test/vision/utils/BUILD index d117ad73d..b1f1640d1 100644 --- a/mediapipe/tasks/ios/test/vision/utils/BUILD +++ b/mediapipe/tasks/ios/test/vision/utils/BUILD @@ -7,7 +7,11 @@ objc_library( srcs = ["sources/MPPImage+TestUtils.m"], hdrs = ["sources/MPPImage+TestUtils.h"], module_name = "MPPImageTestUtils", - deps = ["//mediapipe/tasks/ios/vision/core:MPPImage"], + deps = [ + "//mediapipe/tasks/ios/vision/core:MPPImage", + "//mediapipe/tasks/ios/test/utils:MPPFileInfo"], +) + ) cc_library( diff --git a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h index 8cd1c6a67..585677b3f 100644 --- a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h +++ b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h @@ -14,6 +14,7 @@ #import +#import "mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" NS_ASSUME_NONNULL_BEGIN @@ -23,6 +24,34 @@ NS_ASSUME_NONNULL_BEGIN */ @interface MPPImage (TestUtils) +/** + * Loads an image from a file in an app bundle into a `MPPImage` object. + * + * @param fileInfo The file info specifying the name and extension of the image + * file in the bundle. + * + * @return The `MPPImage` object contains the loaded image. This method returns + * nil if it cannot load the image. + */ ++ (MPPImage *)imageWithFileInfo:(MPPFileInfo *)fileInfo; + +NS_SWIFT_NAME(image(withFileInfo:)); + +/** + * Loads an image from a file in an app bundle into a `MPPImage` object with the specified + * orientation. + * + * @param fileInfo The file info specifying the name and extension of the image + * file in the bundle. + * + * @return The `MPPImage` object contains the loaded image. This method returns + * nil if it cannot load the image. + */ ++ (MPPImage *)imageWithFileInfo:(MPPFileInfo *)fileInfo + orientation:(UIImageOrientation)orientation + NS_SWIFT_NAME(image(withFileInfo:orientation:)); + +// TODO: Remove after all tests are migrated /** * Loads an image from a file in an app bundle into a `MPPImage` object. * @@ -39,6 +68,7 @@ NS_ASSUME_NONNULL_BEGIN ofType:(NSString *)type NS_SWIFT_NAME(imageFromBundle(class:filename:type:)); +// TODO: Remove after all tests are migrated /** * Loads an image from a file in an app bundle into a `MPPImage` object with the specified * orientation. diff --git a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m index 0b0ef9fbf..f922146fc 100644 --- a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m +++ b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.m @@ -14,6 +14,7 @@ #import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +// TODO: Remove this category after all tests are migrated to the new methods. @interface UIImage (FileUtils) + (nullable UIImage *)imageFromBundleWithClass:(Class)classObject @@ -37,6 +38,28 @@ @implementation MPPImage (TestUtils) ++ (MPPImage *)imageWithFileInfo:(MPPFileInfo *)fileInfo { + if (!fileInfo.path) return nil; + + UIImage *image = [[UIImage alloc] initWithContentsOfFile:fileInfo.path]; + + if (!image) return nil; + + return [[MPPImage alloc] initWithUIImage:image error:nil]; +} + ++ (MPPImage *)imageWithFileInfo:(MPPFileInfo *)fileInfo + orientation:(UIImageOrientation)orientation { + if (!fileInfo.path) return nil; + + UIImage *image = [[UIImage alloc] initWithContentsOfFile:fileInfo.path]; + + if (!image) return nil; + + return [[MPPImage alloc] initWithUIImage:image orientation:orientation error:nil]; +} + +// TODO: Remove after all tests are migrated + (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject fileName:(NSString *)name ofType:(NSString *)type { @@ -45,6 +68,7 @@ return [[MPPImage alloc] initWithUIImage:image error:nil]; } +// TODO: Remove after all tests are migrated + (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject fileName:(NSString *)name ofType:(NSString *)type From d3f7368b27007c5ee93ed7ee1f8895beca7bdd14 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 15 Sep 2023 14:18:23 +0530 Subject: [PATCH 3/6] Added iOS MPPMask test utils --- mediapipe/tasks/ios/test/vision/utils/BUILD | 10 +++ .../vision/utils/sources/MPPMask+TestUtils.h | 42 +++++++++++++ .../vision/utils/sources/MPPMask+TestUtils.m | 63 +++++++++++++++++++ 3 files changed, 115 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.h create mode 100644 mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.m diff --git a/mediapipe/tasks/ios/test/vision/utils/BUILD b/mediapipe/tasks/ios/test/vision/utils/BUILD index b1f1640d1..0fca5c7e6 100644 --- a/mediapipe/tasks/ios/test/vision/utils/BUILD +++ b/mediapipe/tasks/ios/test/vision/utils/BUILD @@ -12,6 +12,16 @@ objc_library( "//mediapipe/tasks/ios/test/utils:MPPFileInfo"], ) +objc_library( + name = "MPPMaskTestUtils", + srcs = ["sources/MPPMask+TestUtils.m"], + hdrs = ["sources/MPPMask+TestUtils.h"], + module_name = "MPPMaskTestUtils", + deps = [ + "//mediapipe/tasks/ios/vision/core:MPPMask", + "//mediapipe/tasks/ios/test/utils:MPPFileInfo", + "//third_party/apple_frameworks:UIKit", + ] ) cc_library( diff --git a/mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.h b/mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.h new file mode 100644 index 000000000..066fcfa8a --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.h @@ -0,0 +1,42 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import + +#import "mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPMask.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Helper utility for initializing `MPPMask` for MediaPipe iOS vision library tests. + */ +@interface MPPMask (TestUtils) + +/** + * Loads an image from a file in an app bundle and Creates an `MPPMask` of type + * `MPPMaskDataTypeUInt8` using the gray scale pixel data of a `UIImage` loaded from a file with the + * given `MPPFileInfo`. + * + * @param fileInfo The file info specifying the name and type of the image file in the app bundle. + * + * @return The `MPPMask` with the pixel data of the loaded image. This method returns `nil` if there + * is an error in loading the image correctly. + */ +- (nullable instancetype)initWithImageFileInfo:(MPPFileInfo *)fileInfo; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.m b/mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.m new file mode 100644 index 000000000..ad69953d6 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.m @@ -0,0 +1,63 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.h" + +@implementation MPPMask (TestUtils) + +- (instancetype)initWithImageFileInfo:(MPPFileInfo *)fileInfo { + UIImage *image = [[UIImage alloc] initWithContentsOfFile:fileInfo.path]; + + if (!image.CGImage) { + return nil; + } + + size_t width = CGImageGetWidth(image.CGImage); + size_t height = CGImageGetHeight(image.CGImage); + + NSInteger bitsPerComponent = 8; + + UInt8 *pixelData = NULL; + + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceGray(); + + // For a gray scale image (single component) with no alpha, the bitmap info is + // `kCGImageAlphaNone` in combination with bytesPerRow being equal to width. + CGContextRef context = CGBitmapContextCreate(nil, width, height, bitsPerComponent, width, + colorSpace, kCGImageAlphaNone); + + if (!context) { + CGColorSpaceRelease(colorSpace); + return nil; + } + + CGContextDrawImage(context, CGRectMake(0, 0, width, height), image.CGImage); + pixelData = (UInt8 *)CGBitmapContextGetData(context); + + // A copy is needed to ensure that the pixel data outlives the `CGContextRelease` call. + // Alternative is to make the context, color space instance variables and release them in + // `dealloc()`. Since Categories don't allow adding instance variables, choosing to copy rather + // than creating a new custom class similar to `MPPMask` only for the tests. + MPPMask *mask = [[MPPMask alloc] initWithUInt8Data:pixelData + width:width + height:height + shouldCopy:YES]; + + CGColorSpaceRelease(colorSpace); + CGContextRelease(context); + + return mask; +} + +@end From fad7f9cdb4534b9118b1c8fbc1e6fae38d0608e1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 15 Sep 2023 14:18:54 +0530 Subject: [PATCH 4/6] Added iOS image segmenter basic Objective C tests --- .../ios/test/vision/image_segmenter/BUILD | 62 +++++ .../image_segmenter/MPPImageSegmenterTests.mm | 245 ++++++++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/image_segmenter/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm diff --git a/mediapipe/tasks/ios/test/vision/image_segmenter/BUILD b/mediapipe/tasks/ios/test/vision/image_segmenter/BUILD new file mode 100644 index 000000000..1df56336e --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/image_segmenter/BUILD @@ -0,0 +1,62 @@ +load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "//mediapipe/framework/tool:ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPImageSegmenterObjcTestLibrary", + testonly = 1, + srcs = ["MPPImageSegmenterTests.mm"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + "//mediapipe/tasks/testdata/vision:test_protos", + ], + deps = [ + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + "//mediapipe/tasks/ios/test/vision/utils:MPPMaskTestUtils", + "//mediapipe/tasks/ios/vision/image_segmenter:MPPImageSegmenter", + "//mediapipe/tasks/ios/vision/image_segmenter:MPPImageSegmenterResult", + ] + select({ + "//third_party:opencv_ios_sim_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_x86_64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//conditions:default": ["@ios_opencv//:OpencvFramework"], + }), +) + +ios_unit_test( + name = "MPPImageSegmenterObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPImageSegmenterObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm new file mode 100644 index 000000000..aa42100c4 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm @@ -0,0 +1,245 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import + +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h" + +static MPPFileInfo *const kCatImageFileInfo = [[MPPFileInfo alloc] initWithName:@"cat" + type:@"jpg"]; +static MPPFileInfo *const kCatGoldenImageFileInfo = [[MPPFileInfo alloc] initWithName:@"cat_mask" + type:@"jpg"]; +static MPPFileInfo *const kSegmentationImageFileInfo = + [[MPPFileInfo alloc] initWithName:@"segmentation_input_rotation0" type:@"jpg"]; +static MPPFileInfo *const kSegmentationGoldenImageFileInfo = + [[MPPFileInfo alloc] initWithName:@"segmentation_golden_rotation0" type:@"png"]; +static MPPFileInfo *const kImageSegmenterModel = [[MPPFileInfo alloc] initWithName:@"deeplabv3" + type:@"tflite"]; +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; +constexpr float kSimilarityThreshold = 0.96f; +constexpr NSInteger kMagnificationFactor = 10; + +double sum(const float *mask, size_t size) { + double sum = 0.0; + for (int i = 0; i < size; i++) { + sum += mask[i]; + } + return sum; +} + +float *multiply(const float *mask1, const float *mask2, size_t size) { + double sum = 0.0; + float *multipliedMask = (float *)malloc(size * sizeof(float)); + if (!multipliedMask) { + exit(-1); + } + for (int i = 0; i < size; i++) { + multipliedMask[i] = mask1[i] * mask2[i]; + } + + return multipliedMask; +} + +double softIOU(const float *mask1, const float *mask2, size_t size) { + float *interSectionVector = multiply(mask1, mask2, size); + double interSectionSum = sum(interSectionVector, size); + free(interSectionVector); + + float *m1m1Vector = multiply(mask1, mask1, size); + double m1m1 = sum(m1m1Vector, size); + free(m1m1Vector); + + float *m2m2Vector = multiply(mask2, mask2, size); + double m2m2 = sum(m2m2Vector, size); + free(m2m2Vector); + + double unionSum = m1m1 + m2m2 - interSectionSum; + + return unionSum > 0.0 ? interSectionSum / unionSum : 0.0; +} + +@interface MPPImageSegmenterTests : XCTestCase + +@end + +@implementation MPPImageSegmenterTests + +#pragma mark General Tests + +- (void)setUp { + // When expected and actual mask sizes are not equal, iterating through mask data results in a + // segmentation fault. Setting this property to `NO`, prevents each test case from executing the + // remaining flow after a failure. Since expected and actual mask sizes are compared before + // iterating through them, this prevents any illegal memory access. + self.continueAfterFailure = NO; +} + ++ (NSString *)filePathWithName : (NSString *)fileName extension : (NSString *)extension { + NSString *filePath = + [[NSBundle bundleForClass:[MPPImageSegmenterTests class]] pathForResource:fileName + ofType:extension]; + return filePath; +} + +#pragma mark Image Mode Tests + +- (void)testSegmentWithCategoryMaskSucceeds { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModel]; + options.shouldOutputConfidenceMasks = NO; + options.shouldOutputCategoryMask = YES; + + MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + + [self assertResultsOfSegmentImageWithFileInfo:kSegmentationImageFileInfo + usingImageSegmenter:imageSegmenter + approximatelyEqualsExpectedCategoryMaskImageWithFileInfo:kSegmentationGoldenImageFileInfo + shouldHaveConfidenceMasks:NO]; +} + +- (void)testSegmentWithConfidenceMaskSucceeds { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModel]; + + MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + + [self assertResultsOfSegmentImageWithFileInfo:kCatImageFileInfo + usingImageSegmenter:imageSegmenter + approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo:kCatGoldenImageFileInfo + atIndex:8 + shouldHaveCategoryMask:NO]; +} + +#pragma mark - Image Segmenter Initializers + +- (MPPImageSegmenterOptions *)imageSegmenterOptionsWithModelFileInfo:(MPPFileInfo *)fileInfo { + MPPImageSegmenterOptions *options = [[MPPImageSegmenterOptions alloc] init]; + options.baseOptions.modelAssetPath = fileInfo.path; + return options; +} + +- (MPPImageSegmenter *)createImageSegmenterWithOptionsSucceeds:(MPPImageSegmenterOptions *)options { + NSError *error; + MPPImageSegmenter *imageSegmenter = [[MPPImageSegmenter alloc] initWithOptions:options + error:&error]; + XCTAssertNotNil(imageSegmenter); + XCTAssertNil(error); + + return imageSegmenter; +} + +#pragma mark Assert Segmenter Results +- (void)assertResultsOfSegmentImageWithFileInfo:(MPPFileInfo *)imageFileInfo + usingImageSegmenter:(MPPImageSegmenter *)imageSegmenter + approximatelyEqualsExpectedCategoryMaskImageWithFileInfo: + (MPPFileInfo *)expectedCategoryMaskFileInfo + shouldHaveConfidenceMasks:(BOOL)shouldHaveConfidenceMasks { + MPPImageSegmenterResult *result = [self segmentImageWithFileInfo:imageFileInfo + usingImageSegmenter:imageSegmenter]; + + XCTAssertNotNil(result.categoryMask); + + if (shouldHaveConfidenceMasks) { + XCTAssertNotNil(result.confidenceMasks); + } + else { + XCTAssertNil(result.confidenceMasks); + } + + [self assertCategoryMask:result.categoryMask + approximatelyEqualsExpectedCategoryMaskImageWithFileInfo:expectedCategoryMaskFileInfo]; +} + +- (void)assertResultsOfSegmentImageWithFileInfo:(MPPFileInfo *)imageFileInfo + usingImageSegmenter:(MPPImageSegmenter *)imageSegmenter + approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo: + (MPPFileInfo *)expectedConfidenceMaskFileInfo + atIndex:(NSInteger)index + shouldHaveCategoryMask:(BOOL)shouldHaveCategoryMask { + MPPImageSegmenterResult *result = [self segmentImageWithFileInfo:imageFileInfo + usingImageSegmenter:imageSegmenter]; + + XCTAssertNotNil(result.confidenceMasks); + + if (shouldHaveCategoryMask) { + XCTAssertNotNil(result.categoryMask); + } + else { + XCTAssertNil(result.categoryMask); + } + + XCTAssertLessThan(index, result.confidenceMasks.count); + + [self assertConfidenceMask:result.confidenceMasks[index] + approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo:expectedConfidenceMaskFileInfo]; +} + +- (MPPImageSegmenterResult *)segmentImageWithFileInfo:(MPPFileInfo *)fileInfo + usingImageSegmenter:(MPPImageSegmenter *)imageSegmenter { + MPPImage *image = [MPPImage imageWithFileInfo:fileInfo]; + XCTAssertNotNil(image); + + NSError *error; + MPPImageSegmenterResult *result = [imageSegmenter segmentImage:image error:&error]; + XCTAssertNil(error); + XCTAssertNotNil(result); + + return result; +} + +- (void)assertCategoryMask:(MPPMask *)categoryMask + approximatelyEqualsExpectedCategoryMaskImageWithFileInfo: + (MPPFileInfo *)expectedCategoryMaskImageFileInfo { + MPPMask *expectedCategoryMask = + [[MPPMask alloc] initWithImageFileInfo:expectedCategoryMaskImageFileInfo]; + + XCTAssertEqual(categoryMask.width, expectedCategoryMask.width); + XCTAssertEqual(categoryMask.height, expectedCategoryMask.height); + + size_t maskSize = categoryMask.width * categoryMask.height; + + const UInt8 *categoryMaskPixelData = categoryMask.uint8Data; + const UInt8 *expectedCategoryMaskPixelData = expectedCategoryMask.uint8Data; + + NSInteger consistentPixels = 0; + + for (int i = 0; i < maskSize; i++) { + consistentPixels += + categoryMaskPixelData[i] * kMagnificationFactor == expectedCategoryMaskPixelData[i] ? 1 : 0; + } + + XCTAssertGreaterThan((float)consistentPixels / (float)maskSize, kSimilarityThreshold); +} + +- (void)assertConfidenceMask:(MPPMask *)confidenceMask + approximatelyEqualsExpectedConfidenceMaskImageWithFileInfo: + (MPPFileInfo *)expectedConfidenceMaskImageFileInfo { + MPPMask *expectedConfidenceMask = + [[MPPMask alloc] initWithImageFileInfo:expectedConfidenceMaskImageFileInfo]; + + XCTAssertEqual(confidenceMask.width, expectedConfidenceMask.width); + XCTAssertEqual(confidenceMask.height, expectedConfidenceMask.height); + + size_t maskSize = confidenceMask.width * confidenceMask.height; + + XCTAssertGreaterThan( + softIOU(confidenceMask.float32Data, expectedConfidenceMask.float32Data, maskSize), + kSimilarityThreshold); +} + +@end From b3be1418dafb46c29a2d62393ef7ed6f56e6fe41 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 15 Sep 2023 14:21:33 +0530 Subject: [PATCH 5/6] Updated multiply function in iOS Image Segmenter tests to use C++ vectors --- .../image_segmenter/MPPImageSegmenterTests.mm | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm index aa42100c4..76822152e 100644 --- a/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm +++ b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm @@ -20,6 +20,9 @@ #import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h" #import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h" +#include +#include + static MPPFileInfo *const kCatImageFileInfo = [[MPPFileInfo alloc] initWithName:@"cat" type:@"jpg"]; static MPPFileInfo *const kCatGoldenImageFileInfo = [[MPPFileInfo alloc] initWithName:@"cat_mask" @@ -34,39 +37,36 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; constexpr float kSimilarityThreshold = 0.96f; constexpr NSInteger kMagnificationFactor = 10; -double sum(const float *mask, size_t size) { +double sum(const std::vector& mask) { double sum = 0.0; - for (int i = 0; i < size; i++) { - sum += mask[i]; + for (const float &maskElement : mask) { + sum += maskElement; } return sum; } -float *multiply(const float *mask1, const float *mask2, size_t size) { +std::vector multiply(const float *mask1, const float *mask2, size_t size) { double sum = 0.0; - float *multipliedMask = (float *)malloc(size * sizeof(float)); - if (!multipliedMask) { - exit(-1); - } + + std::vector multipliedMask; + multipliedMask.reserve(size); + for (int i = 0; i < size; i++) { - multipliedMask[i] = mask1[i] * mask2[i]; + multipliedMask.push_back(mask1[i] * mask2[i]); } - + return multipliedMask; } double softIOU(const float *mask1, const float *mask2, size_t size) { - float *interSectionVector = multiply(mask1, mask2, size); - double interSectionSum = sum(interSectionVector, size); - free(interSectionVector); + std::vector interSectionVector = multiply(mask1, mask2, size); + double interSectionSum = sum(interSectionVector); - float *m1m1Vector = multiply(mask1, mask1, size); - double m1m1 = sum(m1m1Vector, size); - free(m1m1Vector); + std::vector m1m1Vector = multiply(mask1, mask1, size); + double m1m1 = sum(m1m1Vector); - float *m2m2Vector = multiply(mask2, mask2, size); - double m2m2 = sum(m2m2Vector, size); - free(m2m2Vector); + std::vector m2m2Vector = multiply(mask2, mask2, size); + double m2m2 = sum(m2m2Vector); double unionSum = m1m1 + m2m2 - interSectionSum; From 0f511d52d61f81465c9dcdcaeb6ba729b73399b1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 15 Sep 2023 14:24:15 +0530 Subject: [PATCH 6/6] Fixed typo in iOS MPPImageSegmenterResult helpers --- .../utils/sources/MPPImageSegmenterResult+Helpers.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.mm b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.mm index d6e3b1be8..885df734d 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.mm +++ b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.mm @@ -52,7 +52,7 @@ using ::mediapipe::Packet; } if (categoryMaskPacket.ValidateAsType().ok()) { - const Image &cppCategoryMask = confidenceMasksPacket.Get(); + const Image &cppCategoryMask = categoryMaskPacket.Get(); categoryMask = [[MPPMask alloc] initWithUInt8Data:(UInt8 *)cppCategoryMask.GetImageFrameSharedPtr().get()->PixelData() width:cppCategoryMask.width()