From 0d4c365b78640c50322853912369cd57e8169ea2 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 28 Mar 2023 22:55:06 +0530 Subject: [PATCH 1/7] Added iOS Image Classifier Swift Tests --- .../ios/test/vision/image_classifier/BUILD | 31 + .../ImageClassifierTests.swift | 779 ++++++++++++++++++ .../vision/utils/sources/MPPImage+TestUtils.h | 4 +- 3 files changed, 812 insertions(+), 2 deletions(-) create mode 100644 mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/BUILD b/mediapipe/tasks/ios/test/vision/image_classifier/BUILD index c274e6e2e..d4d3b37b9 100644 --- a/mediapipe/tasks/ios/test/vision/image_classifier/BUILD +++ b/mediapipe/tasks/ios/test/vision/image_classifier/BUILD @@ -1,4 +1,8 @@ load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "@build_bazel_rules_swift//swift:swift.bzl", + "swift_library", +) load( "//mediapipe/tasks:ios/ios.bzl", "MPP_TASK_MINIMUM_OS_VERSION", @@ -53,3 +57,30 @@ ios_unit_test( ":MPPImageClassifierObjcTestLibrary", ], ) + +swift_library( + name = "MPPImageClassifierSwiftTestLibrary", + testonly = 1, + srcs = ["ImageClassifierTests.swift"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + tags = TFL_DEFAULT_TAGS, + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + "//mediapipe/tasks/ios/vision/image_classifier:MPPImageClassifier", + ], +) + +ios_unit_test( + name = "MPPImageClassifierSwiftTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPImageClassifierSwiftTestLibrary", + ], +) + diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift new file mode 100644 index 000000000..8c34a1ab2 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift @@ -0,0 +1,779 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import MPPCommon +import MPPImageTestUtils +import XCTest + +@testable import MPPImageClassifier + +typealias FileInfo = (name: String, type: String) + +class ImageClassifierTests: XCTestCase { + + static let bundle = Bundle(for: ImageClassifierTests.self) + + static let floatModelPath = bundle.path( + forResource: "mobilenet_v2_1.0_224", + ofType: "tflite") + + static let burgerImage = FileInfo(name: "burger", type: "jpg") + static let burgerRotatedImage = FileInfo(name: "burger_rotated", type: "jpg") + static let multiObjectsImage = FileInfo(name: "multi_objects", type: "jpg") + static let multiObjectsRotatedImage = FileInfo(name: "multi_objects_rotated", type: "jpg") + + static let mobileNetCategoriesCount: Int = 1001; + + static let expectedResultsClassifyBurgerImageWithFloatModel = [ + ResultCategory( + index: 934, + score: 0.786005, + categoryName: "cheeseburger", + displayName: nil), + ResultCategory( + index: 932, + score: 0.023508, + categoryName: "bagel", + displayName: nil), + ResultCategory( + index: 925, + score: 0.021172, + categoryName: "guacamole", + displayName: nil), + ] + + func assertEqualErrorDescriptions( + _ error: Error, expectedLocalizedDescription: String + ) { + XCTAssertEqual( + error.localizedDescription, + expectedLocalizedDescription) + } + + func assertCategoriesAreEqual( + category: ResultCategory, + expectedCategory: ResultCategory, + indexInCategoryList: Int + ) { + XCTAssertEqual( + category.index, + expectedCategory.index, + String( + format: """ + category[%d].index and expectedCategory[%d].index are not equal. + """, indexInCategoryList)) + XCTAssertEqual( + category.score, + expectedCategory.score, + accuracy: 1e-3, + String( + format: """ + category[%d].score and expectedCategory[%d].score are not equal. + """, indexInCategoryList)) + XCTAssertEqual( + category.categoryName, + expectedCategory.categoryName, + String( + format: """ + category[%d].categoryName and expectedCategory[%d].categoryName are \ + not equal. + """, indexInCategoryList)) + XCTAssertEqual( + category.displayName, + expectedCategory.displayName, + String( + format: """ + category[%d].displayName and expectedCategory[%d].displayName are \ + not equal. + """, indexInCategoryList)) + } + + func assertEqualCategoryArrays( + categoryArray: [ResultCategory], + expectedCategoryArray: [ResultCategory] + ) { + XCTAssertEqual( + categoryArray.count, + expectedCategoryArray.count) + + for (index, (category, expectedCategory)) in zip(categoryArray, expectedCategoryArray) + .enumerated() + { + assertCategoriesAreEqual( + category: category, + expectedCategory: expectedCategory, + indexInCategoryList: index) + } + } + + func assertImageClassifierResultHasOneHead( + _ imageClassifierResult: ImageClassifierResult + ) { + XCTAssertEqual(imageClassifierResult.classificationResult.classifications.count, 1) + XCTAssertEqual(imageClassifierResult.classificationResult.classifications[0].headIndex, 0) + } + + func imageClassifierOptionsWithModelPath( + _ modelPath: String? + ) throws -> ImageClassifierOptions { + let modelPath = try XCTUnwrap(modelPath) + + let imageClassifierOptions = ImageClassifierOptions() + imageClassifierOptions.baseOptions.modelAssetPath = modelPath + + return imageClassifierOptions + } + + func assertCreateImageClassifierThrowsError( + imageClassifierOptions: ImageClassifierOptions, + expectedErrorDescription: String + ) { + do { + let imageClassifier = try ImageClassifier(options: imageClassifierOptions) + XCTAssertNil(imageClassifier) + } catch { + assertEqualErrorDescriptions( + error, + expectedLocalizedDescription: expectedErrorDescription) + } + } + + func assertImageClassifierResult( + _ imageClassifierResult: ImageClassifierResult, + hasCategoryCount expectedCategoryCount: Int, + andCategories expectedCategories: [ResultCategory] + ) throws { + assertImageClassifierResultHasOneHead(imageClassifierResult) + let categories = imageClassifierResult.classificationResult.classifications[0].categories + + XCTAssertEqual(categories.count, expectedCategoryCount) + assertEqualCategoryArrays( + categoryArray: + Array(categories.prefix(expectedCategories.count)), + expectedCategoryArray: expectedCategories) + } + + func assertResultsForClassifyImage( + _ image: MPImage, + usingImageClassifier imageClassifier: ImageClassifier, + hasCategoryCount expectedCategoryCount: Int, + andCategories expectedCategories: [ResultCategory] + ) throws { + let imageClassifierResult = + try XCTUnwrap( + imageClassifier.classify(image: image)) + + try assertImageClassifierResult( + imageClassifierResult, + hasCategoryCount: expectedCategoryCount, + andCategories: expectedCategories + ) + } + + func assertResultsForClassifyImageWithFileInfo( + _ fileInfo: FileInfo, + usingImageClassifier imageClassifier: ImageClassifier, + hasCategoryCount expectedCategoryCount: Int, + andCategories expectedCategories: [ResultCategory] + ) throws { + let mpImage = try XCTUnwrap( + MPImage.imageFromBundle( + withClass: type(of: self), + filename: fileInfo.name, + type: fileInfo.type)) + + try assertResultsForClassifyImage( + mpImage, + usingImageClassifier: imageClassifier, + hasCategoryCount: expectedCategoryCount, + andCategories: expectedCategories + ) + } + + // func testCreateImageClassifierWithInvalidMaxResultsFails() throws { + // let textClassifierOptions = + // try XCTUnwrap( + // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) + // textClassifierOptions.maxResults = 0 + + // assertCreateTextClassifierThrowsError( + // textClassifierOptions: textClassifierOptions, + // expectedErrorDescription: """ + // INVALID_ARGUMENT: Invalid `max_results` option: value must be != 0. + // """) + // } + + func testCreateImageClassifierWithCategoryAllowlistAndDenylistFails() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptions.categoryAllowlist = ["bagel"] + imageClassifierOptions.categoryDenylist = ["guacamole"] + + assertCreateImageClassifierThrowsError( + imageClassifierOptions: imageClassifierOptions, + expectedErrorDescription: """ + INVALID_ARGUMENT: `category_allowlist` and `category_denylist` are \ + mutually exclusive options. + """) + } + + func testClassifyWithModelPathAndFloatModelSucceeds() throws { + + let modelPath = try XCTUnwrap(ImageClassifierTests.floatModelPath) + let imageClassifier = try XCTUnwrap(ImageClassifier(modelPath: modelPath)) + + try assertResultsForClassifyImageWithFileInfo( + ImageClassifierTests.burgerImage, + usingImageClassifier: imageClassifier, + hasCategoryCount: ImageClassifierTests.mobileNetCategoriesCount, + andCategories: ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel) + } + + func testClassifyWithOptionsAndFloatModelSucceeds() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + + try assertResultsForClassifyImageWithFileInfo( + ImageClassifierTests.burgerImage, + usingImageClassifier: imageClassifier, + hasCategoryCount: ImageClassifierTests.mobileNetCategoriesCount, + andCategories: ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel) + } + + func testClassifyWithScoreThresholdSucceeds() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptions.scoreThreshold = 0.25 + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + + let expectedCategories = [ + ResultCategory( + index: 934, + score: 0.786005, + categoryName: "cheeseburger", + displayName: nil), + ] + + try assertResultsForClassifyImageWithFileInfo( + ImageClassifierTests.burgerImage, + usingImageClassifier: imageClassifier, + hasCategoryCount: expectedCategories.count, + andCategories: expectedCategories) + } + + func testClassifyWithAllowlistSucceeds() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptions.categoryAllowlist = ["cheeseburger", "guacamole", "meat loaf"] + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + + let expectedCategories = [ + ResultCategory( + index: 934, + score: 0.786005, + categoryName: "cheeseburger", + displayName: nil), + ResultCategory( + index: 925, + score: 0.021172, + categoryName: "guacamole", + displayName: nil), + ResultCategory( + index: 963, + score: 0.006279315, + categoryName: "meat loaf", + displayName: nil), + ] + + try assertResultsForClassifyImageWithFileInfo( + ImageClassifierTests.burgerImage, + usingImageClassifier: imageClassifier, + hasCategoryCount: expectedCategories.count, + andCategories: expectedCategories) + } + + func testClassifyWithDenylistSucceeds() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptions.categoryDenylist = ["bagel"] + + let maxResults = 3; + imageClassifierOptions.maxResults = maxResults; + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + + let expectedCategories = [ + ResultCategory( + index: 934, + score: 0.786005, + categoryName: "cheeseburger", + displayName: nil), + ResultCategory( + index: 925, + score: 0.021172, + categoryName: "guacamole", + displayName: nil), + ResultCategory( + index: 963, + score: 0.006279315, + categoryName: "meat loaf", + displayName: nil), + ] + + try assertResultsForClassifyImageWithFileInfo( + ImageClassifierTests.burgerImage, + usingImageClassifier: imageClassifier, + hasCategoryCount: maxResults, + andCategories: expectedCategories) + } + + func testClassifyWithRegionOfInterestSucceeds() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + + let maxResults = 1; + imageClassifierOptions.maxResults = maxResults; + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + + let mpImage = try XCTUnwrap( + MPImage.imageFromBundle( + withClass: type(of: self), + filename: ImageClassifierTests.multiObjectsImage.name, + type: ImageClassifierTests.multiObjectsImage.type)) + + let imageClassifierResult = try XCTUnwrap( + imageClassifier.classify( + image: mpImage, + regionOfInterest: CGRect( + x: 0.450, + y: 0.308, + width: 0.164, + height: 0.426))) + + let expectedCategories = [ + ResultCategory( + index: 806, + score: 0.997122, + categoryName: "soccer ball", + displayName: nil), + ] + + + try assertImageClassifierResult( + imageClassifierResult, + hasCategoryCount: maxResults, + andCategories: expectedCategories) + } + + func testClassifyWithOrientationSucceeds() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + + let maxResults = 3; + imageClassifierOptions.maxResults = maxResults; + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + + let expectedCategories = [ + ResultCategory( + index: 934, + score: 0.622074, + categoryName: "cheeseburger", + displayName: nil), + ResultCategory( + index: 963, + score: 0.051214, + categoryName: "meat loaf", + displayName: nil), + ResultCategory( + index: 925, + score: 0.048719, + categoryName: "guacamole", + displayName: nil), + ] + + let mpImage = try XCTUnwrap( + MPImage.imageFromBundle( + withClass: type(of: self), + filename: ImageClassifierTests.burgerRotatedImage.name, + type: ImageClassifierTests.burgerRotatedImage.type, + orientation: .right)) + + try assertResultsForClassifyImage( + mpImage, + usingImageClassifier: imageClassifier, + hasCategoryCount: expectedCategories.count, + andCategories: expectedCategories) + } + + func testClassifyWithOrientationAndRegionOfInterestSucceeds() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + + let maxResults = 3; + imageClassifierOptions.maxResults = maxResults; + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + + let expectedCategories = [ + ResultCategory( + index: 560, + score: 0.682305, + categoryName: "folding chair", + displayName: nil), + ] + + let mpImage = try XCTUnwrap( + MPImage.imageFromBundle( + withClass: type(of: self), + filename: ImageClassifierTests.multiObjectsRotatedImage.name, + type: ImageClassifierTests.multiObjectsRotatedImage.type, + orientation: .right)) + + let imageClassifierResult = try XCTUnwrap( + imageClassifier.classify( + image: mpImage, + regionOfInterest: CGRect( + x: 0.0, + y: 0.1763, + width: 0.5642, + height: 0.1286))) + + + try assertImageClassifierResult( + imageClassifierResult, + hasCategoryCount: maxResults, + andCategories: expectedCategories) + } + + func testImageClassifierFailsWithResultListenerInNonLiveStreamMode() throws { + + let runningModesToTest = [RunningMode.image, RunningMode.video]; + + for runningMode in runningModesToTest { + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) + imageClassifierOptions.runningMode = runningMode + imageClassifierOptions.completion = {(result: ImageClassifierResult?, error: Error?) -> () in + } + + assertCreateImageClassifierThrowsError( + imageClassifierOptions: imageClassifierOptions, + expectedErrorDescription: """ + The vision task is in image or video mode, a user-defined result \ + callback should not be provided. + """) + } + } + + func testImageClassifierFailsWithMissingResultListenerInLiveStreamMode() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) + imageClassifierOptions.runningMode = .liveStream + + assertCreateImageClassifierThrowsError( + imageClassifierOptions: imageClassifierOptions, + expectedErrorDescription: """ + The vision task is in live stream mode, a user-defined result callback \ + must be provided. + """) + } + + func testClassifyFailsWithCallingWrongApiInImageMode() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: + imageClassifierOptions)) + + let mpImage = try XCTUnwrap( + MPImage.imageFromBundle( + withClass: type(of: self), + filename: ImageClassifierTests.multiObjectsRotatedImage.name, + type: ImageClassifierTests.multiObjectsRotatedImage.type)) + + do { + try imageClassifier.classifyAsync( + image: mpImage, + timestampMs:0) + } catch { + assertEqualErrorDescriptions( + error, + expectedLocalizedDescription: """ + The vision task is not initialized with live stream mode. Current \ + Running Mode: Image + """) + } + + do { + let imagClassifierResult = try imageClassifier.classify( + videoFrame: mpImage, + timestampMs: 0) + XCTAssertNil(imagClassifierResult) + } catch { + assertEqualErrorDescriptions( + error, + expectedLocalizedDescription: """ + The vision task is not initialized with video mode. Current Running Mode: Image + """) + } + } + + func testClassifyFailsWithCallingWrongApiInVideoMode() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) + + imageClassifierOptions.runningMode = .video + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: + imageClassifierOptions)) + + let mpImage = try XCTUnwrap( + MPImage.imageFromBundle( + withClass: type(of: self), + filename: ImageClassifierTests.multiObjectsRotatedImage.name, + type: ImageClassifierTests.multiObjectsRotatedImage.type)) + + do { + try imageClassifier.classifyAsync( + image: mpImage, + timestampMs:0) + } catch { + assertEqualErrorDescriptions( + error, + expectedLocalizedDescription: """ + The vision task is not initialized with live stream mode. Current \ + Running Mode: Video + """) + } + + do { + let imagClassifierResult = try imageClassifier.classify( + image: mpImage) + XCTAssertNil(imagClassifierResult) + } catch { + assertEqualErrorDescriptions( + error, + expectedLocalizedDescription: """ + The vision task is not initialized with image mode. Current Running \ + Mode: Video + """) + } + } + + func testClassifyFailsWithCallingWrongApiLiveStreamInMode() throws { + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) + + imageClassifierOptions.runningMode = .liveStream + imageClassifierOptions.completion = {( + result: ImageClassifierResult?, + error: Error?) -> () in + } + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: + imageClassifierOptions)) + + let mpImage = try XCTUnwrap( + MPImage.imageFromBundle( + withClass: type(of: self), + filename: ImageClassifierTests.multiObjectsRotatedImage.name, + type: ImageClassifierTests.multiObjectsRotatedImage.type)) + + do { + let imagClassifierResult = try imageClassifier.classify( + image: mpImage) + XCTAssertNil(imagClassifierResult) + } catch { + assertEqualErrorDescriptions( + error, + expectedLocalizedDescription: """ + The vision task is not initialized with image mode. Current Running \ + Mode: Live Stream + """) + } + + do { + let imagClassifierResult = try imageClassifier.classify( + videoFrame: mpImage, + timestampMs: 0) + XCTAssertNil(imagClassifierResult) + } catch { + assertEqualErrorDescriptions( + error, + expectedLocalizedDescription: """ + The vision task is not initialized with video mode. Current Running \ + Mode: Live Stream + """) + } + } + + func testClassifyWithVideoModeSucceeds() throws { + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) + + imageClassifierOptions.runningMode = .video + + let maxResults = 3; + imageClassifierOptions.maxResults = maxResults + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: + imageClassifierOptions)) + + let mpImage = try XCTUnwrap( + MPImage.imageFromBundle( + withClass: type(of: self), + filename: ImageClassifierTests.burgerImage.name, + type: ImageClassifierTests.burgerImage.type)) + + for i in 0..<3 { + let imageClassifierResult = try XCTUnwrap( + imageClassifier.classify( + videoFrame: mpImage, + timestampMs: i)) + try assertImageClassifierResult( + imageClassifierResult, + hasCategoryCount: maxResults, + andCategories: ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel + ) + } + } + + func testClassifyWithLiveStreamModeSucceeds() throws { + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) + + imageClassifierOptions.runningMode = .liveStream + + let maxResults = 3 + imageClassifierOptions.maxResults = maxResults + + let expectation = expectation(description: "liveStreamClassify") + + imageClassifierOptions.completion = {(result: ImageClassifierResult?, error: Error?) -> () in + do { + try self.assertImageClassifierResult( + try XCTUnwrap(result), + hasCategoryCount: maxResults, + andCategories: ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel) + } + catch { + + } + expectation.fulfill() + } + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: + imageClassifierOptions)) + + let mpImage = try XCTUnwrap( + MPImage.imageFromBundle( + withClass: type(of: self), + filename: ImageClassifierTests.burgerImage.name, + type: ImageClassifierTests.burgerImage.type)) + + for i in 0..<3 { + XCTAssertNoThrow( + try imageClassifier.classifyAsync( + image: mpImage, + timestampMs: i)) + } + + wait(for: [expectation], timeout: 10) + + } + + // func testClassifyWithMaxResultsSucceeds() throws { + // let textClassifierOptions = + // try XCTUnwrap( + // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) + // textClassifierOptions.maxResults = 1 + + // let textClassifier = + // try XCTUnwrap(TextClassifier(options: textClassifierOptions)) + + // try assertResultsForClassify( + // text: TextClassifierTests.negativeText, + // using: textClassifier, + // equals: TextClassifierTests.bertNegativeTextResultsForEdgeTestCases) + // } + + // func testClassifyWithCategoryAllowlistSucceeds() throws { + // let textClassifierOptions = + // try XCTUnwrap( + // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) + // textClassifierOptions.categoryAllowlist = ["negative"] + + // let textClassifier = + // try XCTUnwrap(TextClassifier(options: textClassifierOptions)) + + // try assertResultsForClassify( + // text: TextClassifierTests.negativeText, + // using: textClassifier, + // equals: TextClassifierTests.bertNegativeTextResultsForEdgeTestCases) + // } + + // func testClassifyWithCategoryDenylistSucceeds() throws { + // let textClassifierOptions = + // try XCTUnwrap( + // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) + // textClassifierOptions.categoryDenylist = ["positive"] + + // let textClassifier = + // try XCTUnwrap(TextClassifier(options: textClassifierOptions)) + + // try assertResultsForClassify( + // text: TextClassifierTests.negativeText, + // using: textClassifier, + // equals: TextClassifierTests.bertNegativeTextResultsForEdgeTestCases) + // } +} diff --git a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h index 9dfe29fd3..bf225cd16 100644 --- a/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h +++ b/mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h @@ -37,7 +37,7 @@ NS_ASSUME_NONNULL_BEGIN + (nullable MPPImage *)imageFromBundleWithClass:(Class)classObject fileName:(NSString *)name ofType:(NSString *)type - NS_SWIFT_NAME(imageFromBundle(class:filename:type:)); + NS_SWIFT_NAME(imageFromBundle(withClass:filename:type:)); /** * Loads an image from a file in an app bundle into a `MPPImage` object with the specified @@ -56,7 +56,7 @@ NS_ASSUME_NONNULL_BEGIN fileName:(NSString *)name ofType:(NSString *)type orientation:(UIImageOrientation)imageOrientation - NS_SWIFT_NAME(imageFromBundle(class:filename:type:orientation:)); + NS_SWIFT_NAME(imageFromBundle(withClass:filename:type:orientation:)); @end From 7ce9e879dfa77f70ae4b58143ae7b9f9adfa5ee7 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 28 Mar 2023 23:13:48 +0530 Subject: [PATCH 2/7] Removed failing test for live stream mode classification --- .../core/flow_limiter_calculator.cc | 2 + .../ImageClassifierTests.swift | 207 +++++------------- .../tasks/ios/vision/image_classifier/BUILD | 1 + 3 files changed, 62 insertions(+), 148 deletions(-) diff --git a/mediapipe/calculators/core/flow_limiter_calculator.cc b/mediapipe/calculators/core/flow_limiter_calculator.cc index 5b08f3af5..3c6471b8a 100644 --- a/mediapipe/calculators/core/flow_limiter_calculator.cc +++ b/mediapipe/calculators/core/flow_limiter_calculator.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include "mediapipe/calculators/core/flow_limiter_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -262,6 +263,7 @@ class FlowLimiterCalculator : public CalculatorBase { std::deque frames_in_flight_; std::map allowed_; }; + REGISTER_CALCULATOR(FlowLimiterCalculator); } // namespace mediapipe diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift index 8c34a1ab2..3de431078 100644 --- a/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift +++ b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift @@ -107,7 +107,8 @@ class ImageClassifierTests: XCTestCase { categoryArray.count, expectedCategoryArray.count) - for (index, (category, expectedCategory)) in zip(categoryArray, expectedCategoryArray) + for (index, (category, expectedCategory)) in + zip(categoryArray, expectedCategoryArray) .enumerated() { assertCategoriesAreEqual( @@ -120,8 +121,22 @@ class ImageClassifierTests: XCTestCase { func assertImageClassifierResultHasOneHead( _ imageClassifierResult: ImageClassifierResult ) { - XCTAssertEqual(imageClassifierResult.classificationResult.classifications.count, 1) - XCTAssertEqual(imageClassifierResult.classificationResult.classifications[0].headIndex, 0) + XCTAssertEqual( + imageClassifierResult.classificationResult.classifications.count, + 1) + XCTAssertEqual( + imageClassifierResult.classificationResult.classifications[0].headIndex, + 0) + } + + func imageWithFileInfo(_ fileInfo: FileInfo) throws -> MPImage { + let mpImage = try XCTUnwrap( + MPImage.imageFromBundle( + withClass: type(of: self), + filename: fileInfo.name, + type: fileInfo.type)) + + return mpImage } func imageClassifierOptionsWithModelPath( @@ -155,7 +170,8 @@ class ImageClassifierTests: XCTestCase { andCategories expectedCategories: [ResultCategory] ) throws { assertImageClassifierResultHasOneHead(imageClassifierResult) - let categories = imageClassifierResult.classificationResult.classifications[0].categories + let categories = + imageClassifierResult.classificationResult.classifications[0].categories XCTAssertEqual(categories.count, expectedCategoryCount) assertEqualCategoryArrays( @@ -188,10 +204,7 @@ class ImageClassifierTests: XCTestCase { andCategories expectedCategories: [ResultCategory] ) throws { let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: fileInfo.name, - type: fileInfo.type)) + imageWithFileInfo(fileInfo)) try assertResultsForClassifyImage( mpImage, @@ -201,19 +214,6 @@ class ImageClassifierTests: XCTestCase { ) } - // func testCreateImageClassifierWithInvalidMaxResultsFails() throws { - // let textClassifierOptions = - // try XCTUnwrap( - // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) - // textClassifierOptions.maxResults = 0 - - // assertCreateTextClassifierThrowsError( - // textClassifierOptions: textClassifierOptions, - // expectedErrorDescription: """ - // INVALID_ARGUMENT: Invalid `max_results` option: value must be != 0. - // """) - // } - func testCreateImageClassifierWithCategoryAllowlistAndDenylistFails() throws { let imageClassifierOptions = @@ -254,7 +254,8 @@ class ImageClassifierTests: XCTestCase { ImageClassifierTests.burgerImage, usingImageClassifier: imageClassifier, hasCategoryCount: ImageClassifierTests.mobileNetCategoriesCount, - andCategories: ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel) + andCategories: + ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel) } func testClassifyWithScoreThresholdSucceeds() throws { @@ -264,7 +265,8 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) imageClassifierOptions.scoreThreshold = 0.25 - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let expectedCategories = [ ResultCategory( @@ -286,9 +288,11 @@ class ImageClassifierTests: XCTestCase { let imageClassifierOptions = try XCTUnwrap( imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) - imageClassifierOptions.categoryAllowlist = ["cheeseburger", "guacamole", "meat loaf"] + imageClassifierOptions.categoryAllowlist = + ["cheeseburger", "guacamole", "meat loaf"] - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let expectedCategories = [ ResultCategory( @@ -319,13 +323,15 @@ class ImageClassifierTests: XCTestCase { let imageClassifierOptions = try XCTUnwrap( - imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) imageClassifierOptions.categoryDenylist = ["bagel"] let maxResults = 3; imageClassifierOptions.maxResults = maxResults; - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let expectedCategories = [ ResultCategory( @@ -356,18 +362,17 @@ class ImageClassifierTests: XCTestCase { let imageClassifierOptions = try XCTUnwrap( - imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) let maxResults = 1; imageClassifierOptions.maxResults = maxResults; - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.multiObjectsImage.name, - type: ImageClassifierTests.multiObjectsImage.type)) + imageWithFileInfo(ImageClassifierTests.multiObjectsImage)) let imageClassifierResult = try XCTUnwrap( imageClassifier.classify( @@ -397,12 +402,14 @@ class ImageClassifierTests: XCTestCase { let imageClassifierOptions = try XCTUnwrap( - imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) let maxResults = 3; imageClassifierOptions.maxResults = maxResults; - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let expectedCategories = [ ResultCategory( @@ -440,12 +447,14 @@ class ImageClassifierTests: XCTestCase { let imageClassifierOptions = try XCTUnwrap( - imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) let maxResults = 3; imageClassifierOptions.maxResults = maxResults; - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let expectedCategories = [ ResultCategory( @@ -488,7 +497,9 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptionsWithModelPath( ImageClassifierTests.floatModelPath)) imageClassifierOptions.runningMode = runningMode - imageClassifierOptions.completion = {(result: ImageClassifierResult?, error: Error?) -> () in + imageClassifierOptions.completion = {( + result: ImageClassifierResult?, + error: Error?) -> () in } assertCreateImageClassifierThrowsError( @@ -500,7 +511,8 @@ class ImageClassifierTests: XCTestCase { } } - func testImageClassifierFailsWithMissingResultListenerInLiveStreamMode() throws { + func testImageClassifierFailsWithMissingResultListenerInLiveStreamMode() + throws { let imageClassifierOptions = try XCTUnwrap( @@ -527,10 +539,7 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptions)) let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.multiObjectsRotatedImage.name, - type: ImageClassifierTests.multiObjectsRotatedImage.type)) + imageWithFileInfo(ImageClassifierTests.multiObjectsImage)) do { try imageClassifier.classifyAsync( @@ -554,7 +563,8 @@ class ImageClassifierTests: XCTestCase { assertEqualErrorDescriptions( error, expectedLocalizedDescription: """ - The vision task is not initialized with video mode. Current Running Mode: Image + The vision task is not initialized with video mode. Current Running \ + Mode: Image """) } } @@ -572,10 +582,7 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptions)) let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.multiObjectsRotatedImage.name, - type: ImageClassifierTests.multiObjectsRotatedImage.type)) + imageWithFileInfo(ImageClassifierTests.multiObjectsImage)) do { try imageClassifier.classifyAsync( @@ -620,10 +627,7 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptions)) let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.multiObjectsRotatedImage.name, - type: ImageClassifierTests.multiObjectsRotatedImage.type)) + imageWithFileInfo(ImageClassifierTests.multiObjectsImage)) do { let imagClassifierResult = try imageClassifier.classify( @@ -668,10 +672,7 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptions)) let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.burgerImage.name, - type: ImageClassifierTests.burgerImage.type)) + imageWithFileInfo(ImageClassifierTests.burgerImage)) for i in 0..<3 { let imageClassifierResult = try XCTUnwrap( @@ -681,99 +682,9 @@ class ImageClassifierTests: XCTestCase { try assertImageClassifierResult( imageClassifierResult, hasCategoryCount: maxResults, - andCategories: ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel + andCategories: + ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel ) } } - - func testClassifyWithLiveStreamModeSucceeds() throws { - let imageClassifierOptions = - try XCTUnwrap( - imageClassifierOptionsWithModelPath( - ImageClassifierTests.floatModelPath)) - - imageClassifierOptions.runningMode = .liveStream - - let maxResults = 3 - imageClassifierOptions.maxResults = maxResults - - let expectation = expectation(description: "liveStreamClassify") - - imageClassifierOptions.completion = {(result: ImageClassifierResult?, error: Error?) -> () in - do { - try self.assertImageClassifierResult( - try XCTUnwrap(result), - hasCategoryCount: maxResults, - andCategories: ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel) - } - catch { - - } - expectation.fulfill() - } - - let imageClassifier = try XCTUnwrap(ImageClassifier(options: - imageClassifierOptions)) - - let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.burgerImage.name, - type: ImageClassifierTests.burgerImage.type)) - - for i in 0..<3 { - XCTAssertNoThrow( - try imageClassifier.classifyAsync( - image: mpImage, - timestampMs: i)) - } - - wait(for: [expectation], timeout: 10) - - } - - // func testClassifyWithMaxResultsSucceeds() throws { - // let textClassifierOptions = - // try XCTUnwrap( - // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) - // textClassifierOptions.maxResults = 1 - - // let textClassifier = - // try XCTUnwrap(TextClassifier(options: textClassifierOptions)) - - // try assertResultsForClassify( - // text: TextClassifierTests.negativeText, - // using: textClassifier, - // equals: TextClassifierTests.bertNegativeTextResultsForEdgeTestCases) - // } - - // func testClassifyWithCategoryAllowlistSucceeds() throws { - // let textClassifierOptions = - // try XCTUnwrap( - // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) - // textClassifierOptions.categoryAllowlist = ["negative"] - - // let textClassifier = - // try XCTUnwrap(TextClassifier(options: textClassifierOptions)) - - // try assertResultsForClassify( - // text: TextClassifierTests.negativeText, - // using: textClassifier, - // equals: TextClassifierTests.bertNegativeTextResultsForEdgeTestCases) - // } - - // func testClassifyWithCategoryDenylistSucceeds() throws { - // let textClassifierOptions = - // try XCTUnwrap( - // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) - // textClassifierOptions.categoryDenylist = ["positive"] - - // let textClassifier = - // try XCTUnwrap(TextClassifier(options: textClassifierOptions)) - - // try assertResultsForClassify( - // text: TextClassifierTests.negativeText, - // using: textClassifier, - // equals: TextClassifierTests.bertNegativeTextResultsForEdgeTestCases) - // } } diff --git a/mediapipe/tasks/ios/vision/image_classifier/BUILD b/mediapipe/tasks/ios/vision/image_classifier/BUILD index 4ebcd2b29..ac9b41f49 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/BUILD +++ b/mediapipe/tasks/ios/vision/image_classifier/BUILD @@ -50,6 +50,7 @@ objc_library( deps = [ ":MPPImageClassifierOptions", ":MPPImageClassifierResult", + "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto", "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", From 3bd8b75bc530077d9a99d9a527dc97eb7887c585 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 28 Mar 2023 23:14:55 +0530 Subject: [PATCH 3/7] Removed flow limiter calculator from image classifier target --- mediapipe/tasks/ios/vision/image_classifier/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/image_classifier/BUILD b/mediapipe/tasks/ios/vision/image_classifier/BUILD index ac9b41f49..4ebcd2b29 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/BUILD +++ b/mediapipe/tasks/ios/vision/image_classifier/BUILD @@ -50,7 +50,6 @@ objc_library( deps = [ ":MPPImageClassifierOptions", ":MPPImageClassifierResult", - "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto", "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", From b354795d0008e876897db08507c7484f05c912b4 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 29 Mar 2023 20:52:56 +0530 Subject: [PATCH 4/7] Added flow limiter capability to callback in iOS Image Classifier --- .../ImageClassifierTests.swift | 123 ++++++++++++++++++ .../MPPImageClassifierTests.m | 43 +++++- .../sources/MPPImageClassifier.mm | 22 +++- .../sources/MPPImageClassifierOptions.h | 2 +- .../MPPImageClassifierResult+Helpers.h | 4 +- .../MPPImageClassifierResult+Helpers.mm | 16 ++- 6 files changed, 191 insertions(+), 19 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift index 3de431078..62bf0d487 100644 --- a/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift +++ b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift @@ -499,6 +499,7 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptions.runningMode = runningMode imageClassifierOptions.completion = {( result: ImageClassifierResult?, + timestampMs: Int, error: Error?) -> () in } @@ -620,6 +621,7 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptions.runningMode = .liveStream imageClassifierOptions.completion = {( result: ImageClassifierResult?, + timestampMs: Int, error: Error?) -> () in } @@ -687,4 +689,125 @@ class ImageClassifierTests: XCTestCase { ) } } + + func testClassifyWithOutOfOrderTimestampsAndLiveStreamModeSucceeds() throws { + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) + + imageClassifierOptions.runningMode = .liveStream + + let maxResults = 3 + imageClassifierOptions.maxResults = maxResults + + let expectation = expectation( + description: "classifyWithOutOfOrderTimestampsAndLiveStream") + expectation.expectedFulfillmentCount = 1; + + imageClassifierOptions.completion = {( + result: ImageClassifierResult?, + timestampMs: Int, + error: Error?) -> () in + do { + try self.assertImageClassifierResult( + try XCTUnwrap(result), + hasCategoryCount: maxResults, + andCategories: + ImageClassifierTests + .expectedResultsClassifyBurgerImageWithFloatModel) + } + catch { + // Any errors will be thrown by the wait() method of the expectation. + } + expectation.fulfill() + } + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: + imageClassifierOptions)) + + let mpImage = try XCTUnwrap( + imageWithFileInfo(ImageClassifierTests.burgerImage)) + + XCTAssertNoThrow( + try imageClassifier.classifyAsync( + image: mpImage, + timestampMs: 100)) + + XCTAssertThrowsError( + try imageClassifier.classifyAsync( + image: mpImage, + timestampMs: 0)) {(error) in + assertEqualErrorDescriptions( + error, + expectedLocalizedDescription: """ + INVALID_ARGUMENT: Input timestamp must be monotonically \ + increasing. + """) + } + + wait(for:[expectation], timeout: 0.1) + } + + func testClassifyWithLiveStreamModeSucceeds() throws { + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) + + imageClassifierOptions.runningMode = .liveStream + + let maxResults = 3 + imageClassifierOptions.maxResults = maxResults + + let iterationCount = 100; + + // Because of flow limiting, we cannot ensure that the callback will be + // invoked `iterationCount` times. + // An normal expectation will fail if expectation.fullfill() is not called + // `expectation.expectedFulfillmentCount` times. + // If `expectation.isInverted = true`, the test will only succeed if + // expectation is not fullfilled for the specified `expectedFulfillmentCount`. + // Since in our case we cannot predict how many times the expectation is + // supposed to be fullfilled setting, + // `expectation.expectedFulfillmentCount` = `iterationCount` and + // `expectation.isInverted = true` ensures that test succeeds if + // expectation is not fullfilled `iterationCount` times. + let expectation = expectation(description: "liveStreamClassify") + expectation.expectedFulfillmentCount = iterationCount; + expectation.isInverted = true; + + imageClassifierOptions.completion = {( + result: ImageClassifierResult?, + timestampMs: Int, + error: Error?) -> () in + do { + try self.assertImageClassifierResult( + try XCTUnwrap(result), + hasCategoryCount: maxResults, + andCategories: + ImageClassifierTests + .expectedResultsClassifyBurgerImageWithFloatModel) + } + catch { + // Any errors will be thrown by the wait() method of the expectation. + } + expectation.fulfill() + } + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: + imageClassifierOptions)) + + let mpImage = try XCTUnwrap( + imageWithFileInfo(ImageClassifierTests.burgerImage)) + + for i in 0.. status_or_packets) { NSError *callbackError = nil; - MPPImageClassifierResult *result; - if ([MPPCommonUtils checkCppError:status_or_packets.status() toError:&callbackError]) { - result = [MPPImageClassifierResult - imageClassifierResultWithClassificationsPacket: - status_or_packets.value()[kClassificationsStreamName.cppString]]; + if (![MPPCommonUtils checkCppError:status_or_packets.status() toError:&callbackError]) { + options.completion(nil, Timestamp::Unset().Value(), callbackError); + return; } - options.completion(result, callbackError); + + PacketMap &outputPacketMap = status_or_packets.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPImageClassifierResult *result = [MPPImageClassifierResult + imageClassifierResultWithClassificationsPacket: + outputPacketMap[kClassificationsStreamName.cppString]]; + + options.completion(result, outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond, callbackError); }; } diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h index 2e6022041..a79241711 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifierOptions.h @@ -33,7 +33,7 @@ NS_SWIFT_NAME(ImageClassifierOptions) * be specified when the running mode is set to the live stream mode. * TODO: Add parameter `MPPImage` in the callback. */ -@property(nonatomic, copy) void (^completion)(MPPImageClassifierResult *result, NSError *error); +@property(nonatomic, copy) void (^completion)(MPPImageClassifierResult *result, NSInteger timestmapMs, NSError *error); /** * The locale to use for display names specified through the TFLite Model Metadata, if any. Defaults diff --git a/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.h b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.h index 0375ac2a5..68d939f45 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.h +++ b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.h @@ -18,6 +18,8 @@ NS_ASSUME_NONNULL_BEGIN +static const int kMicroSecondsPerMilliSecond = 1000; + @interface MPPImageClassifierResult (Helpers) /** @@ -28,7 +30,7 @@ NS_ASSUME_NONNULL_BEGIN * * @return An `MPPImageClassifierResult` object that contains a list of image classifications. */ -+ (MPPImageClassifierResult *)imageClassifierResultWithClassificationsPacket: ++ (nullable MPPImageClassifierResult *)imageClassifierResultWithClassificationsPacket: (const mediapipe::Packet &)packet; @end diff --git a/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.mm b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.mm index 09e21b278..03fbdd793 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/utils/sources/MPPImageClassifierResult+Helpers.mm @@ -17,8 +17,6 @@ #include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" -static const int kMicroSecondsPerMilliSecond = 1000; - namespace { using ClassificationResultProto = ::mediapipe::tasks::components::containers::proto::ClassificationResult; @@ -27,10 +25,18 @@ using ::mediapipe::Packet; @implementation MPPImageClassifierResult (Helpers) -+ (MPPImageClassifierResult *)imageClassifierResultWithClassificationsPacket: ++ (nullable MPPImageClassifierResult *)imageClassifierResultWithClassificationsPacket: (const Packet &)packet { - MPPClassificationResult *classificationResult = [MPPClassificationResult - classificationResultWithProto:packet.Get()]; + + MPPClassificationResult *classificationResult; + MPPImageClassifierResult *imageClassifierResult; + + if (!packet.ValidateAsType().ok()) { + return nil; + } + + classificationResult = [MPPClassificationResult + classificationResultWithProto:packet.Get()]; return [[MPPImageClassifierResult alloc] initWithClassificationResult:classificationResult From d2780e4251fe2b27561f2dc7ede43579db842fa8 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 29 Mar 2023 21:06:50 +0530 Subject: [PATCH 5/7] Added swift image classifier test for quantized model --- .../ImageClassifierTests.swift | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift index 62bf0d487..837319d6f 100644 --- a/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift +++ b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift @@ -28,6 +28,10 @@ class ImageClassifierTests: XCTestCase { forResource: "mobilenet_v2_1.0_224", ofType: "tflite") + static let quantizedModelPath = bundle.path( + forResource: "mobilenet_v1_0.25_224_quant", + ofType: "tflite") + static let burgerImage = FileInfo(name: "burger", type: "jpg") static let burgerRotatedImage = FileInfo(name: "burger_rotated", type: "jpg") static let multiObjectsImage = FileInfo(name: "multi_objects", type: "jpg") @@ -258,6 +262,30 @@ class ImageClassifierTests: XCTestCase { ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel) } + func testClassifyWithQuantizedModelSucceeds() throws { + + let imageClassifierOptions = + try XCTUnwrap( + imageClassifierOptionsWithModelPath( + ImageClassifierTests.quantizedModelPath)) + + let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + + let expectedCategories = [ + ResultCategory( + index: 934, + score: 0.972656, + categoryName: "cheeseburger", + displayName: nil), + ] + + try assertResultsForClassifyImageWithFileInfo( + ImageClassifierTests.burgerImage, + usingImageClassifier: imageClassifier, + hasCategoryCount: ImageClassifierTests.mobileNetCategoriesCount, + andCategories: expectedCategories) + } + func testClassifyWithScoreThresholdSucceeds() throws { let imageClassifierOptions = From 1ba8a79f806e15354637fd9494cf95d2ae1cacbd Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Mar 2023 22:22:24 +0530 Subject: [PATCH 6/7] Updated expected fullfilment count of async calls in test --- .../vision/image_classifier/ImageClassifierTests.swift | 7 ++++--- .../test/vision/image_classifier/MPPImageClassifierTests.m | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift index 837319d6f..5aa5f32ee 100644 --- a/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift +++ b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift @@ -798,11 +798,12 @@ class ImageClassifierTests: XCTestCase { // expectation is not fullfilled for the specified `expectedFulfillmentCount`. // Since in our case we cannot predict how many times the expectation is // supposed to be fullfilled setting, - // `expectation.expectedFulfillmentCount` = `iterationCount` and + // `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and // `expectation.isInverted = true` ensures that test succeeds if - // expectation is not fullfilled `iterationCount` times. + // expectation is fullfilled <= `iterationCount` times. let expectation = expectation(description: "liveStreamClassify") - expectation.expectedFulfillmentCount = iterationCount; + + expectation.expectedFulfillmentCount = iterationCount + 1; expectation.isInverted = true; imageClassifierOptions.completion = {( diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m index 472dbe98f..b31025fbb 100644 --- a/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m +++ b/mediapipe/tasks/ios/test/vision/image_classifier/MPPImageClassifierTests.m @@ -670,13 +670,13 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; // expectation is not fullfilled for the specified `expectedFulfillmentCount`. // Since in our case we cannot predict how many times the expectation is // supposed to be fullfilled setting, - // `expectation.expectedFulfillmentCount` = `iterationCount` and + // `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and // `expectation.isInverted = true` ensures that test succeeds if - // expectation is not fullfilled `iterationCount` times. + // expectation is fullfilled <= `iterationCount` times. XCTestExpectation *expectation = [[XCTestExpectation alloc] initWithDescription:@"classifyWithLiveStream"]; - expectation.expectedFulfillmentCount = iterationCount; + expectation.expectedFulfillmentCount = iterationCount + 1; expectation.inverted = YES; options.runningMode = MPPRunningModeLiveStream; From 84710d01c3ab3047a127e54496e6fb3803cdd073 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 6 Apr 2023 20:06:16 +0530 Subject: [PATCH 7/7] Reverted changes in flow limiter calculator --- mediapipe/calculators/core/flow_limiter_calculator.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/mediapipe/calculators/core/flow_limiter_calculator.cc b/mediapipe/calculators/core/flow_limiter_calculator.cc index 3c6471b8a..5b08f3af5 100644 --- a/mediapipe/calculators/core/flow_limiter_calculator.cc +++ b/mediapipe/calculators/core/flow_limiter_calculator.cc @@ -15,7 +15,6 @@ #include #include #include -#include #include "mediapipe/calculators/core/flow_limiter_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -263,7 +262,6 @@ class FlowLimiterCalculator : public CalculatorBase { std::deque frames_in_flight_; std::map allowed_; }; - REGISTER_CALCULATOR(FlowLimiterCalculator); } // namespace mediapipe