From 7ce9e879dfa77f70ae4b58143ae7b9f9adfa5ee7 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 28 Mar 2023 23:13:48 +0530 Subject: [PATCH] Removed failing test for live stream mode classification --- .../core/flow_limiter_calculator.cc | 2 + .../ImageClassifierTests.swift | 207 +++++------------- .../tasks/ios/vision/image_classifier/BUILD | 1 + 3 files changed, 62 insertions(+), 148 deletions(-) diff --git a/mediapipe/calculators/core/flow_limiter_calculator.cc b/mediapipe/calculators/core/flow_limiter_calculator.cc index 5b08f3af5..3c6471b8a 100644 --- a/mediapipe/calculators/core/flow_limiter_calculator.cc +++ b/mediapipe/calculators/core/flow_limiter_calculator.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include "mediapipe/calculators/core/flow_limiter_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -262,6 +263,7 @@ class FlowLimiterCalculator : public CalculatorBase { std::deque frames_in_flight_; std::map allowed_; }; + REGISTER_CALCULATOR(FlowLimiterCalculator); } // namespace mediapipe diff --git a/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift index 8c34a1ab2..3de431078 100644 --- a/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift +++ b/mediapipe/tasks/ios/test/vision/image_classifier/ImageClassifierTests.swift @@ -107,7 +107,8 @@ class ImageClassifierTests: XCTestCase { categoryArray.count, expectedCategoryArray.count) - for (index, (category, expectedCategory)) in zip(categoryArray, expectedCategoryArray) + for (index, (category, expectedCategory)) in + zip(categoryArray, expectedCategoryArray) .enumerated() { assertCategoriesAreEqual( @@ -120,8 +121,22 @@ class ImageClassifierTests: XCTestCase { func assertImageClassifierResultHasOneHead( _ imageClassifierResult: ImageClassifierResult ) { - XCTAssertEqual(imageClassifierResult.classificationResult.classifications.count, 1) - XCTAssertEqual(imageClassifierResult.classificationResult.classifications[0].headIndex, 0) + XCTAssertEqual( + imageClassifierResult.classificationResult.classifications.count, + 1) + XCTAssertEqual( + imageClassifierResult.classificationResult.classifications[0].headIndex, + 0) + } + + func imageWithFileInfo(_ fileInfo: FileInfo) throws -> MPImage { + let mpImage = try XCTUnwrap( + MPImage.imageFromBundle( + withClass: type(of: self), + filename: fileInfo.name, + type: fileInfo.type)) + + return mpImage } func imageClassifierOptionsWithModelPath( @@ -155,7 +170,8 @@ class ImageClassifierTests: XCTestCase { andCategories expectedCategories: [ResultCategory] ) throws { assertImageClassifierResultHasOneHead(imageClassifierResult) - let categories = imageClassifierResult.classificationResult.classifications[0].categories + let categories = + imageClassifierResult.classificationResult.classifications[0].categories XCTAssertEqual(categories.count, expectedCategoryCount) assertEqualCategoryArrays( @@ -188,10 +204,7 @@ class ImageClassifierTests: XCTestCase { andCategories expectedCategories: [ResultCategory] ) throws { let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: fileInfo.name, - type: fileInfo.type)) + imageWithFileInfo(fileInfo)) try assertResultsForClassifyImage( mpImage, @@ -201,19 +214,6 @@ class ImageClassifierTests: XCTestCase { ) } - // func testCreateImageClassifierWithInvalidMaxResultsFails() throws { - // let textClassifierOptions = - // try XCTUnwrap( - // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) - // textClassifierOptions.maxResults = 0 - - // assertCreateTextClassifierThrowsError( - // textClassifierOptions: textClassifierOptions, - // expectedErrorDescription: """ - // INVALID_ARGUMENT: Invalid `max_results` option: value must be != 0. - // """) - // } - func testCreateImageClassifierWithCategoryAllowlistAndDenylistFails() throws { let imageClassifierOptions = @@ -254,7 +254,8 @@ class ImageClassifierTests: XCTestCase { ImageClassifierTests.burgerImage, usingImageClassifier: imageClassifier, hasCategoryCount: ImageClassifierTests.mobileNetCategoriesCount, - andCategories: ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel) + andCategories: + ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel) } func testClassifyWithScoreThresholdSucceeds() throws { @@ -264,7 +265,8 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) imageClassifierOptions.scoreThreshold = 0.25 - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let expectedCategories = [ ResultCategory( @@ -286,9 +288,11 @@ class ImageClassifierTests: XCTestCase { let imageClassifierOptions = try XCTUnwrap( imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) - imageClassifierOptions.categoryAllowlist = ["cheeseburger", "guacamole", "meat loaf"] + imageClassifierOptions.categoryAllowlist = + ["cheeseburger", "guacamole", "meat loaf"] - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let expectedCategories = [ ResultCategory( @@ -319,13 +323,15 @@ class ImageClassifierTests: XCTestCase { let imageClassifierOptions = try XCTUnwrap( - imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) imageClassifierOptions.categoryDenylist = ["bagel"] let maxResults = 3; imageClassifierOptions.maxResults = maxResults; - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let expectedCategories = [ ResultCategory( @@ -356,18 +362,17 @@ class ImageClassifierTests: XCTestCase { let imageClassifierOptions = try XCTUnwrap( - imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) let maxResults = 1; imageClassifierOptions.maxResults = maxResults; - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.multiObjectsImage.name, - type: ImageClassifierTests.multiObjectsImage.type)) + imageWithFileInfo(ImageClassifierTests.multiObjectsImage)) let imageClassifierResult = try XCTUnwrap( imageClassifier.classify( @@ -397,12 +402,14 @@ class ImageClassifierTests: XCTestCase { let imageClassifierOptions = try XCTUnwrap( - imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) let maxResults = 3; imageClassifierOptions.maxResults = maxResults; - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let expectedCategories = [ ResultCategory( @@ -440,12 +447,14 @@ class ImageClassifierTests: XCTestCase { let imageClassifierOptions = try XCTUnwrap( - imageClassifierOptionsWithModelPath(ImageClassifierTests.floatModelPath)) + imageClassifierOptionsWithModelPath( + ImageClassifierTests.floatModelPath)) let maxResults = 3; imageClassifierOptions.maxResults = maxResults; - let imageClassifier = try XCTUnwrap(ImageClassifier(options: imageClassifierOptions)) + let imageClassifier = try XCTUnwrap(ImageClassifier( + options: imageClassifierOptions)) let expectedCategories = [ ResultCategory( @@ -488,7 +497,9 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptionsWithModelPath( ImageClassifierTests.floatModelPath)) imageClassifierOptions.runningMode = runningMode - imageClassifierOptions.completion = {(result: ImageClassifierResult?, error: Error?) -> () in + imageClassifierOptions.completion = {( + result: ImageClassifierResult?, + error: Error?) -> () in } assertCreateImageClassifierThrowsError( @@ -500,7 +511,8 @@ class ImageClassifierTests: XCTestCase { } } - func testImageClassifierFailsWithMissingResultListenerInLiveStreamMode() throws { + func testImageClassifierFailsWithMissingResultListenerInLiveStreamMode() + throws { let imageClassifierOptions = try XCTUnwrap( @@ -527,10 +539,7 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptions)) let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.multiObjectsRotatedImage.name, - type: ImageClassifierTests.multiObjectsRotatedImage.type)) + imageWithFileInfo(ImageClassifierTests.multiObjectsImage)) do { try imageClassifier.classifyAsync( @@ -554,7 +563,8 @@ class ImageClassifierTests: XCTestCase { assertEqualErrorDescriptions( error, expectedLocalizedDescription: """ - The vision task is not initialized with video mode. Current Running Mode: Image + The vision task is not initialized with video mode. Current Running \ + Mode: Image """) } } @@ -572,10 +582,7 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptions)) let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.multiObjectsRotatedImage.name, - type: ImageClassifierTests.multiObjectsRotatedImage.type)) + imageWithFileInfo(ImageClassifierTests.multiObjectsImage)) do { try imageClassifier.classifyAsync( @@ -620,10 +627,7 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptions)) let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.multiObjectsRotatedImage.name, - type: ImageClassifierTests.multiObjectsRotatedImage.type)) + imageWithFileInfo(ImageClassifierTests.multiObjectsImage)) do { let imagClassifierResult = try imageClassifier.classify( @@ -668,10 +672,7 @@ class ImageClassifierTests: XCTestCase { imageClassifierOptions)) let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.burgerImage.name, - type: ImageClassifierTests.burgerImage.type)) + imageWithFileInfo(ImageClassifierTests.burgerImage)) for i in 0..<3 { let imageClassifierResult = try XCTUnwrap( @@ -681,99 +682,9 @@ class ImageClassifierTests: XCTestCase { try assertImageClassifierResult( imageClassifierResult, hasCategoryCount: maxResults, - andCategories: ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel + andCategories: + ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel ) } } - - func testClassifyWithLiveStreamModeSucceeds() throws { - let imageClassifierOptions = - try XCTUnwrap( - imageClassifierOptionsWithModelPath( - ImageClassifierTests.floatModelPath)) - - imageClassifierOptions.runningMode = .liveStream - - let maxResults = 3 - imageClassifierOptions.maxResults = maxResults - - let expectation = expectation(description: "liveStreamClassify") - - imageClassifierOptions.completion = {(result: ImageClassifierResult?, error: Error?) -> () in - do { - try self.assertImageClassifierResult( - try XCTUnwrap(result), - hasCategoryCount: maxResults, - andCategories: ImageClassifierTests.expectedResultsClassifyBurgerImageWithFloatModel) - } - catch { - - } - expectation.fulfill() - } - - let imageClassifier = try XCTUnwrap(ImageClassifier(options: - imageClassifierOptions)) - - let mpImage = try XCTUnwrap( - MPImage.imageFromBundle( - withClass: type(of: self), - filename: ImageClassifierTests.burgerImage.name, - type: ImageClassifierTests.burgerImage.type)) - - for i in 0..<3 { - XCTAssertNoThrow( - try imageClassifier.classifyAsync( - image: mpImage, - timestampMs: i)) - } - - wait(for: [expectation], timeout: 10) - - } - - // func testClassifyWithMaxResultsSucceeds() throws { - // let textClassifierOptions = - // try XCTUnwrap( - // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) - // textClassifierOptions.maxResults = 1 - - // let textClassifier = - // try XCTUnwrap(TextClassifier(options: textClassifierOptions)) - - // try assertResultsForClassify( - // text: TextClassifierTests.negativeText, - // using: textClassifier, - // equals: TextClassifierTests.bertNegativeTextResultsForEdgeTestCases) - // } - - // func testClassifyWithCategoryAllowlistSucceeds() throws { - // let textClassifierOptions = - // try XCTUnwrap( - // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) - // textClassifierOptions.categoryAllowlist = ["negative"] - - // let textClassifier = - // try XCTUnwrap(TextClassifier(options: textClassifierOptions)) - - // try assertResultsForClassify( - // text: TextClassifierTests.negativeText, - // using: textClassifier, - // equals: TextClassifierTests.bertNegativeTextResultsForEdgeTestCases) - // } - - // func testClassifyWithCategoryDenylistSucceeds() throws { - // let textClassifierOptions = - // try XCTUnwrap( - // textClassifierOptionsWithModelPath(TextClassifierTests.bertModelPath)) - // textClassifierOptions.categoryDenylist = ["positive"] - - // let textClassifier = - // try XCTUnwrap(TextClassifier(options: textClassifierOptions)) - - // try assertResultsForClassify( - // text: TextClassifierTests.negativeText, - // using: textClassifier, - // equals: TextClassifierTests.bertNegativeTextResultsForEdgeTestCases) - // } } diff --git a/mediapipe/tasks/ios/vision/image_classifier/BUILD b/mediapipe/tasks/ios/vision/image_classifier/BUILD index 4ebcd2b29..ac9b41f49 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/BUILD +++ b/mediapipe/tasks/ios/vision/image_classifier/BUILD @@ -50,6 +50,7 @@ objc_library( deps = [ ":MPPImageClassifierOptions", ":MPPImageClassifierResult", + "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto", "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils",