From abf0ee892a2139f575c00bc27a62db30193744de Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 21 Sep 2023 11:52:40 -0700 Subject: [PATCH 01/13] Internal Changes PiperOrigin-RevId: 567374962 --- mediapipe/framework/formats/image_frame.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/framework/formats/image_frame.h b/mediapipe/framework/formats/image_frame.h index 6fcefbd38..6931f217f 100644 --- a/mediapipe/framework/formats/image_frame.h +++ b/mediapipe/framework/formats/image_frame.h @@ -44,7 +44,7 @@ #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/tool/type_util.h" -#define IMAGE_FRAME_RAW_IMAGE MEDIAPIPE_HAS_RTTI +#define IMAGE_FRAME_RAW_IMAGE MEDIAPIPE_HAS_RTTI && !defined(NO_RAW_IMAGE) namespace mediapipe { From 859d90b68baaf46add70606d85e329b96d3de37c Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 21 Sep 2023 15:38:20 -0700 Subject: [PATCH 02/13] No public description PiperOrigin-RevId: 567439132 --- mediapipe/framework/formats/image_frame.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/framework/formats/image_frame.h b/mediapipe/framework/formats/image_frame.h index 6931f217f..6fcefbd38 100644 --- a/mediapipe/framework/formats/image_frame.h +++ b/mediapipe/framework/formats/image_frame.h @@ -44,7 +44,7 @@ #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/tool/type_util.h" -#define IMAGE_FRAME_RAW_IMAGE MEDIAPIPE_HAS_RTTI && !defined(NO_RAW_IMAGE) +#define IMAGE_FRAME_RAW_IMAGE MEDIAPIPE_HAS_RTTI namespace mediapipe { From 743118a04ac84e689748bc9f303f5503fb2c1019 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 21 Sep 2023 21:22:26 -0700 Subject: [PATCH 03/13] Fixes multiple typos in the tasks internal files. PiperOrigin-RevId: 567506050 --- mediapipe/tasks/ios/test/vision/face_landmarker/utils/BUILD | 4 ++-- .../tasks/ios/vision/face_stylizer/sources/MPPFaceStylizer.h | 4 ++-- .../java/com/google/mediapipe/tasks/core/TaskOptions.java | 2 +- .../tasks/python/metadata/metadata_writers/metadata_writer.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/face_landmarker/utils/BUILD b/mediapipe/tasks/ios/test/vision/face_landmarker/utils/BUILD index 74f2bd11a..09b215ec5 100644 --- a/mediapipe/tasks/ios/test/vision/face_landmarker/utils/BUILD +++ b/mediapipe/tasks/ios/test/vision/face_landmarker/utils/BUILD @@ -25,7 +25,7 @@ TFL_DISABLED_SANITIZER_TAGS = [ ] objc_library( - name = "MPPFaceLandmarkeResultHelpersTestLibary", + name = "MPPFaceLandmarkeResultHelpersTestLibrary", testonly = 1, srcs = ["sources/MPPFaceLandmarkerResult+HelpersTests.mm"], copts = [ @@ -50,6 +50,6 @@ ios_unit_test( runner = tflite_ios_lab_runner("IOS_LATEST"), tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, deps = [ - ":MPPFaceLandmarkeResultHelpersTestLibary", + ":MPPFaceLandmarkeResultHelpersTestLibrary", ], ) diff --git a/mediapipe/tasks/ios/vision/face_stylizer/sources/MPPFaceStylizer.h b/mediapipe/tasks/ios/vision/face_stylizer/sources/MPPFaceStylizer.h index f4c17ca85..d5e9b84a5 100644 --- a/mediapipe/tasks/ios/vision/face_stylizer/sources/MPPFaceStylizer.h +++ b/mediapipe/tasks/ios/vision/face_stylizer/sources/MPPFaceStylizer.h @@ -68,7 +68,7 @@ NS_SWIFT_NAME(FaceStylizer) * @return A `FaceStylizerResult` that contains the stylized image of the most visible face. The * returned image is copied. The stylized output image size is the same as the model output * size. The `stylizedImage` of the `FaceStylizerResult` is `nil` if there is no face detected in - * the imput image. `FaceStylizerResult` is `nil` if there is an error in initializing the face + * the input image. `FaceStylizerResult` is `nil` if there is an error in initializing the face * stylizer. */ - (nullable MPPFaceStylizerResult *)stylizeImage:(MPPImage *)image @@ -92,7 +92,7 @@ NS_SWIFT_NAME(FaceStylizer) * * @param image The `MPImage` on which face stylization is to be performed. * @param completionHandler A block to be invoked with the results of performing face stylization on - * the imput image. The block takes two arguments, the optional `FaceStylizerResult` that contains + * the input image. The block takes two arguments, the optional `FaceStylizerResult` that contains * the zero-copied stylized image if face stylization was successful and an optional error populated * upon failure. The lifetime of the stylized image is only guaranteed for the duration of the * block. diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskOptions.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskOptions.java index 4ca258429..5b77fd18e 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskOptions.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskOptions.java @@ -25,7 +25,7 @@ import com.google.protobuf.ByteString; /** * MediaPipe Tasks options base class. Any MediaPipe task-specific options class should extend - * {@link TaskOptions} and implement exactly one of converTo*Proto() methods. + * {@link TaskOptions} and implement exactly one of convertTo*Proto() methods. */ public abstract class TaskOptions { /** diff --git a/mediapipe/tasks/python/metadata/metadata_writers/metadata_writer.py b/mediapipe/tasks/python/metadata/metadata_writers/metadata_writer.py index bf0a1ee12..168d4e5ba 100644 --- a/mediapipe/tasks/python/metadata/metadata_writers/metadata_writer.py +++ b/mediapipe/tasks/python/metadata/metadata_writers/metadata_writer.py @@ -736,7 +736,7 @@ class MetadataWriter(object): content is used to interpret the metadata content. Returns: - A tuple of (model_with_metadata_in_bytes, metdata_json_content) + A tuple of (model_with_metadata_in_bytes, metadata_json_content) """ # Populates metadata and associated files into TFLite model buffer. populator = metadata.MetadataPopulator.with_model_buffer(self._model_buffer) @@ -840,6 +840,6 @@ class MetadataWriterBase: content is used to interpret the metadata content. Returns: - A tuple of (model_with_metadata_in_bytes, metdata_json_content) + A tuple of (model_with_metadata_in_bytes, metadata_json_content) """ return self.writer.populate() From 34cedb980b21ddae457b1ce816edbee87b99d407 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 22 Sep 2023 02:21:54 -0700 Subject: [PATCH 04/13] No public description PiperOrigin-RevId: 567562548 --- mediapipe/tasks/web/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/web/package.json b/mediapipe/tasks/web/package.json index 3cb39947e..50b076683 100644 --- a/mediapipe/tasks/web/package.json +++ b/mediapipe/tasks/web/package.json @@ -9,7 +9,7 @@ "import": "./__NAME___bundle.mjs", "require": "./__NAME___bundle.cjs", "default": "./__NAME___bundle.mjs", - "types": "./__NAME___.d.ts" + "types": "./__TYPES__" }, "author": "mediapipe@google.com", "license": "Apache-2.0", From d7c57e4edaf5c461a22cea7c8020518f4105bac5 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Sat, 23 Sep 2023 00:07:20 +0530 Subject: [PATCH 05/13] Updated iOS Face Detector Objective C API names --- .../face_detector/MPPFaceDetectorTests.mm | 52 +++++++++---------- .../face_detector/sources/MPPFaceDetector.h | 6 +-- .../face_detector/sources/MPPFaceDetector.mm | 6 +-- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm b/mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm index 548c4bdbf..752f4bfb9 100644 --- a/mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm +++ b/mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm @@ -109,7 +109,7 @@ static const float kKeypointErrorThreshold = 1e-2; NSError *error; MPPImage *mppImage = [self imageWithFileInfo:kCatImage]; - MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectInImage:mppImage error:&error]; + MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectImage:mppImage error:&error]; XCTAssertNil(error); XCTAssertNotNil(faceDetectorResult); XCTAssertEqual(faceDetectorResult.detections.count, 0); @@ -125,9 +125,9 @@ static const float kKeypointErrorThreshold = 1e-2; MPPImage *image = [self imageWithFileInfo:kPortraitImage]; for (int i = 0; i < 3; i++) { - MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectInVideoFrame:image - timestampInMilliseconds:i - error:nil]; + MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectVideoFrame:image + timestampInMilliseconds:i + error:nil]; [self assertFaceDetectorResult:faceDetectorResult containsExpectedKeypoints:kPortraitExpectedKeypoints]; } @@ -141,9 +141,9 @@ static const float kKeypointErrorThreshold = 1e-2; MPPImage *image = [self imageWithFileInfo:kPortraitRotatedImage]; for (int i = 0; i < 3; i++) { - MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectInVideoFrame:image - timestampInMilliseconds:i - error:nil]; + MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectVideoFrame:image + timestampInMilliseconds:i + error:nil]; [self assertFaceDetectorResult:faceDetectorResult containsExpectedKeypoints:kPortraitRotatedExpectedKeypoints]; } @@ -181,7 +181,7 @@ static const float kKeypointErrorThreshold = 1e-2; }; for (int i = 0; i < iterationCount; i++) { - XCTAssertTrue([faceDetector detectAsyncInImage:image timestampInMilliseconds:i error:nil]); + XCTAssertTrue([faceDetector detectAsyncImage:image timestampInMilliseconds:i error:nil]); } NSTimeInterval timeout = 0.5f; @@ -205,10 +205,10 @@ static const float kKeypointErrorThreshold = 1e-2; }; MPPImage *image = [self imageWithFileInfo:kPortraitImage]; - XCTAssertTrue([faceDetector detectAsyncInImage:image timestampInMilliseconds:1 error:nil]); + XCTAssertTrue([faceDetector detectAsyncImage:image timestampInMilliseconds:1 error:nil]); NSError *error; - XCTAssertFalse([faceDetector detectAsyncInImage:image timestampInMilliseconds:0 error:&error]); + XCTAssertFalse([faceDetector detectAsyncImage:image timestampInMilliseconds:0 error:&error]); NSError *expectedError = [NSError errorWithDomain:kExpectedErrorDomain @@ -274,9 +274,9 @@ static const float kKeypointErrorThreshold = 1e-2; MPPImage *image = [self imageWithFileInfo:kPortraitImage]; NSError *liveStreamApiCallError; - XCTAssertFalse([faceDetector detectAsyncInImage:image - timestampInMilliseconds:0 - error:&liveStreamApiCallError]); + XCTAssertFalse([faceDetector detectAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); NSError *expectedLiveStreamApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -288,9 +288,9 @@ static const float kKeypointErrorThreshold = 1e-2; AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); NSError *videoApiCallError; - XCTAssertFalse([faceDetector detectInVideoFrame:image - timestampInMilliseconds:0 - error:&videoApiCallError]); + XCTAssertFalse([faceDetector detectVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); NSError *expectedVideoApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -312,9 +312,9 @@ static const float kKeypointErrorThreshold = 1e-2; MPPImage *image = [self imageWithFileInfo:kPortraitImage]; NSError *liveStreamApiCallError; - XCTAssertFalse([faceDetector detectAsyncInImage:image - timestampInMilliseconds:0 - error:&liveStreamApiCallError]); + XCTAssertFalse([faceDetector detectAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); NSError *expectedLiveStreamApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -326,7 +326,7 @@ static const float kKeypointErrorThreshold = 1e-2; AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); NSError *imageApiCallError; - XCTAssertFalse([faceDetector detectInImage:image error:&imageApiCallError]); + XCTAssertFalse([faceDetector detectImage:image error:&imageApiCallError]); NSError *expectedImageApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -350,7 +350,7 @@ static const float kKeypointErrorThreshold = 1e-2; MPPImage *image = [self imageWithFileInfo:kPortraitImage]; NSError *imageApiCallError; - XCTAssertFalse([faceDetector detectInImage:image error:&imageApiCallError]); + XCTAssertFalse([faceDetector detectImage:image error:&imageApiCallError]); NSError *expectedImageApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -362,9 +362,9 @@ static const float kKeypointErrorThreshold = 1e-2; AssertEqualErrors(imageApiCallError, expectedImageApiCallError); NSError *videoApiCallError; - XCTAssertFalse([faceDetector detectInVideoFrame:image - timestampInMilliseconds:0 - error:&videoApiCallError]); + XCTAssertFalse([faceDetector detectVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); NSError *expectedVideoApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -407,7 +407,7 @@ static const float kKeypointErrorThreshold = 1e-2; MPPImage *image = [self imageWithFileInfo:kPortraitImage]; for (int i = 0; i < iterationCount; i++) { - XCTAssertTrue([faceDetector detectAsyncInImage:image timestampInMilliseconds:i error:nil]); + XCTAssertTrue([faceDetector detectAsyncImage:image timestampInMilliseconds:i error:nil]); } NSTimeInterval timeout = 0.5f; @@ -503,7 +503,7 @@ static const float kKeypointErrorThreshold = 1e-2; usingFaceDetector:(MPPFaceDetector *)faceDetector containsExpectedKeypoints:(NSArray *)expectedKeypoints { NSError *error; - MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectInImage:mppImage error:&error]; + MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectImage:mppImage error:&error]; XCTAssertNil(error); XCTAssertNotNil(faceDetectorResult); [self assertFaceDetectorResult:faceDetectorResult containsExpectedKeypoints:expectedKeypoints]; diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h index 8adb40679..89cf3d2d3 100644 --- a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h @@ -100,7 +100,7 @@ NS_SWIFT_NAME(FaceDetector) * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying * image data. */ -- (nullable MPPFaceDetectorResult *)detectInImage:(MPPImage *)image +- (nullable MPPFaceDetectorResult *)detectImage:(MPPImage *)image error:(NSError **)error NS_SWIFT_NAME(detect(image:)); /** @@ -127,7 +127,7 @@ NS_SWIFT_NAME(FaceDetector) * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying * image data. */ -- (nullable MPPFaceDetectorResult *)detectInVideoFrame:(MPPImage *)image +- (nullable MPPFaceDetectorResult *)detectVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); @@ -165,7 +165,7 @@ NS_SWIFT_NAME(FaceDetector) * * @return `true` if the image was sent to the task successfully, otherwise `false`. */ -- (BOOL)detectAsyncInImage:(MPPImage *)image +- (BOOL)detectAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error NS_SWIFT_NAME(detectAsync(image:timestampInMilliseconds:)); diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm index 8e82281b4..96434eb47 100644 --- a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm @@ -130,13 +130,13 @@ static NSString *const kTaskName = @"faceDetector"; return [self initWithOptions:options error:error]; } -- (nullable MPPFaceDetectorResult *)detectInImage:(MPPImage *)image error:(NSError **)error { +- (nullable MPPFaceDetectorResult *)detectImage:(MPPImage *)image error:(NSError **)error { std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; return [MPPFaceDetector faceDetectorResultWithOptionalOutputPacketMap:outputPacketMap]; } -- (nullable MPPFaceDetectorResult *)detectInVideoFrame:(MPPImage *)image +- (nullable MPPFaceDetectorResult *)detectVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { std::optional outputPacketMap = @@ -147,7 +147,7 @@ static NSString *const kTaskName = @"faceDetector"; return [MPPFaceDetector faceDetectorResultWithOptionalOutputPacketMap:outputPacketMap]; } -- (BOOL)detectAsyncInImage:(MPPImage *)image +- (BOOL)detectAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { return [_visionTaskRunner processLiveStreamImage:image From 9d42744f8ab0996651d9894062a7308997849f1a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Sat, 23 Sep 2023 00:07:49 +0530 Subject: [PATCH 06/13] Updated iOS Face Landmarker Objective C API names --- .../face_landmarker/MPPFaceLandmarkerTests.mm | 48 +++++++++---------- .../sources/MPPFaceLandmarker.h | 12 ++--- .../sources/MPPFaceLandmarker.mm | 10 ++-- 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm b/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm index 3ebc89466..9e92b76c9 100644 --- a/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm +++ b/mediapipe/tasks/ios/test/vision/face_landmarker/MPPFaceLandmarkerTests.mm @@ -137,8 +137,8 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; NSError *error; MPPImage *mppImage = [self imageWithFileInfo:kCatImage]; - MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInImage:mppImage - error:&error]; + MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectImage:mppImage + error:&error]; XCTAssertNil(error); XCTAssertNotNil(faceLandmarkerResult); XCTAssertEqualObjects(faceLandmarkerResult.faceLandmarks, [NSArray array]); @@ -158,9 +158,9 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; NSArray *expectedLandmarks = [MPPFaceLandmarkerTests expectedLandmarksFromFileInfo:kPortraitExpectedLandmarksName]; for (int i = 0; i < 3; i++) { - MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInVideoFrame:image - timestampInMilliseconds:i - error:nil]; + MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectVideoFrame:image + timestampInMilliseconds:i + error:nil]; [self assertFaceLandmarkerResult:faceLandmarkerResult containsExpectedLandmarks:expectedLandmarks expectedBlendshapes:NULL @@ -200,7 +200,7 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; }; for (int i = 0; i < iterationCount; i++) { - XCTAssertTrue([faceLandmarker detectAsyncInImage:image timestampInMilliseconds:i error:nil]); + XCTAssertTrue([faceLandmarker detectAsyncImage:image timestampInMilliseconds:i error:nil]); } NSTimeInterval timeout = 0.5f; @@ -224,10 +224,10 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; }; MPPImage *image = [self imageWithFileInfo:kPortraitImage]; - XCTAssertTrue([faceLandmarker detectAsyncInImage:image timestampInMilliseconds:1 error:nil]); + XCTAssertTrue([faceLandmarker detectAsyncImage:image timestampInMilliseconds:1 error:nil]); NSError *error; - XCTAssertFalse([faceLandmarker detectAsyncInImage:image timestampInMilliseconds:0 error:&error]); + XCTAssertFalse([faceLandmarker detectAsyncImage:image timestampInMilliseconds:0 error:&error]); NSError *expectedError = [NSError errorWithDomain:kExpectedErrorDomain @@ -292,9 +292,9 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; MPPImage *image = [self imageWithFileInfo:kPortraitImage]; NSError *liveStreamAPICallError; - XCTAssertFalse([faceLandmarker detectAsyncInImage:image - timestampInMilliseconds:0 - error:&liveStreamAPICallError]); + XCTAssertFalse([faceLandmarker detectAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamAPICallError]); NSError *expectedLiveStreamAPICallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -306,9 +306,9 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; AssertEqualErrors(liveStreamAPICallError, expectedLiveStreamAPICallError); NSError *videoAPICallError; - XCTAssertFalse([faceLandmarker detectInVideoFrame:image - timestampInMilliseconds:0 - error:&videoAPICallError]); + XCTAssertFalse([faceLandmarker detectVideoFrame:image + timestampInMilliseconds:0 + error:&videoAPICallError]); NSError *expectedVideoAPICallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -329,9 +329,9 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; MPPImage *image = [self imageWithFileInfo:kPortraitImage]; NSError *liveStreamAPICallError; - XCTAssertFalse([faceLandmarker detectAsyncInImage:image - timestampInMilliseconds:0 - error:&liveStreamAPICallError]); + XCTAssertFalse([faceLandmarker detectAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamAPICallError]); NSError *expectedLiveStreamAPICallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -343,7 +343,7 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; AssertEqualErrors(liveStreamAPICallError, expectedLiveStreamAPICallError); NSError *imageAPICallError; - XCTAssertFalse([faceLandmarker detectInImage:image error:&imageAPICallError]); + XCTAssertFalse([faceLandmarker detectImage:image error:&imageAPICallError]); NSError *expectedImageAPICallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -365,7 +365,7 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; MPPImage *image = [self imageWithFileInfo:kPortraitImage]; NSError *imageAPICallError; - XCTAssertFalse([faceLandmarker detectInImage:image error:&imageAPICallError]); + XCTAssertFalse([faceLandmarker detectImage:image error:&imageAPICallError]); NSError *expectedImageAPICallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -377,9 +377,9 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; AssertEqualErrors(imageAPICallError, expectedImageAPICallError); NSError *videoAPICallError; - XCTAssertFalse([faceLandmarker detectInVideoFrame:image - timestampInMilliseconds:0 - error:&videoAPICallError]); + XCTAssertFalse([faceLandmarker detectVideoFrame:image + timestampInMilliseconds:0 + error:&videoAPICallError]); NSError *expectedVideoAPICallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -539,8 +539,8 @@ constexpr float kFacialTransformationMatrixErrorThreshold = 0.2f; MPPImage *mppImage = [self imageWithFileInfo:fileInfo]; NSError *error; - MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectInImage:mppImage - error:&error]; + MPPFaceLandmarkerResult *faceLandmarkerResult = [faceLandmarker detectImage:mppImage + error:&error]; XCTAssertNil(error); XCTAssertNotNil(faceLandmarkerResult); diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h index ce4e991dd..53cb8ecd7 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.h @@ -71,8 +71,8 @@ NS_SWIFT_NAME(FaceLandmarker) * @return An `FaceLandmarkerResult` that contains a list of landmarks. `nil` if there is an error * in initializing the face landmaker. */ -- (nullable MPPFaceLandmarkerResult *)detectInImage:(MPPImage *)image - error:(NSError **)error NS_SWIFT_NAME(detect(image:)); +- (nullable MPPFaceLandmarkerResult *)detectImage:(MPPImage *)image + error:(NSError **)error NS_SWIFT_NAME(detect(image:)); /** * Performs face landmark detection on the provided video frame of type `MPImage` using the whole @@ -95,9 +95,9 @@ NS_SWIFT_NAME(FaceLandmarker) * @return An `FaceLandmarkerResult` that contains a list of landmarks. `nil` if there is an * error in initializing the face landmaker. */ -- (nullable MPPFaceLandmarkerResult *)detectInVideoFrame:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - error:(NSError **)error +- (nullable MPPFaceLandmarkerResult *)detectVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); /** @@ -132,7 +132,7 @@ NS_SWIFT_NAME(FaceLandmarker) * * @return `true` if the image was sent to the task successfully, otherwise `false`. */ -- (BOOL)detectAsyncInImage:(MPPImage *)image +- (BOOL)detectAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error NS_SWIFT_NAME(detectAsync(image:timestampInMilliseconds:)); diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm index 820ab44ae..8a128475d 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm @@ -154,15 +154,15 @@ static NSString *const kTaskName = @"faceLandmarker"; return [self initWithOptions:options error:error]; } -- (nullable MPPFaceLandmarkerResult *)detectInImage:(MPPImage *)image error:(NSError **)error { +- (nullable MPPFaceLandmarkerResult *)detectImage:(MPPImage *)image error:(NSError **)error { std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; return [MPPFaceLandmarker faceLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } -- (nullable MPPFaceLandmarkerResult *)detectInVideoFrame:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - error:(NSError **)error { +- (nullable MPPFaceLandmarkerResult *)detectVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { std::optional outputPacketMap = [_visionTaskRunner processVideoFrame:image timestampInMilliseconds:timestampInMilliseconds @@ -171,7 +171,7 @@ static NSString *const kTaskName = @"faceLandmarker"; return [MPPFaceLandmarker faceLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } -- (BOOL)detectAsyncInImage:(MPPImage *)image +- (BOOL)detectAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { return [_visionTaskRunner processLiveStreamImage:image From 435bee71e8bbdef994c10e81f115d7cbdf7f61d3 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Sat, 23 Sep 2023 00:08:07 +0530 Subject: [PATCH 07/13] Updated iOS hand landmarker Objective C API names --- .../hand_landmarker/MPPHandLandmarkerTests.m | 60 +++++++++---------- .../sources/MPPHandLandmarker.h | 6 +- .../sources/MPPHandLandmarker.mm | 6 +- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m b/mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m index 1e7470d8e..9e27937d8 100644 --- a/mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m +++ b/mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m @@ -208,10 +208,10 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; return image; } -- (MPPHandLandmarkerResult *)detectInImageWithFileInfo:(ResourceFileInfo *)imageFileInfo - usingHandLandmarker:(MPPHandLandmarker *)handLandmarker { +- (MPPHandLandmarkerResult *)detectImageWithFileInfo:(ResourceFileInfo *)imageFileInfo + usingHandLandmarker:(MPPHandLandmarker *)handLandmarker { MPPImage *mppImage = [self imageWithFileInfo:imageFileInfo]; - MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectInImage:mppImage error:nil]; + MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectImage:mppImage error:nil]; XCTAssertNotNil(handLandmarkerResult); return handLandmarkerResult; @@ -221,8 +221,8 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; usingHandLandmarker:(MPPHandLandmarker *)handLandmarker approximatelyEqualsHandLandmarkerResult: (MPPHandLandmarkerResult *)expectedHandLandmarkerResult { - MPPHandLandmarkerResult *handLandmarkerResult = [self detectInImageWithFileInfo:fileInfo - usingHandLandmarker:handLandmarker]; + MPPHandLandmarkerResult *handLandmarkerResult = [self detectImageWithFileInfo:fileInfo + usingHandLandmarker:handLandmarker]; [self assertHandLandmarkerResult:handLandmarkerResult isApproximatelyEqualToExpectedResult:expectedHandLandmarkerResult]; } @@ -249,8 +249,8 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:handLandmarkerOptions]; - MPPHandLandmarkerResult *handLandmarkerResult = [self detectInImageWithFileInfo:kNoHandsImage - usingHandLandmarker:handLandmarker]; + MPPHandLandmarkerResult *handLandmarkerResult = [self detectImageWithFileInfo:kNoHandsImage + usingHandLandmarker:handLandmarker]; AssertHandLandmarkerResultIsEmpty(handLandmarkerResult); } @@ -264,8 +264,8 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:handLandmarkerOptions]; - MPPHandLandmarkerResult *handLandmarkerResult = [self detectInImageWithFileInfo:kTwoHandsImage - usingHandLandmarker:handLandmarker]; + MPPHandLandmarkerResult *handLandmarkerResult = [self detectImageWithFileInfo:kTwoHandsImage + usingHandLandmarker:handLandmarker]; XCTAssertTrue(handLandmarkerResult.handedness.count == numHands); } @@ -280,7 +280,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *mppImage = [self imageWithFileInfo:kPointingUpRotatedImage orientation:UIImageOrientationRight]; - MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectInImage:mppImage error:nil]; + MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectImage:mppImage error:nil]; [self assertHandLandmarkerResult:handLandmarkerResult isApproximatelyEqualToExpectedResult:[MPPHandLandmarkerTests @@ -339,9 +339,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; NSError *liveStreamApiCallError; - XCTAssertFalse([handLandmarker detectAsyncInImage:image - timestampInMilliseconds:0 - error:&liveStreamApiCallError]); + XCTAssertFalse([handLandmarker detectAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); NSError *expectedLiveStreamApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -354,9 +354,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); NSError *videoApiCallError; - XCTAssertFalse([handLandmarker detectInVideoFrame:image - timestampInMilliseconds:0 - error:&videoApiCallError]); + XCTAssertFalse([handLandmarker detectVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); NSError *expectedVideoApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -378,9 +378,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; NSError *liveStreamApiCallError; - XCTAssertFalse([handLandmarker detectAsyncInImage:image - timestampInMilliseconds:0 - error:&liveStreamApiCallError]); + XCTAssertFalse([handLandmarker detectAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); NSError *expectedLiveStreamApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -393,7 +393,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); NSError *imageApiCallError; - XCTAssertFalse([handLandmarker detectInImage:image error:&imageApiCallError]); + XCTAssertFalse([handLandmarker detectImage:image error:&imageApiCallError]); NSError *expectedImageApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -416,7 +416,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; NSError *imageApiCallError; - XCTAssertFalse([handLandmarker detectInImage:image error:&imageApiCallError]); + XCTAssertFalse([handLandmarker detectImage:image error:&imageApiCallError]); NSError *expectedImageApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -428,9 +428,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; AssertEqualErrors(imageApiCallError, expectedImageApiCallError); NSError *videoApiCallError; - XCTAssertFalse([handLandmarker detectInVideoFrame:image - timestampInMilliseconds:0 - error:&videoApiCallError]); + XCTAssertFalse([handLandmarker detectVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); NSError *expectedVideoApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -452,9 +452,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; for (int i = 0; i < 3; i++) { - MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectInVideoFrame:image - timestampInMilliseconds:i - error:nil]; + MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectVideoFrame:image + timestampInMilliseconds:i + error:nil]; [self assertHandLandmarkerResult:handLandmarkerResult isApproximatelyEqualToExpectedResult:[MPPHandLandmarkerTests thumbUpHandLandmarkerResult]]; } @@ -480,10 +480,10 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; - XCTAssertTrue([handLandmarker detectAsyncInImage:image timestampInMilliseconds:1 error:nil]); + XCTAssertTrue([handLandmarker detectAsyncImage:image timestampInMilliseconds:1 error:nil]); NSError *error; - XCTAssertFalse([handLandmarker detectAsyncInImage:image timestampInMilliseconds:0 error:&error]); + XCTAssertFalse([handLandmarker detectAsyncImage:image timestampInMilliseconds:0 error:&error]); NSError *expectedError = [NSError errorWithDomain:kExpectedErrorDomain @@ -533,7 +533,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kThumbUpImage]; for (int i = 0; i < iterationCount; i++) { - XCTAssertTrue([handLandmarker detectAsyncInImage:image timestampInMilliseconds:i error:nil]); + XCTAssertTrue([handLandmarker detectAsyncImage:image timestampInMilliseconds:i error:nil]); } NSTimeInterval timeout = 0.5f; diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.h b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.h index 80973d391..645f33cd1 100644 --- a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.h +++ b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.h @@ -146,7 +146,7 @@ NS_SWIFT_NAME(HandLandmarker) * @return An `HandLandmarkerResult` object that contains the hand hand landmarks detection * results. */ -- (nullable MPPHandLandmarkerResult *)detectInImage:(MPPImage *)image +- (nullable MPPHandLandmarkerResult *)detectImage:(MPPImage *)image error:(NSError **)error NS_SWIFT_NAME(detect(image:)); /** @@ -176,7 +176,7 @@ NS_SWIFT_NAME(HandLandmarker) * @return An `HandLandmarkerResult` object that contains the hand hand landmarks detection * results. */ -- (nullable MPPHandLandmarkerResult *)detectInVideoFrame:(MPPImage *)image +- (nullable MPPHandLandmarkerResult *)detectVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); @@ -216,7 +216,7 @@ NS_SWIFT_NAME(HandLandmarker) * * @return `YES` if the image was sent to the task successfully, otherwise `NO`. */ -- (BOOL)detectAsyncInImage:(MPPImage *)image +- (BOOL)detectAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error NS_SWIFT_NAME(detectAsync(image:timestampInMilliseconds:)); diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm index cedbf0f23..950cfae91 100644 --- a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm +++ b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm @@ -140,13 +140,13 @@ static NSString *const kTaskName = @"handLandmarker"; return [self initWithOptions:options error:error]; } -- (nullable MPPHandLandmarkerResult *)detectInImage:(MPPImage *)image error:(NSError **)error { +- (nullable MPPHandLandmarkerResult *)detectImage:(MPPImage *)image error:(NSError **)error { std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; return [MPPHandLandmarker handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } -- (nullable MPPHandLandmarkerResult *)detectInVideoFrame:(MPPImage *)image +- (nullable MPPHandLandmarkerResult *)detectVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { std::optional outputPacketMap = @@ -157,7 +157,7 @@ static NSString *const kTaskName = @"handLandmarker"; return [MPPHandLandmarker handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } -- (BOOL)detectAsyncInImage:(MPPImage *)image +- (BOOL)detectAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { return [_visionTaskRunner processLiveStreamImage:image From 2d4e5a75b37c4d12321e36f970b2f26984e6f887 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Sat, 23 Sep 2023 00:08:43 +0530 Subject: [PATCH 08/13] Updated iOS Object Detector Objective C API names --- .../object_detector/MPPObjectDetectorTests.m | 50 +++++++++---------- .../sources/MPPObjectDetector.h | 6 +-- .../sources/MPPObjectDetector.mm | 6 +-- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m index 079682df1..7e483c25c 100644 --- a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m +++ b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m @@ -28,10 +28,10 @@ static const float scoreDifferenceTolerance = 0.02f; static NSString *const kLiveStreamTestsDictObjectDetectorKey = @"object_detector"; static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; -#define AssertEqualErrors(error, expectedError) \ - XCTAssertNotNil(error); \ - XCTAssertEqualObjects(error.domain, expectedError.domain); \ - XCTAssertEqual(error.code, expectedError.code); \ +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ XCTAssertEqualObjects(error.localizedDescription, expectedError.localizedDescription) #define AssertEqualCategories(category, expectedCategory, detectionIndex, categoryIndex) \ @@ -194,7 +194,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; usingObjectDetector:(MPPObjectDetector *)objectDetector maxResults:(NSInteger)maxResults equalsObjectDetectorResult:(MPPObjectDetectorResult *)expectedObjectDetectorResult { - MPPObjectDetectorResult *ObjectDetectorResult = [objectDetector detectInImage:mppImage error:nil]; + MPPObjectDetectorResult *ObjectDetectorResult = [objectDetector detectImage:mppImage error:nil]; [self assertObjectDetectorResult:ObjectDetectorResult isEqualToExpectedResult:expectedObjectDetectorResult @@ -495,9 +495,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; NSError *liveStreamApiCallError; - XCTAssertFalse([objectDetector detectAsyncInImage:image - timestampInMilliseconds:0 - error:&liveStreamApiCallError]); + XCTAssertFalse([objectDetector detectAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); NSError *expectedLiveStreamApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -510,9 +510,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); NSError *videoApiCallError; - XCTAssertFalse([objectDetector detectInVideoFrame:image - timestampInMilliseconds:0 - error:&videoApiCallError]); + XCTAssertFalse([objectDetector detectVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); NSError *expectedVideoApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -533,9 +533,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; NSError *liveStreamApiCallError; - XCTAssertFalse([objectDetector detectAsyncInImage:image - timestampInMilliseconds:0 - error:&liveStreamApiCallError]); + XCTAssertFalse([objectDetector detectAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); NSError *expectedLiveStreamApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -548,7 +548,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); NSError *imageApiCallError; - XCTAssertFalse([objectDetector detectInImage:image error:&imageApiCallError]); + XCTAssertFalse([objectDetector detectImage:image error:&imageApiCallError]); NSError *expectedImageApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -571,7 +571,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; NSError *imageApiCallError; - XCTAssertFalse([objectDetector detectInImage:image error:&imageApiCallError]); + XCTAssertFalse([objectDetector detectImage:image error:&imageApiCallError]); NSError *expectedImageApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -583,9 +583,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; AssertEqualErrors(imageApiCallError, expectedImageApiCallError); NSError *videoApiCallError; - XCTAssertFalse([objectDetector detectInVideoFrame:image - timestampInMilliseconds:0 - error:&videoApiCallError]); + XCTAssertFalse([objectDetector detectVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); NSError *expectedVideoApiCallError = [NSError errorWithDomain:kExpectedErrorDomain @@ -610,9 +610,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; for (int i = 0; i < 3; i++) { - MPPObjectDetectorResult *ObjectDetectorResult = [objectDetector detectInVideoFrame:image - timestampInMilliseconds:i - error:nil]; + MPPObjectDetectorResult *ObjectDetectorResult = [objectDetector detectVideoFrame:image + timestampInMilliseconds:i + error:nil]; [self assertObjectDetectorResult:ObjectDetectorResult isEqualToExpectedResult: @@ -643,10 +643,10 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; - XCTAssertTrue([objectDetector detectAsyncInImage:image timestampInMilliseconds:1 error:nil]); + XCTAssertTrue([objectDetector detectAsyncImage:image timestampInMilliseconds:1 error:nil]); NSError *error; - XCTAssertFalse([objectDetector detectAsyncInImage:image timestampInMilliseconds:0 error:&error]); + XCTAssertFalse([objectDetector detectAsyncImage:image timestampInMilliseconds:0 error:&error]); NSError *expectedError = [NSError errorWithDomain:kExpectedErrorDomain @@ -702,7 +702,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; for (int i = 0; i < iterationCount; i++) { - XCTAssertTrue([objectDetector detectAsyncInImage:image timestampInMilliseconds:i error:nil]); + XCTAssertTrue([objectDetector detectAsyncImage:image timestampInMilliseconds:i error:nil]); } NSTimeInterval timeout = 0.5f; diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h index 82721f47b..4b10fb558 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h @@ -112,7 +112,7 @@ NS_SWIFT_NAME(ObjectDetector) * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying * image data. */ -- (nullable MPPObjectDetectorResult *)detectInImage:(MPPImage *)image +- (nullable MPPObjectDetectorResult *)detectImage:(MPPImage *)image error:(NSError **)error NS_SWIFT_NAME(detect(image:)); /** @@ -138,7 +138,7 @@ NS_SWIFT_NAME(ObjectDetector) * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying * image data. */ -- (nullable MPPObjectDetectorResult *)detectInVideoFrame:(MPPImage *)image +- (nullable MPPObjectDetectorResult *)detectVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); @@ -176,7 +176,7 @@ NS_SWIFT_NAME(ObjectDetector) * * @return `true` if the image was sent to the task successfully, otherwise `false`. */ -- (BOOL)detectAsyncInImage:(MPPImage *)image +- (BOOL)detectAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error NS_SWIFT_NAME(detectAsync(image:timestampInMilliseconds:)); diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm index 86063a1f0..7d2276b2a 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm @@ -128,13 +128,13 @@ static NSString *const kTaskName = @"objectDetector"; return [self initWithOptions:options error:error]; } -- (nullable MPPObjectDetectorResult *)detectInImage:(MPPImage *)image error:(NSError **)error { +- (nullable MPPObjectDetectorResult *)detectImage:(MPPImage *)image error:(NSError **)error { std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; return [MPPObjectDetector objectDetectorResultWithOptionalOutputPacketMap:outputPacketMap]; } -- (nullable MPPObjectDetectorResult *)detectInVideoFrame:(MPPImage *)image +- (nullable MPPObjectDetectorResult *)detectVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { std::optional outputPacketMap = @@ -145,7 +145,7 @@ static NSString *const kTaskName = @"objectDetector"; return [MPPObjectDetector objectDetectorResultWithOptionalOutputPacketMap:outputPacketMap]; } -- (BOOL)detectAsyncInImage:(MPPImage *)image +- (BOOL)detectAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { return [_visionTaskRunner processLiveStreamImage:image From 9d851412272461ec1db1cb190bb10fe41eb32b99 Mon Sep 17 00:00:00 2001 From: Chris McClanahan Date: Fri, 22 Sep 2023 15:04:15 -0700 Subject: [PATCH 09/13] No public description PiperOrigin-RevId: 567726262 --- .../calculators/tensor/image_to_tensor_converter_gl_texture.cc | 1 + mediapipe/calculators/tensor/tensor_converter_calculator.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc index 465e7e0bc..5604aea8f 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc @@ -304,6 +304,7 @@ class GlProcessor : public ImageToTensorConverter { glBindTexture(GL_TEXTURE_2D, 0); glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, 0); + glFlush(); return absl::OkStatus(); } diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator.cc b/mediapipe/calculators/tensor/tensor_converter_calculator.cc index f8268f3d2..e86436589 100644 --- a/mediapipe/calculators/tensor/tensor_converter_calculator.cc +++ b/mediapipe/calculators/tensor/tensor_converter_calculator.cc @@ -406,6 +406,7 @@ absl::Status TensorConverterCalculator::ProcessGPU(CalculatorContext* cc) { glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, 0); #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 + glFlush(); src.Release(); return absl::OkStatus(); })); From 573fdad1732a3801075b6292e35cda5858e1d978 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 22 Sep 2023 16:28:36 -0700 Subject: [PATCH 10/13] Add export_model_with_tokenizer to Text Classifier API. PiperOrigin-RevId: 567744604 --- .../python/text/text_classifier/BUILD | 21 +++- .../text/text_classifier/bert_tokenizer.py | 41 ++++++- .../text_classifier/model_with_tokenizer.py | 35 ++++++ .../model_with_tokenizer_test.py | 105 ++++++++++++++++++ .../text/text_classifier/preprocessor.py | 4 + .../text/text_classifier/text_classifier.py | 55 +++++++++ .../text_classifier/text_classifier_test.py | 6 + 7 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer.py create mode 100644 mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer_test.py diff --git a/mediapipe/model_maker/python/text/text_classifier/BUILD b/mediapipe/model_maker/python/text/text_classifier/BUILD index 2c239e4b0..8b5721590 100644 --- a/mediapipe/model_maker/python/text/text_classifier/BUILD +++ b/mediapipe/model_maker/python/text/text_classifier/BUILD @@ -93,6 +93,23 @@ py_test( deps = [":dataset"], ) +py_library( + name = "model_with_tokenizer", + srcs = ["model_with_tokenizer.py"], +) + +py_test( + name = "model_with_tokenizer_test", + srcs = ["model_with_tokenizer_test.py"], + tags = ["requires-net:external"], + deps = [ + ":bert_tokenizer", + ":model_spec", + ":model_with_tokenizer", + "//mediapipe/model_maker/python/core/utils:hub_loader", + ], +) + py_library( name = "bert_tokenizer", srcs = ["bert_tokenizer.py"], @@ -145,10 +162,12 @@ py_library( name = "text_classifier", srcs = ["text_classifier.py"], deps = [ + ":bert_tokenizer", ":dataset", ":hyperparameters", ":model_options", ":model_spec", + ":model_with_tokenizer", ":preprocessor", ":text_classifier_options", "//mediapipe/model_maker/python/core/data:dataset", @@ -165,7 +184,7 @@ py_library( py_test( name = "text_classifier_test", - size = "large", + size = "enormous", srcs = ["text_classifier_test.py"], data = [ "//mediapipe/model_maker/python/text/text_classifier/testdata", diff --git a/mediapipe/model_maker/python/text/text_classifier/bert_tokenizer.py b/mediapipe/model_maker/python/text/text_classifier/bert_tokenizer.py index 8e92bc29c..ce4b47d4c 100644 --- a/mediapipe/model_maker/python/text/text_classifier/bert_tokenizer.py +++ b/mediapipe/model_maker/python/text/text_classifier/bert_tokenizer.py @@ -56,6 +56,15 @@ class BertFullTokenizer(BertTokenizer): self._seq_len = seq_len def process(self, input_tensor: tf.Tensor) -> Mapping[str, Sequence[int]]: + """Processes one input_tensor example. + + Args: + input_tensor: A tensor with shape (1, None) of a utf-8 encoded string. + + Returns: + A dictionary of lists all with shape (1, self._seq_len) containing the + keys "input_word_ids", "input_type_ids", and "input_mask". + """ tokens = self._tokenizer.tokenize(input_tensor.numpy()[0].decode("utf-8")) tokens = tokens[0 : (self._seq_len - 2)] # account for [CLS] and [SEP] tokens.insert(0, "[CLS]") @@ -96,7 +105,18 @@ class BertFastTokenizer(BertTokenizer): self._sep_id = vocab.index("[SEP]") self._pad_id = vocab.index("[PAD]") - def process(self, input_tensor: tf.Tensor) -> Mapping[str, Sequence[int]]: + def process_fn(self, input_tensor: tf.Tensor) -> Mapping[str, tf.Tensor]: + """Tensor implementation of the process function. + + This implementation can be used within a model graph directly since it + takes in tensors and outputs tensors. + + Args: + input_tensor: Input string tensor + + Returns: + Dictionary of tf.Tensors. + """ input_ids = self._tokenizer.tokenize(input_tensor).flat_values input_ids = input_ids[: (self._seq_len - 2)] input_ids = tf.concat( @@ -112,7 +132,20 @@ class BertFastTokenizer(BertTokenizer): input_type_ids = tf.zeros(self._seq_len, dtype=tf.int32) input_mask = tf.cast(input_ids != self._pad_id, dtype=tf.int32) return { - "input_word_ids": input_ids.numpy().tolist(), - "input_type_ids": input_type_ids.numpy().tolist(), - "input_mask": input_mask.numpy().tolist(), + "input_word_ids": input_ids, + "input_type_ids": input_type_ids, + "input_mask": input_mask, } + + def process(self, input_tensor: tf.Tensor) -> Mapping[str, Sequence[int]]: + """Processes one input_tensor example. + + Args: + input_tensor: A tensor with shape (1, None) of a utf-8 encoded string. + + Returns: + A dictionary of lists all with shape (1, self._seq_len) containing the + keys "input_word_ids", "input_type_ids", and "input_mask". + """ + result = self.process_fn(input_tensor) + return {k: v.numpy().tolist() for k, v in result.items()} diff --git a/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer.py b/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer.py new file mode 100644 index 000000000..95328fb43 --- /dev/null +++ b/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer.py @@ -0,0 +1,35 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Text classifier export module library.""" +import tensorflow as tf + + +class ModelWithTokenizer(tf.keras.Model): + """A model with the tokenizer included in graph for exporting to TFLite.""" + + def __init__(self, tokenizer, model): + super().__init__() + self._tokenizer = tokenizer + self._model = model + + @tf.function( + input_signature=[ + tf.TensorSpec(shape=[None], dtype=tf.string, name="input") + ] + ) + def call(self, input_tensor): + x = self._tokenizer.process_fn(input_tensor) + x = {k: tf.expand_dims(v, axis=0) for k, v in x.items()} + x = self._model(x) + return x diff --git a/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer_test.py b/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer_test.py new file mode 100644 index 000000000..f6c5d2477 --- /dev/null +++ b/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer_test.py @@ -0,0 +1,105 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +from unittest import mock as unittest_mock + +import tensorflow as tf +import tensorflow_hub + +from mediapipe.model_maker.python.core.utils import hub_loader +from mediapipe.model_maker.python.text.text_classifier import bert_tokenizer +from mediapipe.model_maker.python.text.text_classifier import model_spec +from mediapipe.model_maker.python.text.text_classifier import model_with_tokenizer + + +class BertTokenizerTest(tf.test.TestCase): + _SEQ_LEN = 128 + + def setUp(self): + super().setUp() + # Mock tempfile.gettempdir() to be unique for each test to avoid race + # condition when downloading model since these tests may run in parallel. + mock_gettempdir = unittest_mock.patch.object( + tempfile, + "gettempdir", + return_value=self.create_tempdir(), + autospec=True, + ) + self.mock_gettempdir = mock_gettempdir.start() + self.addCleanup(mock_gettempdir.stop) + self._ms = model_spec.SupportedModels.MOBILEBERT_CLASSIFIER.value() + self._tokenizer = self._create_tokenizer() + self._model = self._create_model() + + def _create_tokenizer(self): + vocab_file = os.path.join( + tensorflow_hub.resolve(self._ms.get_path()), "assets", "vocab.txt" + ) + return bert_tokenizer.BertFastTokenizer(vocab_file, True, self._SEQ_LEN) + + def _create_model(self): + encoder_inputs = dict( + input_word_ids=tf.keras.layers.Input( + shape=(self._SEQ_LEN,), + dtype=tf.int32, + name="input_word_ids", + ), + input_mask=tf.keras.layers.Input( + shape=(self._SEQ_LEN,), + dtype=tf.int32, + name="input_mask", + ), + input_type_ids=tf.keras.layers.Input( + shape=(self._SEQ_LEN,), + dtype=tf.int32, + name="input_type_ids", + ), + ) + renamed_inputs = dict( + input_ids=encoder_inputs["input_word_ids"], + input_mask=encoder_inputs["input_mask"], + segment_ids=encoder_inputs["input_type_ids"], + ) + encoder = hub_loader.HubKerasLayerV1V2( + self._ms.get_path(), + signature="tokens", + output_key="pooled_output", + trainable=True, + ) + pooled_output = encoder(renamed_inputs) + + output = tf.keras.layers.Dropout(rate=0.1)(pooled_output) + initializer = tf.keras.initializers.TruncatedNormal(stddev=0.02) + output = tf.keras.layers.Dense( + 2, + kernel_initializer=initializer, + name="output", + activation="softmax", + dtype=tf.float32, + )(output) + return tf.keras.Model(inputs=encoder_inputs, outputs=output) + + def test_model_with_tokenizer(self): + model = model_with_tokenizer.ModelWithTokenizer( + self._tokenizer, self._model + ) + output = model(tf.constant(["Example input".encode("utf-8")])) + self.assertAllEqual(output.shape, (1, 2)) + self.assertEqual(tf.reduce_sum(output), 1) + + +if __name__ == "__main__": + tf.test.main() diff --git a/mediapipe/model_maker/python/text/text_classifier/preprocessor.py b/mediapipe/model_maker/python/text/text_classifier/preprocessor.py index 5954f4ca3..24130f6f8 100644 --- a/mediapipe/model_maker/python/text/text_classifier/preprocessor.py +++ b/mediapipe/model_maker/python/text/text_classifier/preprocessor.py @@ -368,6 +368,10 @@ class BertClassifierPreprocessor: tfrecord_cache_files=tfrecord_cache_files, ) + @property + def tokenizer(self) -> bert_tokenizer.BertTokenizer: + return self._tokenizer + TextClassifierPreprocessor = Union[ BertClassifierPreprocessor, AverageWordEmbeddingClassifierPreprocessor diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py index c067a4ed6..623edbc38 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py @@ -29,10 +29,12 @@ from mediapipe.model_maker.python.core.utils import loss_functions from mediapipe.model_maker.python.core.utils import metrics from mediapipe.model_maker.python.core.utils import model_util from mediapipe.model_maker.python.core.utils import quantization +from mediapipe.model_maker.python.text.text_classifier import bert_tokenizer from mediapipe.model_maker.python.text.text_classifier import dataset as text_ds from mediapipe.model_maker.python.text.text_classifier import hyperparameters as hp from mediapipe.model_maker.python.text.text_classifier import model_options as mo from mediapipe.model_maker.python.text.text_classifier import model_spec as ms +from mediapipe.model_maker.python.text.text_classifier import model_with_tokenizer from mediapipe.model_maker.python.text.text_classifier import preprocessor from mediapipe.model_maker.python.text.text_classifier import text_classifier_options from mediapipe.tasks.python.metadata.metadata_writers import metadata_writer @@ -620,3 +622,56 @@ class _BertClassifier(TextClassifier): ids_name=self._model_spec.tflite_input_name["ids"], mask_name=self._model_spec.tflite_input_name["mask"], segment_name=self._model_spec.tflite_input_name["segment_ids"]) + + def export_model_with_tokenizer( + self, + model_name: str = "model_with_tokenizer.tflite", + quantization_config: Optional[quantization.QuantizationConfig] = None, + ): + """Converts and saves the model to a TFLite file with the tokenizer. + + Note that unlike the export_model method, this export method will include + a FastBertTokenizer in the TFLite graph. The resulting TFLite will not have + metadata information to use with MediaPipe Tasks, but can be run directly + using TFLite Inference: https://www.tensorflow.org/lite/guide/inference + + For more information on the tokenizer, see: + https://www.tensorflow.org/text/api_docs/python/text/FastBertTokenizer + + Args: + model_name: File name to save TFLite model with tokenizer. The full export + path is {self._hparams.export_dir}/{model_name}. + quantization_config: The configuration for model quantization. + """ + tf.io.gfile.makedirs(self._hparams.export_dir) + tflite_file = os.path.join(self._hparams.export_dir, model_name) + if ( + self._hparams.tokenizer + != bert_tokenizer.SupportedBertTokenizers.FAST_BERT_TOKENIZER + ): + print( + f"WARNING: This model was trained with {self._hparams.tokenizer} " + "tokenizer, but the exported model with tokenizer will have a " + f"{bert_tokenizer.SupportedBertTokenizers.FAST_BERT_TOKENIZER} " + "tokenizer." + ) + tokenizer = bert_tokenizer.BertFastTokenizer( + vocab_file=self._text_preprocessor.get_vocab_file(), + do_lower_case=self._model_spec.do_lower_case, + seq_len=self._model_options.seq_len, + ) + else: + tokenizer = self._text_preprocessor.tokenizer + + model = model_with_tokenizer.ModelWithTokenizer(tokenizer, self._model) + model(tf.constant(["Example input data".encode("utf-8")])) # build model + saved_model_file = os.path.join( + self._hparams.export_dir, "saved_model_with_tokenizer" + ) + model.save(saved_model_file) + tflite_model = model_util.convert_to_tflite_from_file( + saved_model_file, + quantization_config=quantization_config, + allow_custom_ops=True, + ) + model_util.save_tflite(tflite_model, tflite_file) diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py index 122182ddd..fdc2613a9 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py @@ -149,6 +149,12 @@ class TextClassifierTest(tf.test.TestCase, parameterized.TestCase): output_metadata_file, self._BERT_CLASSIFIER_JSON_FILE, shallow=False ) ) + bert_classifier.export_model_with_tokenizer() + output_tflite_with_tokenizer_file = os.path.join( + options.hparams.export_dir, 'model_with_tokenizer.tflite' + ) + self.assertTrue(os.path.exists(output_tflite_with_tokenizer_file)) + self.assertGreater(os.path.getsize(output_tflite_with_tokenizer_file), 0) def test_label_mismatch(self): options = text_classifier.TextClassifierOptions( From e5e75eac5e3ce1ebb647ead5e453c0dfeb5bd916 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 26 Sep 2023 10:30:18 -0700 Subject: [PATCH 11/13] No public description PiperOrigin-RevId: 568581409 --- third_party/com_github_glog_glog.diff | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/third_party/com_github_glog_glog.diff b/third_party/com_github_glog_glog.diff index bf08045b3..1e83001bd 100644 --- a/third_party/com_github_glog_glog.diff +++ b/third_party/com_github_glog_glog.diff @@ -39,3 +39,18 @@ index 4028ccc..483e639 100644 if (append_newline) { // Fix the ostrstream back how it was before we screwed with it. // It's 99.44% certain that we don't need to worry about doing this. + +diff --git a/bazel/glog.bzl b/bazel/glog.bzl +index 4208d9e..3c86abf 100644 +--- a/bazel/glog.bzl ++++ b/bazel/glog.bzl +@@ -180,6 +180,9 @@ def glog_library(namespace = "google", with_gflags = 1, **kwargs): + }), + copts = + select({ ++ "@bazel_tools//src/conditions:windows": ["-std=c++17"], ++ "//conditions:default": [], ++ }) + select({ + "@bazel_tools//src/conditions:windows": common_copts + windows_only_copts, + "@bazel_tools//src/conditions:darwin": common_copts + linux_or_darwin_copts + darwin_only_copts, + "@bazel_tools//src/conditions:freebsd": common_copts + linux_or_darwin_copts + freebsd_only_copts, From 199b42278bb85b2fa5b459ebf0d30ca011dc9315 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 26 Sep 2023 10:42:41 -0700 Subject: [PATCH 12/13] Fixes multiple typos in the tasks internal files. PiperOrigin-RevId: 568585517 --- mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm | 6 +++--- mediapipe/tasks/ios/test/vision/core/MPPImageTests.m | 2 +- .../tasks/vision/imagesegmenter/ImageSegmenter.java | 2 +- .../vision/gesturerecognizer/GestureRecognizerTest.java | 2 +- .../tasks/web/audio/audio_classifier/audio_classifier.ts | 2 +- .../web/vision/gesture_recognizer/gesture_recognizer.ts | 4 ++-- .../tasks/web/vision/hand_landmarker/hand_landmarker.ts | 4 ++-- .../tasks/web/vision/pose_landmarker/pose_landmarker.ts | 4 ++-- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm index 9a2aab6ae..5510cbb3f 100644 --- a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm @@ -59,12 +59,12 @@ using absl::StatusCode; return NULL; } - void *allocedMemory = malloc(memSize); - if (!allocedMemory) { + void *allocatedMemory = malloc(memSize); + if (!allocatedMemory) { exit(-1); } - return allocedMemory; + return allocatedMemory; } + (BOOL)checkCppError:(const absl::Status &)status toError:(NSError *_Nullable *)error { diff --git a/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m b/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m index a7fa97bfa..d9a84a29e 100644 --- a/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m +++ b/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m @@ -82,7 +82,7 @@ static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; AssertEqualErrors(error, expectedError); } -- (void)testInitWithImageSuceeds { +- (void)testInitWithImageSucceeds { MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:self.image error:nil]; [self assertMPPImage:mppImage hasSourceType:MPPImageSourceTypeImage diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java index 5837e6d06..813dba93c 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java @@ -739,7 +739,7 @@ public final class ImageSegmenter extends BaseVisionTaskApi { @AutoValue public abstract static class SegmentationOptions { - /** Builder fo {@link SegmentationOptions} */ + /** Builder for {@link SegmentationOptions} */ @AutoValue.Builder public abstract static class Builder { diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java index 68d800fe9..3858c2387 100644 --- a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerTest.java @@ -289,7 +289,7 @@ public class GestureRecognizerTest { } @Test - public void recognize_successWithPreferAlowListThanDenyList() throws Exception { + public void recognize_successWithPreferAllowListThanDenyList() throws Exception { GestureRecognizerOptions options = GestureRecognizerOptions.builder() .setBaseOptions( diff --git a/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts b/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts index 35b502f82..2cc0ebe51 100644 --- a/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts +++ b/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts @@ -140,7 +140,7 @@ export class AudioClassifier extends AudioTaskRunner { * @param sampleRate The sample rate in Hz of the provided audio data. If not * set, defaults to the sample rate set via `setDefaultSampleRate()` or * `48000` if no custom default was set. - * @return The classification result of the audio datas + * @return The classification result of the audio data */ classify(audioData: Float32Array, sampleRate?: number): AudioClassifierResult[] { diff --git a/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts b/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts index 295d64f28..1a6a43f2d 100644 --- a/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts +++ b/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts @@ -342,7 +342,7 @@ export class GestureRecognizer extends VisionTaskRunner { * Converts raw data into a landmark, and adds it to our worldLandmarks * list. */ - private adddJsWorldLandmarks(data: Uint8Array[]): void { + private addJsWorldLandmarks(data: Uint8Array[]): void { for (const binaryProto of data) { const handWorldLandmarksProto = LandmarkList.deserializeBinary(binaryProto); @@ -396,7 +396,7 @@ export class GestureRecognizer extends VisionTaskRunner { this.graphRunner.attachProtoVectorListener( WORLD_LANDMARKS_STREAM, (binaryProto, timestamp) => { - this.adddJsWorldLandmarks(binaryProto); + this.addJsWorldLandmarks(binaryProto); this.setLatestOutputTimestamp(timestamp); }); this.graphRunner.attachEmptyPacketListener( diff --git a/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts b/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts index 6b65362a7..efb3fc6e1 100644 --- a/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts +++ b/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts @@ -277,7 +277,7 @@ export class HandLandmarker extends VisionTaskRunner { * Converts raw data into a world landmark, and adds it to our worldLandmarks * list. */ - private adddJsWorldLandmarks(data: Uint8Array[]): void { + private addJsWorldLandmarks(data: Uint8Array[]): void { for (const binaryProto of data) { const handWorldLandmarksProto = LandmarkList.deserializeBinary(binaryProto); @@ -322,7 +322,7 @@ export class HandLandmarker extends VisionTaskRunner { this.graphRunner.attachProtoVectorListener( WORLD_LANDMARKS_STREAM, (binaryProto, timestamp) => { - this.adddJsWorldLandmarks(binaryProto); + this.addJsWorldLandmarks(binaryProto); this.setLatestOutputTimestamp(timestamp); }); this.graphRunner.attachEmptyPacketListener( diff --git a/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts b/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts index 5df91a3d4..8f6531827 100644 --- a/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts +++ b/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts @@ -403,7 +403,7 @@ export class PoseLandmarker extends VisionTaskRunner { * Converts raw data into a world landmark, and adds it to our * worldLandmarks list. */ - private adddJsWorldLandmarks(data: Uint8Array[]): void { + private addJsWorldLandmarks(data: Uint8Array[]): void { this.worldLandmarks = []; for (const binaryProto of data) { const poseWorldLandmarksProto = @@ -452,7 +452,7 @@ export class PoseLandmarker extends VisionTaskRunner { this.graphRunner.attachProtoVectorListener( WORLD_LANDMARKS_STREAM, (binaryProto, timestamp) => { - this.adddJsWorldLandmarks(binaryProto); + this.addJsWorldLandmarks(binaryProto); this.setLatestOutputTimestamp(timestamp); }); this.graphRunner.attachEmptyPacketListener( From 31346255081a083be45b3e95f546bea6a214f38d Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 26 Sep 2023 15:08:36 -0700 Subject: [PATCH 13/13] No public description PiperOrigin-RevId: 568660415 --- WORKSPACE | 19 ++++++++++++++ third_party/BUILD | 2 +- third_party/com_github_glog_glog.diff | 15 ----------- .../com_github_glog_glog_windows_patch.diff | 26 +++++++++++++++++++ 4 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 third_party/com_github_glog_glog_windows_patch.diff diff --git a/WORKSPACE b/WORKSPACE index 7ca808505..4f7bb3b2e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -176,6 +176,25 @@ http_archive( ], ) +# 2023-06-05 +# This version of Glog is required for Windows support, but currently causes +# crashes on some Android devices. +http_archive( + name = "com_github_glog_glog_windows", + strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372", + sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb", + urls = [ + "https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip", + ], + patches = [ + "@//third_party:com_github_glog_glog.diff", + "@//third_party:com_github_glog_glog_windows_patch.diff", + ], + patch_args = [ + "-p1", + ], +) + # easyexif http_archive( name = "easyexif", diff --git a/third_party/BUILD b/third_party/BUILD index 229252087..d784fc3cc 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -45,7 +45,7 @@ cc_library( "@com_github_glog_glog//:glog", ], "//mediapipe:windows": [ - "@com_github_glog_glog//:glog", + "@com_github_glog_glog_windows//:glog", ], "//conditions:default": [ "@com_github_glog_glog//:glog", diff --git a/third_party/com_github_glog_glog.diff b/third_party/com_github_glog_glog.diff index 1e83001bd..bf08045b3 100644 --- a/third_party/com_github_glog_glog.diff +++ b/third_party/com_github_glog_glog.diff @@ -39,18 +39,3 @@ index 4028ccc..483e639 100644 if (append_newline) { // Fix the ostrstream back how it was before we screwed with it. // It's 99.44% certain that we don't need to worry about doing this. - -diff --git a/bazel/glog.bzl b/bazel/glog.bzl -index 4208d9e..3c86abf 100644 ---- a/bazel/glog.bzl -+++ b/bazel/glog.bzl -@@ -180,6 +180,9 @@ def glog_library(namespace = "google", with_gflags = 1, **kwargs): - }), - copts = - select({ -+ "@bazel_tools//src/conditions:windows": ["-std=c++17"], -+ "//conditions:default": [], -+ }) + select({ - "@bazel_tools//src/conditions:windows": common_copts + windows_only_copts, - "@bazel_tools//src/conditions:darwin": common_copts + linux_or_darwin_copts + darwin_only_copts, - "@bazel_tools//src/conditions:freebsd": common_copts + linux_or_darwin_copts + freebsd_only_copts, diff --git a/third_party/com_github_glog_glog_windows_patch.diff b/third_party/com_github_glog_glog_windows_patch.diff new file mode 100644 index 000000000..250be1110 --- /dev/null +++ b/third_party/com_github_glog_glog_windows_patch.diff @@ -0,0 +1,26 @@ +diff --git a/bazel/glog.bzl b/bazel/glog.bzl +index dacd934..d7b3d78 100644 +--- a/bazel/glog.bzl ++++ b/bazel/glog.bzl +@@ -53,7 +53,6 @@ def glog_library(namespace = "google", with_gflags = 1, **kwargs): + ) + + common_copts = [ +- "-std=c++14", + "-DGLOG_BAZEL_BUILD", + # Inject a C++ namespace. + "-DGOOGLE_NAMESPACE='%s'" % namespace, +@@ -145,7 +144,13 @@ def glog_library(namespace = "google", with_gflags = 1, **kwargs): + ], + }) + ++ c14_opts = ["-std=c++14"] ++ c17_opts = ["-std=c++17"] ++ + final_lib_copts = select({ ++ "@bazel_tools//src/conditions:windows": c17_opts, ++ "//conditions:default": c14_opts, ++ }) + select({ + "@bazel_tools//src/conditions:windows": common_copts + windows_only_copts, + "@bazel_tools//src/conditions:darwin": common_copts + linux_or_darwin_copts + darwin_only_copts, + "@bazel_tools//src/conditions:freebsd": common_copts + linux_or_darwin_copts + freebsd_only_copts,