From 9e30b00685fa8c39c9a2ee20ec47b119ce6ac2b0 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 25 Apr 2023 10:32:49 -0700 Subject: [PATCH] Invoke the FaceStylizer callback even if no faces are detected PiperOrigin-RevId: 527008261 --- mediapipe/tasks/web/vision/core/types.d.ts | 10 ------ .../web/vision/face_stylizer/face_stylizer.ts | 32 ++++++++++++------- .../face_stylizer/face_stylizer_test.ts | 24 ++++++++++++++ 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/mediapipe/tasks/web/vision/core/types.d.ts b/mediapipe/tasks/web/vision/core/types.d.ts index c43956f87..1cc2e36fd 100644 --- a/mediapipe/tasks/web/vision/core/types.d.ts +++ b/mediapipe/tasks/web/vision/core/types.d.ts @@ -25,16 +25,6 @@ import {NormalizedKeypoint} from '../../../../tasks/web/components/containers/ke */ export type SegmentationMask = Uint8ClampedArray|Float32Array|WebGLTexture; -/** - * A callback that receives an `ImageData` object from a Vision task. The - * lifetime of the underlying data is limited to the duration of the callback. - * If asynchronous processing is needed, all data needs to be copied before the - * callback returns. - * - * The `WebGLTexture` output type is reserved for future usage. - */ -export type ImageCallback = - (image: ImageData|WebGLTexture, width: number, height: number) => void; /** A Region-Of-Interest (ROI) to represent a region within an image. */ export declare interface RegionOfInterest { diff --git a/mediapipe/tasks/web/vision/face_stylizer/face_stylizer.ts b/mediapipe/tasks/web/vision/face_stylizer/face_stylizer.ts index c7c04a40c..13558e235 100644 --- a/mediapipe/tasks/web/vision/face_stylizer/face_stylizer.ts +++ b/mediapipe/tasks/web/vision/face_stylizer/face_stylizer.ts @@ -20,7 +20,6 @@ import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/b import {FaceStylizerGraphOptions as FaceStylizerGraphOptionsProto} from '../../../../tasks/cc/vision/face_stylizer/proto/face_stylizer_graph_options_pb'; import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset'; import {ImageProcessingOptions} from '../../../../tasks/web/vision/core/image_processing_options'; -import {ImageCallback} from '../../../../tasks/web/vision/core/types'; import {VisionGraphRunner, VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner'; import {ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner'; // Placeholder for internal dependency on trusted resource url @@ -39,11 +38,20 @@ const FACE_STYLIZER_GRAPH = // The OSS JS API does not support the builder pattern. // tslint:disable:jspb-use-builder-pattern -export {ImageCallback}; +/** + * A callback that receives an image from the face stylizer, or `null` if no + * face was detected. The lifetime of the underlying data is limited to the + * duration of the callback. If asynchronous processing is needed, all data + * needs to be copied before the callback returns. + * + * The `WebGLTexture` output type is reserved for future usage. + */ +export type FaceStylizerCallback = + (image: ImageData|WebGLTexture|null, width: number, height: number) => void; /** Performs face stylization on images. */ export class FaceStylizer extends VisionTaskRunner { - private userCallback: ImageCallback = () => {}; + private userCallback: FaceStylizerCallback = () => {}; private readonly options: FaceStylizerGraphOptionsProto; /** @@ -134,7 +142,7 @@ export class FaceStylizer extends VisionTaskRunner { * lifetime of the returned data is only guaranteed for the duration of the * callback. */ - stylize(image: ImageSource, callback: ImageCallback): void; + stylize(image: ImageSource, callback: FaceStylizerCallback): void; /** * Performs face stylization on the provided single image. The method returns * synchronously once the callback returns. Only use this method when the @@ -158,11 +166,12 @@ export class FaceStylizer extends VisionTaskRunner { */ stylize( image: ImageSource, imageProcessingOptions: ImageProcessingOptions, - callback: ImageCallback): void; + callback: FaceStylizerCallback): void; stylize( image: ImageSource, - imageProcessingOptionsOrCallback: ImageProcessingOptions|ImageCallback, - callback?: ImageCallback): void { + imageProcessingOptionsOrCallback: ImageProcessingOptions| + FaceStylizerCallback, + callback?: FaceStylizerCallback): void { const imageProcessingOptions = typeof imageProcessingOptionsOrCallback !== 'function' ? imageProcessingOptionsOrCallback : @@ -191,7 +200,7 @@ export class FaceStylizer extends VisionTaskRunner { */ stylizeForVideo( videoFrame: ImageSource, timestamp: number, - callback: ImageCallback): void; + callback: FaceStylizerCallback): void; /** * Performs face stylization on the provided video frame. Only use this * method when the FaceStylizer is created with the video running mode. @@ -219,12 +228,12 @@ export class FaceStylizer extends VisionTaskRunner { */ stylizeForVideo( videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions, - timestamp: number, callback: ImageCallback): void; + timestamp: number, callback: FaceStylizerCallback): void; stylizeForVideo( videoFrame: ImageSource, timestampOrImageProcessingOptions: number|ImageProcessingOptions, - timestampOrCallback: number|ImageCallback, - callback?: ImageCallback): void { + timestampOrCallback: number|FaceStylizerCallback, + callback?: FaceStylizerCallback): void { const imageProcessingOptions = typeof timestampOrImageProcessingOptions !== 'number' ? timestampOrImageProcessingOptions : @@ -272,6 +281,7 @@ export class FaceStylizer extends VisionTaskRunner { }); this.graphRunner.attachEmptyPacketListener( STYLIZED_IMAGE_STREAM, timestamp => { + this.userCallback(null, /* width= */ 0, /* height= */ 0); this.setLatestOutputTimestamp(timestamp); }); diff --git a/mediapipe/tasks/web/vision/face_stylizer/face_stylizer_test.ts b/mediapipe/tasks/web/vision/face_stylizer/face_stylizer_test.ts index b042ebac5..3ef35728d 100644 --- a/mediapipe/tasks/web/vision/face_stylizer/face_stylizer_test.ts +++ b/mediapipe/tasks/web/vision/face_stylizer/face_stylizer_test.ts @@ -30,6 +30,7 @@ class FaceStylizerFake extends FaceStylizer implements MediapipeTasksFake { fakeWasmModule: SpyWasmModule; imageListener: ((images: WasmImage, timestamp: number) => void)|undefined; + emptyPacketListener: ((timestamp: number) => void)|undefined; constructor() { super(createSpyWasmModule(), /* glCanvas= */ null); @@ -42,6 +43,12 @@ class FaceStylizerFake extends FaceStylizer implements MediapipeTasksFake { expect(stream).toEqual('stylized_image'); this.imageListener = listener; }); + this.attachListenerSpies[1] = + spyOn(this.graphRunner, 'attachEmptyPacketListener') + .and.callFake((stream, listener) => { + expect(stream).toEqual('stylized_image'); + this.emptyPacketListener = listener; + }); spyOn(this.graphRunner, 'setGraph').and.callFake(binaryGraph => { this.graph = CalculatorGraphConfig.deserializeBinary(binaryGraph); }); @@ -111,4 +118,21 @@ describe('FaceStylizer', () => { done(); }); }); + + it('invokes callback even when no faes are detected', (done) => { + // Pass the test data to our listener + faceStylizer.fakeWasmModule._waitUntilIdle.and.callFake(() => { + verifyListenersRegistered(faceStylizer); + faceStylizer.emptyPacketListener!(/* timestamp= */ 1337); + }); + + // Invoke the face stylizeer + faceStylizer.stylize({} as HTMLImageElement, (image, width, height) => { + expect(faceStylizer.fakeWasmModule._waitUntilIdle).toHaveBeenCalled(); + expect(image).toBeNull(); + expect(width).toEqual(0); + expect(height).toEqual(0); + done(); + }); + }); });