Invoke the FaceStylizer callback even if no faces are detected

PiperOrigin-RevId: 527008261

mediapipe/tasks/web/vision/core/types.d.ts

@@ -25,16 +25,6 @@ import {NormalizedKeypoint} from '../../../../tasks/web/components/containers/ke
  */
 export type SegmentationMask = Uint8ClampedArray|Float32Array|WebGLTexture;
 
-/**
- * A callback that receives an `ImageData` object from a Vision task. The
- * lifetime of the underlying data is limited to the duration of the callback.
- * If asynchronous processing is needed, all data needs to be copied before the
- * callback returns.
- *
- * The `WebGLTexture` output type is reserved for future usage.
- */
-export type ImageCallback =
-    (image: ImageData|WebGLTexture, width: number, height: number) => void;
 
 /** A Region-Of-Interest (ROI) to represent a region within an image. */
 export declare interface RegionOfInterest {

mediapipe/tasks/web/vision/face_stylizer/face_stylizer.ts

@@ -20,7 +20,6 @@ import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/b
 import {FaceStylizerGraphOptions as FaceStylizerGraphOptionsProto} from '../../../../tasks/cc/vision/face_stylizer/proto/face_stylizer_graph_options_pb';
 import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
 import {ImageProcessingOptions} from '../../../../tasks/web/vision/core/image_processing_options';
-import {ImageCallback} from '../../../../tasks/web/vision/core/types';
 import {VisionGraphRunner, VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner';
 import {ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner';
 // Placeholder for internal dependency on trusted resource url
@@ -39,11 +38,20 @@ const FACE_STYLIZER_GRAPH =
 // The OSS JS API does not support the builder pattern.
 // tslint:disable:jspb-use-builder-pattern
 
-export {ImageCallback};
+/**
+ * A callback that receives an image from the face stylizer, or `null` if no
+ * face was detected. The lifetime of the underlying data is limited to the
+ * duration of the callback. If asynchronous processing is needed, all data
+ * needs to be copied before the callback returns.
+ *
+ * The `WebGLTexture` output type is reserved for future usage.
+ */
+export type FaceStylizerCallback =
+    (image: ImageData|WebGLTexture|null, width: number, height: number) => void;
 
 /** Performs face stylization on images. */
 export class FaceStylizer extends VisionTaskRunner {
-  private userCallback: ImageCallback = () => {};
+  private userCallback: FaceStylizerCallback = () => {};
   private readonly options: FaceStylizerGraphOptionsProto;
 
   /**
@@ -134,7 +142,7 @@ export class FaceStylizer extends VisionTaskRunner {
    *    lifetime of the returned data is only guaranteed for the duration of the
    *    callback.
    */
-  stylize(image: ImageSource, callback: ImageCallback): void;
+  stylize(image: ImageSource, callback: FaceStylizerCallback): void;
   /**
    * Performs face stylization on the provided single image. The method returns
    * synchronously once the callback returns. Only use this method when the
@@ -158,11 +166,12 @@ export class FaceStylizer extends VisionTaskRunner {
    */
   stylize(
       image: ImageSource, imageProcessingOptions: ImageProcessingOptions,
-      callback: ImageCallback): void;
+      callback: FaceStylizerCallback): void;
   stylize(
       image: ImageSource,
-      imageProcessingOptionsOrCallback: ImageProcessingOptions|ImageCallback,
-      callback?: ImageCallback): void {
+      imageProcessingOptionsOrCallback: ImageProcessingOptions|
+      FaceStylizerCallback,
+      callback?: FaceStylizerCallback): void {
     const imageProcessingOptions =
         typeof imageProcessingOptionsOrCallback !== 'function' ?
         imageProcessingOptionsOrCallback :
@@ -191,7 +200,7 @@ export class FaceStylizer extends VisionTaskRunner {
    */
   stylizeForVideo(
       videoFrame: ImageSource, timestamp: number,
-      callback: ImageCallback): void;
+      callback: FaceStylizerCallback): void;
   /**
    * Performs face stylization on the provided video frame. Only use this
    * method when the FaceStylizer is created with the video running mode.
@@ -219,12 +228,12 @@ export class FaceStylizer extends VisionTaskRunner {
    */
   stylizeForVideo(
       videoFrame: ImageSource, imageProcessingOptions: ImageProcessingOptions,
-      timestamp: number, callback: ImageCallback): void;
+      timestamp: number, callback: FaceStylizerCallback): void;
   stylizeForVideo(
       videoFrame: ImageSource,
       timestampOrImageProcessingOptions: number|ImageProcessingOptions,
-      timestampOrCallback: number|ImageCallback,
-      callback?: ImageCallback): void {
+      timestampOrCallback: number|FaceStylizerCallback,
+      callback?: FaceStylizerCallback): void {
     const imageProcessingOptions =
         typeof timestampOrImageProcessingOptions !== 'number' ?
         timestampOrImageProcessingOptions :
@@ -272,6 +281,7 @@ export class FaceStylizer extends VisionTaskRunner {
         });
     this.graphRunner.attachEmptyPacketListener(
         STYLIZED_IMAGE_STREAM, timestamp => {
+          this.userCallback(null, /* width= */ 0, /* height= */ 0);
           this.setLatestOutputTimestamp(timestamp);
         });
 

mediapipe/tasks/web/vision/face_stylizer/face_stylizer_test.ts

@@ -30,6 +30,7 @@ class FaceStylizerFake extends FaceStylizer implements MediapipeTasksFake {
 
   fakeWasmModule: SpyWasmModule;
   imageListener: ((images: WasmImage, timestamp: number) => void)|undefined;
+  emptyPacketListener: ((timestamp: number) => void)|undefined;
 
   constructor() {
     super(createSpyWasmModule(), /* glCanvas= */ null);
@@ -42,6 +43,12 @@ class FaceStylizerFake extends FaceStylizer implements MediapipeTasksFake {
               expect(stream).toEqual('stylized_image');
               this.imageListener = listener;
             });
+    this.attachListenerSpies[1] =
+        spyOn(this.graphRunner, 'attachEmptyPacketListener')
+            .and.callFake((stream, listener) => {
+              expect(stream).toEqual('stylized_image');
+              this.emptyPacketListener = listener;
+            });
     spyOn(this.graphRunner, 'setGraph').and.callFake(binaryGraph => {
       this.graph = CalculatorGraphConfig.deserializeBinary(binaryGraph);
     });
@@ -111,4 +118,21 @@ describe('FaceStylizer', () => {
       done();
     });
   });
+
+  it('invokes callback even when no faes are detected', (done) => {
+    // Pass the test data to our listener
+    faceStylizer.fakeWasmModule._waitUntilIdle.and.callFake(() => {
+      verifyListenersRegistered(faceStylizer);
+      faceStylizer.emptyPacketListener!(/* timestamp= */ 1337);
+    });
+
+    // Invoke the face stylizeer
+    faceStylizer.stylize({} as HTMLImageElement, (image, width, height) => {
+      expect(faceStylizer.fakeWasmModule._waitUntilIdle).toHaveBeenCalled();
+      expect(image).toBeNull();
+      expect(width).toEqual(0);
+      expect(height).toEqual(0);
+      done();
+    });
+  });
 });