From 680cfcc99bba29095e36835e0eb0e16bdc79f103 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 3 Feb 2023 15:16:52 -0800 Subject: [PATCH] Update MP Tasks to observe timestamp bounds PiperOrigin-RevId: 507006313 --- .../audio/audio_classifier/audio_classifier.ts | 4 ++++ .../web/audio/audio_embedder/audio_embedder.ts | 7 +++++++ .../web/text/text_classifier/text_classifier.ts | 4 ++++ .../web/text/text_embedder/text_embedder.ts | 3 +++ .../gesture_recognizer/gesture_recognizer.ts | 17 +++++++++++++++++ .../vision/hand_landmarker/hand_landmarker.ts | 14 ++++++++++++++ .../vision/image_classifier/image_classifier.ts | 4 ++++ .../web/vision/image_embedder/image_embedder.ts | 3 +++ .../vision/image_segmenter/image_segmenter.ts | 4 ++++ .../vision/object_detector/object_detector.ts | 4 ++++ 10 files changed, 64 insertions(+) diff --git a/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts b/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts index e26ead6a9..4e14ae437 100644 --- a/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts +++ b/mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts @@ -200,6 +200,10 @@ export class AudioClassifier extends AudioTaskRunner { this.addJsAudioClassificationResults(binaryProtos); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener( + TIMESTAMPED_CLASSIFICATIONS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); const binaryGraph = graphConfig.serializeBinary(); this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true); diff --git a/mediapipe/tasks/web/audio/audio_embedder/audio_embedder.ts b/mediapipe/tasks/web/audio/audio_embedder/audio_embedder.ts index 7411f95ef..7d8c7a5b4 100644 --- a/mediapipe/tasks/web/audio/audio_embedder/audio_embedder.ts +++ b/mediapipe/tasks/web/audio/audio_embedder/audio_embedder.ts @@ -203,6 +203,9 @@ export class AudioEmbedder extends AudioTaskRunner { convertFromEmbeddingResultProto(embeddingResult)); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener(EMBEDDINGS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); this.graphRunner.attachProtoVectorListener( TIMESTAMPED_EMBEDDINGS_STREAM, (data, timestamp) => { @@ -214,6 +217,10 @@ export class AudioEmbedder extends AudioTaskRunner { } this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener( + TIMESTAMPED_EMBEDDINGS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); const binaryGraph = graphConfig.serializeBinary(); this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true); diff --git a/mediapipe/tasks/web/text/text_classifier/text_classifier.ts b/mediapipe/tasks/web/text/text_classifier/text_classifier.ts index ff314cfc3..b28817613 100644 --- a/mediapipe/tasks/web/text/text_classifier/text_classifier.ts +++ b/mediapipe/tasks/web/text/text_classifier/text_classifier.ts @@ -164,6 +164,10 @@ export class TextClassifier extends TaskRunner { ClassificationResult.deserializeBinary(binaryProto)); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener( + CLASSIFICATIONS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); const binaryGraph = graphConfig.serializeBinary(); this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true); diff --git a/mediapipe/tasks/web/text/text_embedder/text_embedder.ts b/mediapipe/tasks/web/text/text_embedder/text_embedder.ts index daa1d24ed..1034de033 100644 --- a/mediapipe/tasks/web/text/text_embedder/text_embedder.ts +++ b/mediapipe/tasks/web/text/text_embedder/text_embedder.ts @@ -182,6 +182,9 @@ export class TextEmbedder extends TaskRunner { convertFromEmbeddingResultProto(embeddingResult); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener(EMBEDDINGS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); const binaryGraph = graphConfig.serializeBinary(); this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true); diff --git a/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts b/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts index beea263ce..3271191ca 100644 --- a/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts +++ b/mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts @@ -363,11 +363,20 @@ export class GestureRecognizer extends VisionTaskRunner { this.addJsLandmarks(binaryProto); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener(LANDMARKS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); + this.graphRunner.attachProtoVectorListener( WORLD_LANDMARKS_STREAM, (binaryProto, timestamp) => { this.adddJsWorldLandmarks(binaryProto); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener( + WORLD_LANDMARKS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); + this.graphRunner.attachProtoVectorListener( HAND_GESTURES_STREAM, (binaryProto, timestamp) => { // Gesture index is not used, because the final gesture result comes @@ -376,11 +385,19 @@ export class GestureRecognizer extends VisionTaskRunner { ...this.toJsCategories(binaryProto, /* populateIndex= */ false)); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener( + HAND_GESTURES_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); + this.graphRunner.attachProtoVectorListener( HANDEDNESS_STREAM, (binaryProto, timestamp) => { this.handednesses.push(...this.toJsCategories(binaryProto)); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener(HANDEDNESS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); const binaryGraph = graphConfig.serializeBinary(); this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true); diff --git a/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts b/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts index cd0459372..32684faff 100644 --- a/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts +++ b/mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts @@ -317,16 +317,30 @@ export class HandLandmarker extends VisionTaskRunner { this.addJsLandmarks(binaryProto); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener( + LANDMARKS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); + this.graphRunner.attachProtoVectorListener( WORLD_LANDMARKS_STREAM, (binaryProto, timestamp) => { this.adddJsWorldLandmarks(binaryProto); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener( + WORLD_LANDMARKS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); + this.graphRunner.attachProtoVectorListener( HANDEDNESS_STREAM, (binaryProto, timestamp) => { this.handednesses.push(...this.toJsCategories(binaryProto)); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener( + HANDEDNESS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); const binaryGraph = graphConfig.serializeBinary(); this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true); diff --git a/mediapipe/tasks/web/vision/image_classifier/image_classifier.ts b/mediapipe/tasks/web/vision/image_classifier/image_classifier.ts index 071513b19..70ce93aff 100644 --- a/mediapipe/tasks/web/vision/image_classifier/image_classifier.ts +++ b/mediapipe/tasks/web/vision/image_classifier/image_classifier.ts @@ -192,6 +192,10 @@ export class ImageClassifier extends VisionTaskRunner { ClassificationResult.deserializeBinary(binaryProto)); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener( + CLASSIFICATIONS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); const binaryGraph = graphConfig.serializeBinary(); this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true); diff --git a/mediapipe/tasks/web/vision/image_embedder/image_embedder.ts b/mediapipe/tasks/web/vision/image_embedder/image_embedder.ts index fdeb92f3f..0c7d8866b 100644 --- a/mediapipe/tasks/web/vision/image_embedder/image_embedder.ts +++ b/mediapipe/tasks/web/vision/image_embedder/image_embedder.ts @@ -211,6 +211,9 @@ export class ImageEmbedder extends VisionTaskRunner { this.addJsImageEmdedding(binaryProto); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener(EMBEDDINGS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); const binaryGraph = graphConfig.serializeBinary(); this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true); diff --git a/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts b/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts index 4f81977eb..3a2e9f2af 100644 --- a/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts +++ b/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts @@ -291,6 +291,10 @@ export class ImageSegmenter extends VisionTaskRunner { } this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener( + GROUPED_SEGMENTATIONS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); const binaryGraph = graphConfig.serializeBinary(); this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true); diff --git a/mediapipe/tasks/web/vision/object_detector/object_detector.ts b/mediapipe/tasks/web/vision/object_detector/object_detector.ts index 5b581432d..a65b64e91 100644 --- a/mediapipe/tasks/web/vision/object_detector/object_detector.ts +++ b/mediapipe/tasks/web/vision/object_detector/object_detector.ts @@ -252,6 +252,10 @@ export class ObjectDetector extends VisionTaskRunner { this.addJsObjectDetections(binaryProto); this.setLatestOutputTimestamp(timestamp); }); + this.graphRunner.attachEmptyPacketListener( + DETECTIONS_STREAM, timestamp => { + this.setLatestOutputTimestamp(timestamp); + }); const binaryGraph = graphConfig.serializeBinary(); this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);