cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update MP Tasks to observe timestamp bounds

Browse Source 
PiperOrigin-RevId: 507006313
This commit is contained in:
Sebastian Schmidt 2023-02-03 15:16:52 -08:00 committed by Copybara-Service
parent 01834a5cdf
commit 680cfcc99b
10 changed files with 64 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts
View File

@ -200,6 +200,10 @@ export class AudioClassifier extends AudioTaskRunner<AudioClassifierResult[]> {
this.addJsAudioClassificationResults(binaryProtos);
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(
TIMESTAMPED_CLASSIFICATIONS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);

7
mediapipe/tasks/web/audio/audio_embedder/audio_embedder.ts
View File

@ -203,6 +203,9 @@ export class AudioEmbedder extends AudioTaskRunner<AudioEmbedderResult[]> {
convertFromEmbeddingResultProto(embeddingResult));
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(EMBEDDINGS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachProtoVectorListener(
TIMESTAMPED_EMBEDDINGS_STREAM, (data, timestamp) => {
@ -214,6 +217,10 @@ export class AudioEmbedder extends AudioTaskRunner<AudioEmbedderResult[]> {
}
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(
TIMESTAMPED_EMBEDDINGS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);

4
mediapipe/tasks/web/text/text_classifier/text_classifier.ts
View File

@ -164,6 +164,10 @@ export class TextClassifier extends TaskRunner {
ClassificationResult.deserializeBinary(binaryProto));
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(
CLASSIFICATIONS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);

3
mediapipe/tasks/web/text/text_embedder/text_embedder.ts
View File

@ -182,6 +182,9 @@ export class TextEmbedder extends TaskRunner {
convertFromEmbeddingResultProto(embeddingResult);
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(EMBEDDINGS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);

17
mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts
View File

@ -363,11 +363,20 @@ export class GestureRecognizer extends VisionTaskRunner {
this.addJsLandmarks(binaryProto);
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(LANDMARKS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachProtoVectorListener(
WORLD_LANDMARKS_STREAM, (binaryProto, timestamp) => {
this.adddJsWorldLandmarks(binaryProto);
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(
WORLD_LANDMARKS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachProtoVectorListener(
HAND_GESTURES_STREAM, (binaryProto, timestamp) => {
// Gesture index is not used, because the final gesture result comes
@ -376,11 +385,19 @@ export class GestureRecognizer extends VisionTaskRunner {
...this.toJsCategories(binaryProto, /* populateIndex= */ false));
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(
HAND_GESTURES_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachProtoVectorListener(
HANDEDNESS_STREAM, (binaryProto, timestamp) => {
this.handednesses.push(...this.toJsCategories(binaryProto));
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(HANDEDNESS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);

14
mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts
View File

@ -317,16 +317,30 @@ export class HandLandmarker extends VisionTaskRunner {
this.addJsLandmarks(binaryProto);
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(
LANDMARKS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachProtoVectorListener(
WORLD_LANDMARKS_STREAM, (binaryProto, timestamp) => {
this.adddJsWorldLandmarks(binaryProto);
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(
WORLD_LANDMARKS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachProtoVectorListener(
HANDEDNESS_STREAM, (binaryProto, timestamp) => {
this.handednesses.push(...this.toJsCategories(binaryProto));
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(
HANDEDNESS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);

4
mediapipe/tasks/web/vision/image_classifier/image_classifier.ts
View File

@ -192,6 +192,10 @@ export class ImageClassifier extends VisionTaskRunner {
ClassificationResult.deserializeBinary(binaryProto));
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(
CLASSIFICATIONS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);

3
mediapipe/tasks/web/vision/image_embedder/image_embedder.ts
View File

@ -211,6 +211,9 @@ export class ImageEmbedder extends VisionTaskRunner {
this.addJsImageEmdedding(binaryProto);
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(EMBEDDINGS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);

4
mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts
View File

@ -291,6 +291,10 @@ export class ImageSegmenter extends VisionTaskRunner {
}
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(
GROUPED_SEGMENTATIONS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);

4
mediapipe/tasks/web/vision/object_detector/object_detector.ts
View File

@ -252,6 +252,10 @@ export class ObjectDetector extends VisionTaskRunner {
this.addJsObjectDetections(binaryProto);
this.setLatestOutputTimestamp(timestamp);
});
this.graphRunner.attachEmptyPacketListener(
DETECTIONS_STREAM, timestamp => {
this.setLatestOutputTimestamp(timestamp);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);