From 463cbb60eea6af436bbec6d13fceae0f65cdbe64 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 5 Jan 2023 07:55:57 -0800 Subject: [PATCH 1/3] Fix RGBA vs RGB selection when creating GLTexture. PiperOrigin-RevId: 499877590 --- .../calculators/tensor/image_to_tensor_converter_gl_buffer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc index a551e7f8d..eb1726aac 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc @@ -285,7 +285,7 @@ class GlProcessor : public ImageToTensorConverter { auto source_texture = gl_helper_.CreateSourceTexture(input); tflite::gpu::gl::GlTexture input_texture( GL_TEXTURE_2D, source_texture.name(), - input_num_channels == 4 ? GL_RGB : GL_RGBA, + input_num_channels == 4 ? GL_RGBA : GL_RGB, source_texture.width() * source_texture.height() * input_num_channels * sizeof(uint8_t), /*layer=*/0, From 35293d88bcb35b87162fbbb40b76226677f98d3f Mon Sep 17 00:00:00 2001 From: Nikolay Chirkov Date: Thu, 5 Jan 2023 08:54:25 -0800 Subject: [PATCH 2/3] Tensor: move into tensor sub-directory. PiperOrigin-RevId: 499896489 --- mediapipe/framework/formats/BUILD | 2 +- mediapipe/framework/formats/tensor.h | 2 +- mediapipe/framework/formats/tensor/BUILD | 24 +++++++++++++++++++ .../{tensor_internal.h => tensor/internal.h} | 0 .../framework/formats/tensor_ahwb_gpu_test.cc | 2 +- 5 files changed, 27 insertions(+), 3 deletions(-) create mode 100644 mediapipe/framework/formats/tensor/BUILD rename mediapipe/framework/formats/{tensor_internal.h => tensor/internal.h} (100%) diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index cce7e5bd0..371f23ed1 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -430,7 +430,7 @@ cc_library( ], hdrs = [ "tensor.h", - "tensor_internal.h", + "//mediapipe/framework/formats/tensor:internal.h", ], copts = select({ "//mediapipe:apple": [ diff --git a/mediapipe/framework/formats/tensor.h b/mediapipe/framework/formats/tensor.h index 0f19bb5ee..4a952ae09 100644 --- a/mediapipe/framework/formats/tensor.h +++ b/mediapipe/framework/formats/tensor.h @@ -26,7 +26,7 @@ #include "absl/container/flat_hash_map.h" #include "absl/synchronization/mutex.h" -#include "mediapipe/framework/formats/tensor_internal.h" +#include "mediapipe/framework/formats/tensor/internal.h" #include "mediapipe/framework/port.h" #if MEDIAPIPE_METAL_ENABLED diff --git a/mediapipe/framework/formats/tensor/BUILD b/mediapipe/framework/formats/tensor/BUILD new file mode 100644 index 000000000..c634b0dda --- /dev/null +++ b/mediapipe/framework/formats/tensor/BUILD @@ -0,0 +1,24 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package( + default_visibility = ["//visibility:public"], + features = ["-layering_check"], +) + +licenses(["notice"]) + +exports_files([ + "internal.h", +]) diff --git a/mediapipe/framework/formats/tensor_internal.h b/mediapipe/framework/formats/tensor/internal.h similarity index 100% rename from mediapipe/framework/formats/tensor_internal.h rename to mediapipe/framework/formats/tensor/internal.h diff --git a/mediapipe/framework/formats/tensor_ahwb_gpu_test.cc b/mediapipe/framework/formats/tensor_ahwb_gpu_test.cc index ff78d1f88..b06bd3ef2 100644 --- a/mediapipe/framework/formats/tensor_ahwb_gpu_test.cc +++ b/mediapipe/framework/formats/tensor_ahwb_gpu_test.cc @@ -7,7 +7,7 @@ #include #include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/formats/tensor_data_types.h" +#include "mediapipe/framework/formats/tensor/views/data_types.h" #include "mediapipe/gpu/gpu_test_base.h" #include "mediapipe/gpu/shader_util.h" #include "tensorflow/lite/delegates/gpu/gl/gl_call.h" From 81a46bb31a5da15a0ddd7123b92499c6ca14dc86 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 5 Jan 2023 09:12:06 -0800 Subject: [PATCH 3/3] Internal change PiperOrigin-RevId: 499902323 --- mediapipe/web/graph_runner/graph_runner.ts | 73 ++++++++++++++-------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/mediapipe/web/graph_runner/graph_runner.ts b/mediapipe/web/graph_runner/graph_runner.ts index ef866bc91..644d74918 100644 --- a/mediapipe/web/graph_runner/graph_runner.ts +++ b/mediapipe/web/graph_runner/graph_runner.ts @@ -73,10 +73,11 @@ export declare interface WasmModule { // Wasm Module output listener entrypoints. Also built as part of // gl_graph_runner_internal_multi_input. - simpleListeners?: {[outputStreamName: string]: (data: unknown) => void}; + simpleListeners?: + {[outputStreamName: string]: (data: unknown, timestamp: number) => void}; vectorListeners?: { [outputStreamName: string]: ( - data: unknown, index: number, length: number) => void + data: unknown, index: number, length: number, timestamp: number) => void }; _attachBoolListener: (streamNamePtr: number) => void; _attachBoolVectorListener: (streamNamePtr: number) => void; @@ -418,10 +419,12 @@ export class GraphRunner { * Ensures existence of the simple listeners table and registers the callback. * Intended for internal usage. */ - setListener(outputStreamName: string, callbackFcn: (data: T) => void) { + setListener( + outputStreamName: string, + callbackFcn: (data: T, timestamp: number) => void) { this.wasmModule.simpleListeners = this.wasmModule.simpleListeners || {}; this.wasmModule.simpleListeners[outputStreamName] = - callbackFcn as (data: unknown) => void; + callbackFcn as (data: unknown, timestamp: number) => void; } /** @@ -429,11 +432,12 @@ export class GraphRunner { * Intended for internal usage. */ setVectorListener( - outputStreamName: string, callbackFcn: (data: T[]) => void) { + outputStreamName: string, + callbackFcn: (data: T[], timestamp: number) => void) { let buffer: T[] = []; this.wasmModule.vectorListeners = this.wasmModule.vectorListeners || {}; this.wasmModule.vectorListeners[outputStreamName] = - (data: unknown, index: number, length: number) => { + (data: unknown, index: number, length: number, timestamp: number) => { // The Wasm listener gets invoked once for each element. Once we // receive all elements, we invoke the registered callback with the // full array. @@ -442,7 +446,7 @@ export class GraphRunner { // Invoke the user callback directly, as the Wasm layer may clean up // the underlying data elements once we leave the scope of the // listener. - callbackFcn(buffer); + callbackFcn(buffer, timestamp); buffer = []; } }; @@ -740,7 +744,8 @@ export class GraphRunner { * should not perform overly complicated (or any async) behavior. */ attachBoolListener( - outputStreamName: string, callbackFcn: (data: boolean) => void): void { + outputStreamName: string, + callbackFcn: (data: boolean, timestamp: number) => void): void { // Set up our TS listener to receive any packets for this stream. this.setListener(outputStreamName, callbackFcn); @@ -760,7 +765,8 @@ export class GraphRunner { * should not perform overly complicated (or any async) behavior. */ attachBoolVectorListener( - outputStreamName: string, callbackFcn: (data: boolean[]) => void): void { + outputStreamName: string, + callbackFcn: (data: boolean[], timestamp: number) => void): void { // Set up our TS listener to receive any packets for this stream. this.setVectorListener(outputStreamName, callbackFcn); @@ -780,7 +786,8 @@ export class GraphRunner { * should not perform overly complicated (or any async) behavior. */ attachIntListener( - outputStreamName: string, callbackFcn: (data: number) => void): void { + outputStreamName: string, + callbackFcn: (data: number, timestamp: number) => void): void { // Set up our TS listener to receive any packets for this stream. this.setListener(outputStreamName, callbackFcn); @@ -800,7 +807,8 @@ export class GraphRunner { * should not perform overly complicated (or any async) behavior. */ attachIntVectorListener( - outputStreamName: string, callbackFcn: (data: number[]) => void): void { + outputStreamName: string, + callbackFcn: (data: number[], timestamp: number) => void): void { // Set up our TS listener to receive any packets for this stream. this.setVectorListener(outputStreamName, callbackFcn); @@ -820,7 +828,8 @@ export class GraphRunner { * should not perform overly complicated (or any async) behavior. */ attachDoubleListener( - outputStreamName: string, callbackFcn: (data: number) => void): void { + outputStreamName: string, + callbackFcn: (data: number, timestamp: number) => void): void { // Set up our TS listener to receive any packets for this stream. this.setListener(outputStreamName, callbackFcn); @@ -840,7 +849,8 @@ export class GraphRunner { * should not perform overly complicated (or any async) behavior. */ attachDoubleVectorListener( - outputStreamName: string, callbackFcn: (data: number[]) => void): void { + outputStreamName: string, + callbackFcn: (data: number[], timestamp: number) => void): void { // Set up our TS listener to receive any packets for this stream. this.setVectorListener(outputStreamName, callbackFcn); @@ -860,7 +870,8 @@ export class GraphRunner { * should not perform overly complicated (or any async) behavior. */ attachFloatListener( - outputStreamName: string, callbackFcn: (data: number) => void): void { + outputStreamName: string, + callbackFcn: (data: number, timestamp: number) => void): void { // Set up our TS listener to receive any packets for this stream. this.setListener(outputStreamName, callbackFcn); @@ -880,7 +891,8 @@ export class GraphRunner { * should not perform overly complicated (or any async) behavior. */ attachFloatVectorListener( - outputStreamName: string, callbackFcn: (data: number[]) => void): void { + outputStreamName: string, + callbackFcn: (data: number[], timestamp: number) => void): void { // Set up our TS listener to receive any packets for this stream. this.setVectorListener(outputStreamName, callbackFcn); @@ -900,7 +912,8 @@ export class GraphRunner { * should not perform overly complicated (or any async) behavior. */ attachStringListener( - outputStreamName: string, callbackFcn: (data: string) => void): void { + outputStreamName: string, + callbackFcn: (data: string, timestamp: number) => void): void { // Set up our TS listener to receive any packets for this stream. this.setListener(outputStreamName, callbackFcn); @@ -920,7 +933,8 @@ export class GraphRunner { * should not perform overly complicated (or any async) behavior. */ attachStringVectorListener( - outputStreamName: string, callbackFcn: (data: string[]) => void): void { + outputStreamName: string, + callbackFcn: (data: string[], timestamp: number) => void): void { // Set up our TS listener to receive any packets for this stream. this.setVectorListener(outputStreamName, callbackFcn); @@ -950,7 +964,8 @@ export class GraphRunner { * with it). */ attachProtoListener( - outputStreamName: string, callbackFcn: (data: Uint8Array) => void, + outputStreamName: string, + callbackFcn: (data: Uint8Array, timestamp: number) => void, makeDeepCopy?: boolean): void { // Set up our TS listener to receive any packets for this stream. this.setListener(outputStreamName, callbackFcn); @@ -984,7 +999,8 @@ export class GraphRunner { * with it). */ attachProtoVectorListener( - outputStreamName: string, callbackFcn: (data: Uint8Array[]) => void, + outputStreamName: string, + callbackFcn: (data: Uint8Array[], timestamp: number) => void, makeDeepCopy?: boolean): void { // Set up our TS listener to receive any packets for this stream. this.setVectorListener(outputStreamName, callbackFcn); @@ -1017,8 +1033,10 @@ export class GraphRunner { * up automatically by JS garbage collection whenever the user is finished * with it). */ - attachAudioListener(outputStreamName: string, - callbackFcn: (data: Float32Array) => void, makeDeepCopy?: boolean): void { + attachAudioListener( + outputStreamName: string, + callbackFcn: (data: Float32Array, timestamp: number) => void, + makeDeepCopy?: boolean): void { if (!this.wasmModule._attachAudioListener) { console.warn( 'Attempting to use attachAudioListener without support for ' + @@ -1027,12 +1045,13 @@ export class GraphRunner { // Set up our TS listener to receive any packets for this stream, and // additionally reformat our Uint8Array into a Float32Array for the user. - this.setListener(outputStreamName, (data: Uint8Array) => { - // Should be very fast - const floatArray = - new Float32Array(data.buffer, data.byteOffset, data.length / 4); - callbackFcn(floatArray); - }); + this.setListener( + outputStreamName, (data: Uint8Array, timestamp: number) => { + // Should be very fast + const floatArray = + new Float32Array(data.buffer, data.byteOffset, data.length / 4); + callbackFcn(floatArray, timestamp); + }); // Tell our graph to listen for string packets on this stream. this.wrapStringPtr(outputStreamName, (outputStreamNamePtr: number) => {