TypeScript: adding VideoFrame typings support to video input

PiperOrigin-RevId: 592640146
This commit is contained in:
MediaPipe Team 2023-12-20 13:14:10 -08:00 committed by Copybara-Service
parent 1fa79195ec
commit 52c1d44561
3 changed files with 27 additions and 34 deletions

View File

@ -21,7 +21,7 @@ import {MPImage} from '../../../../tasks/web/vision/core/image';
import {ImageProcessingOptions} from '../../../../tasks/web/vision/core/image_processing_options'; import {ImageProcessingOptions} from '../../../../tasks/web/vision/core/image_processing_options';
import {MPImageShaderContext} from '../../../../tasks/web/vision/core/image_shader_context'; import {MPImageShaderContext} from '../../../../tasks/web/vision/core/image_shader_context';
import {MPMask} from '../../../../tasks/web/vision/core/mask'; import {MPMask} from '../../../../tasks/web/vision/core/mask';
import {GraphRunner, ImageSource, WasmMediaPipeConstructor} from '../../../../web/graph_runner/graph_runner'; import {getImageSourceSize, GraphRunner, ImageSource, WasmMediaPipeConstructor} from '../../../../web/graph_runner/graph_runner';
import {SupportImage, WasmImage} from '../../../../web/graph_runner/graph_runner_image_lib'; import {SupportImage, WasmImage} from '../../../../web/graph_runner/graph_runner_image_lib';
import {isWebKit} from '../../../../web/graph_runner/platform_utils'; import {isWebKit} from '../../../../web/graph_runner/platform_utils';
import {SupportModelResourcesGraphService} from '../../../../web/graph_runner/register_model_resources_graph_service'; import {SupportModelResourcesGraphService} from '../../../../web/graph_runner/register_model_resources_graph_service';
@ -134,22 +134,6 @@ export abstract class VisionTaskRunner extends TaskRunner {
this.process(imageFrame, imageProcessingOptions, timestamp); this.process(imageFrame, imageProcessingOptions, timestamp);
} }
private getImageSourceSize(imageSource: ImageSource): [number, number] {
if ((imageSource as HTMLVideoElement).videoWidth !== undefined) {
return [
(imageSource as HTMLVideoElement).videoWidth,
(imageSource as HTMLVideoElement).videoHeight
];
} else if ((imageSource as HTMLImageElement).naturalWidth !== undefined) {
return [
(imageSource as HTMLImageElement).naturalWidth,
(imageSource as HTMLImageElement).naturalHeight
];
} else {
return [imageSource.width, imageSource.height];
}
}
private convertToNormalizedRect( private convertToNormalizedRect(
imageSource: ImageSource, imageSource: ImageSource,
imageProcessingOptions?: ImageProcessingOptions): NormalizedRect { imageProcessingOptions?: ImageProcessingOptions): NormalizedRect {
@ -199,7 +183,7 @@ export abstract class VisionTaskRunner extends TaskRunner {
// uses this for cropping, // uses this for cropping,
// - then finally rotates this back. // - then finally rotates this back.
if (imageProcessingOptions?.rotationDegrees % 180 !== 0) { if (imageProcessingOptions?.rotationDegrees % 180 !== 0) {
const [imageWidth, imageHeight] = this.getImageSourceSize(imageSource); const [imageWidth, imageHeight] = getImageSourceSize(imageSource);
// tslint:disable:no-unnecessary-type-assertion // tslint:disable:no-unnecessary-type-assertion
const width = normalizedRect.getHeight()! * imageHeight / imageWidth; const width = normalizedRect.getHeight()! * imageHeight / imageWidth;
const height = normalizedRect.getWidth()! * imageWidth / imageHeight; const height = normalizedRect.getWidth()! * imageWidth / imageHeight;

View File

@ -52,6 +52,26 @@ declare global {
*/ */
declare function importScripts(...urls: Array<string|URL>): void; declare function importScripts(...urls: Array<string|URL>): void;
/**
* Detects image source size.
*/
export function getImageSourceSize(imageSource: ImageSource): [number, number] {
if ((imageSource as HTMLVideoElement).videoWidth !== undefined) {
const videoElement = imageSource as HTMLVideoElement;
return [videoElement.videoWidth, videoElement.videoHeight];
} else if ((imageSource as HTMLImageElement).naturalWidth !== undefined) {
// TODO: Ensure this works with SVG images
const imageElement = imageSource as HTMLImageElement;
return [imageElement.naturalWidth, imageElement.naturalHeight];
} else if ((imageSource as VideoFrame).displayWidth !== undefined) {
const videoFrame = imageSource as VideoFrame;
return [videoFrame.displayWidth, videoFrame.displayHeight];
} else {
const notVideoFrame = imageSource as Exclude<ImageSource, VideoFrame>;
return [notVideoFrame.width, notVideoFrame.height];
}
}
/** /**
* Simple class to run an arbitrary image-in/image-out MediaPipe graph (i.e. * Simple class to run an arbitrary image-in/image-out MediaPipe graph (i.e.
* as created by wasm_mediapipe_demo BUILD macro), and either render results * as created by wasm_mediapipe_demo BUILD macro), and either render results
@ -64,7 +84,7 @@ export class GraphRunner implements GraphRunnerApi {
// should be somewhat fixed when we create our .d.ts files. // should be somewhat fixed when we create our .d.ts files.
readonly wasmModule: WasmModule; readonly wasmModule: WasmModule;
readonly hasMultiStreamSupport: boolean; readonly hasMultiStreamSupport: boolean;
autoResizeCanvas: boolean = true; autoResizeCanvas = true;
audioPtr: number|null; audioPtr: number|null;
audioSize: number; audioSize: number;
@ -196,18 +216,7 @@ export class GraphRunner implements GraphRunnerApi {
gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false); gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, false);
} }
let width, height; const [width, height] = getImageSourceSize(imageSource);
if ((imageSource as HTMLVideoElement).videoWidth) {
width = (imageSource as HTMLVideoElement).videoWidth;
height = (imageSource as HTMLVideoElement).videoHeight;
} else if ((imageSource as HTMLImageElement).naturalWidth) {
// TODO: Ensure this works with SVG images
width = (imageSource as HTMLImageElement).naturalWidth;
height = (imageSource as HTMLImageElement).naturalHeight;
} else {
width = imageSource.width;
height = imageSource.height;
}
if (this.autoResizeCanvas && if (this.autoResizeCanvas &&
(width !== this.wasmModule.canvas.width || (width !== this.wasmModule.canvas.width ||
@ -295,7 +304,7 @@ export class GraphRunner implements GraphRunnerApi {
* format). * format).
* *
* Consumers must deserialize the binary representation themselves as this * Consumers must deserialize the binary representation themselves as this
* avoids addding a direct dependency on the Protobuf JSPB target in the graph * avoids adding a direct dependency on the Protobuf JSPB target in the graph
* library. * library.
*/ */
getCalculatorGraphConfig( getCalculatorGraphConfig(

View File

@ -26,8 +26,8 @@ export {
/** /**
* Valid types of image sources which we can run our GraphRunner over. * Valid types of image sources which we can run our GraphRunner over.
*/ */
export type ImageSource = export type ImageSource = HTMLCanvasElement|HTMLVideoElement|HTMLImageElement|
HTMLCanvasElement|HTMLVideoElement|HTMLImageElement|ImageData|ImageBitmap; ImageData|ImageBitmap|VideoFrame;
/** /**
* Simple interface for a class to run an arbitrary MediaPipe graph on web, and * Simple interface for a class to run an arbitrary MediaPipe graph on web, and