diff --git a/mediapipe/tasks/web/vision/BUILD b/mediapipe/tasks/web/vision/BUILD index 19c795fd9..c86801955 100644 --- a/mediapipe/tasks/web/vision/BUILD +++ b/mediapipe/tasks/web/vision/BUILD @@ -19,6 +19,7 @@ mediapipe_files(srcs = [ VISION_LIBS = [ "//mediapipe/tasks/web/core:fileset_resolver", + "//mediapipe/tasks/web/vision/core:drawing_utils", "//mediapipe/tasks/web/vision/face_detector", "//mediapipe/tasks/web/vision/face_landmarker", "//mediapipe/tasks/web/vision/face_stylizer", diff --git a/mediapipe/tasks/web/vision/core/BUILD b/mediapipe/tasks/web/vision/core/BUILD index 8077f30d5..8f53dc2cb 100644 --- a/mediapipe/tasks/web/vision/core/BUILD +++ b/mediapipe/tasks/web/vision/core/BUILD @@ -29,6 +29,16 @@ mediapipe_ts_declaration( ], ) +mediapipe_ts_library( + name = "drawing_utils", + srcs = ["drawing_utils.ts"], + deps = [ + ":types", + "//mediapipe/tasks/web/components/containers:bounding_box", + "//mediapipe/tasks/web/components/containers:landmark", + ], +) + mediapipe_ts_library( name = "vision_task_runner", srcs = ["vision_task_runner.ts"], diff --git a/mediapipe/tasks/web/vision/core/drawing_utils.ts b/mediapipe/tasks/web/vision/core/drawing_utils.ts new file mode 100644 index 000000000..a4013a003 --- /dev/null +++ b/mediapipe/tasks/web/vision/core/drawing_utils.ts @@ -0,0 +1,218 @@ +/** + * Copyright 2023 The MediaPipe Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {BoundingBox} from '../../../../tasks/web/components/containers/bounding_box'; +import {NormalizedLandmark} from '../../../../tasks/web/components/containers/landmark'; +import {Connection} from '../../../../tasks/web/vision/core/types'; + +/** + * A user-defined callback to take input data and map it to a custom output + * value. + */ +export type Callback = (input: I) => O; + +/** Data that a user can use to specialize drawing options. */ +export declare interface LandmarkData { + index?: number; + from?: NormalizedLandmark; + to?: NormalizedLandmark; +} + +/** + * Options for customizing the drawing routines + */ +export declare interface DrawingOptions { + /** The color that is used to draw the shape. Defaults to white. */ + color?: string|CanvasGradient|CanvasPattern| + Callback; + /** + * The color that is used to fill the shape. Defaults to `.color` (or black + * if color is not set). + */ + fillColor?: string|CanvasGradient|CanvasPattern| + Callback; + /** The width of the line boundary of the shape. Defaults to 4. */ + lineWidth?: number|Callback; + /** The radius of location marker. Defaults to 6. */ + radius?: number|Callback; +} + +/** + * This will be merged with user supplied options. + */ +const DEFAULT_OPTIONS: DrawingOptions = { + color: 'white', + lineWidth: 4, + radius: 6 +}; + +/** Merges the user's options with the default options. */ +function addDefaultOptions(style?: DrawingOptions): DrawingOptions { + style = style || {}; + return { + ...DEFAULT_OPTIONS, + ...{fillColor: style.color}, + ...style, + }; +} + +/** + * Resolve the value from `value`. Invokes `value` with `data` if it is a + * function. + */ +function resolve(value: O|Callback, data: I): O { + return value instanceof Function ? value(data) : value; +} + +/** Helper class to visualize the result of a MediaPipe Vision task. */ +export class DrawingUtils { + /** + * Creates a new DrawingUtils class. + * + * @param ctx The canvas to render onto. + */ + constructor(private readonly ctx: CanvasRenderingContext2D) {} + + /** + * Restricts a number between two endpoints (order doesn't matter). + * + * @param x The number to clamp. + * @param x0 The first boundary. + * @param x1 The second boundary. + * @return The clamped value. + */ + static clamp(x: number, x0: number, x1: number): number { + const lo = Math.min(x0, x1); + const hi = Math.max(x0, x1); + return Math.max(lo, Math.min(hi, x)); + } + + /** + * Linearly interpolates a value between two points, clamping that value to + * the endpoints. + * + * @param x The number to interpolate. + * @param x0 The x coordinate of the start value. + * @param x1 The x coordinate of the end value. + * @param y0 The y coordinate of the start value. + * @param y1 The y coordinate of the end value. + * @return The interpolated value. + */ + static lerp(x: number, x0: number, x1: number, y0: number, y1: number): + number { + const out = + y0 * (1 - (x - x0) / (x1 - x0)) + y1 * (1 - (x1 - x) / (x1 - x0)); + return DrawingUtils.clamp(out, y0, y1); + } + + /** + * Draws circles onto the provided landmarks. + * + * @param landmarks The landmarks to draw. + * @param style The style to visualize the landmarks. + */ + drawLandmarks(landmarks?: NormalizedLandmark[], style?: DrawingOptions): + void { + if (!landmarks) { + return; + } + const ctx = this.ctx; + const options = addDefaultOptions(style); + ctx.save(); + const canvas = ctx.canvas; + let index = 0; + for (const landmark of landmarks) { + // All of our points are normalized, so we need to scale the unit canvas + // to match our actual canvas size. + ctx.fillStyle = resolve(options.fillColor!, {index, from: landmark}); + ctx.strokeStyle = resolve(options.color!, {index, from: landmark}); + ctx.lineWidth = resolve(options.lineWidth!, {index, from: landmark}); + + const circle = new Path2D(); + // Decrease the size of the arc to compensate for the scale() + circle.arc( + landmark.x * canvas.width, landmark.y * canvas.height, + resolve(options.radius!, {index, from: landmark}), 0, 2 * Math.PI); + ctx.fill(circle); + ctx.stroke(circle); + ++index; + } + ctx.restore(); + } + + /** + * Draws lines between landmarks (given a connection graph). + * + * @param landmarks The landmarks to draw. + * @param connections The connections array that contains the start and the + * end indices for the connections to draw. + * @param style The style to visualize the landmarks. + */ + drawConnectors( + landmarks?: NormalizedLandmark[], connections?: Connection[], + style?: DrawingOptions): void { + if (!landmarks || !connections) { + return; + } + const ctx = this.ctx; + const options = addDefaultOptions(style); + ctx.save(); + const canvas = ctx.canvas; + let index = 0; + for (const connection of connections) { + ctx.beginPath(); + const from = landmarks[connection.start]; + const to = landmarks[connection.end]; + if (from && to) { + ctx.strokeStyle = resolve(options.color!, {index, from, to}); + ctx.lineWidth = resolve(options.lineWidth!, {index, from, to}); + ctx.moveTo(from.x * canvas.width, from.y * canvas.height); + ctx.lineTo(to.x * canvas.width, to.y * canvas.height); + } + ++index; + ctx.stroke(); + } + ctx.restore(); + } + + /** + * Draws a bounding box. + * + * @param boundingBox The bounding box to draw. + * @param style The style to visualize the boundin box. + */ + drawBoundingBox(boundingBox: BoundingBox, style?: DrawingOptions): void { + const ctx = this.ctx; + const options = addDefaultOptions(style); + ctx.save(); + ctx.beginPath(); + ctx.lineWidth = resolve(options.lineWidth!, {}); + ctx.strokeStyle = resolve(options.color!, {}); + ctx.fillStyle = resolve(options.fillColor!, {}); + ctx.moveTo(boundingBox.originX, boundingBox.originY); + ctx.lineTo(boundingBox.originX + boundingBox.width, boundingBox.originY); + ctx.lineTo( + boundingBox.originX + boundingBox.width, + boundingBox.originY + boundingBox.height); + ctx.lineTo(boundingBox.originX, boundingBox.originY + boundingBox.height); + ctx.lineTo(boundingBox.originX, boundingBox.originY); + ctx.stroke(); + ctx.fill(); + ctx.restore(); + } +} + + diff --git a/mediapipe/tasks/web/vision/core/types.d.ts b/mediapipe/tasks/web/vision/core/types.d.ts index b48b5045d..5699126b9 100644 --- a/mediapipe/tasks/web/vision/core/types.d.ts +++ b/mediapipe/tasks/web/vision/core/types.d.ts @@ -52,3 +52,9 @@ export declare interface RegionOfInterest { /** The ROI in keypoint format. */ keypoint: NormalizedKeypoint; } + +/** A connection between two landmarks. */ +export declare interface Connection { + start: number; + end: number; +} diff --git a/mediapipe/tasks/web/vision/face_landmarker/BUILD b/mediapipe/tasks/web/vision/face_landmarker/BUILD index 0069ad510..19108be3a 100644 --- a/mediapipe/tasks/web/vision/face_landmarker/BUILD +++ b/mediapipe/tasks/web/vision/face_landmarker/BUILD @@ -40,10 +40,9 @@ mediapipe_ts_library( mediapipe_ts_library( name = "face_landmarks_connections", - srcs = [ - "face_landmarks_connections.ts", - ], + srcs = ["face_landmarks_connections.ts"], visibility = ["//visibility:public"], + deps = ["//mediapipe/tasks/web/vision/core:types"], ) mediapipe_ts_declaration( diff --git a/mediapipe/tasks/web/vision/face_landmarker/face_landmarks_connections.ts b/mediapipe/tasks/web/vision/face_landmarker/face_landmarks_connections.ts index b338c14bc..978324750 100644 --- a/mediapipe/tasks/web/vision/face_landmarker/face_landmarks_connections.ts +++ b/mediapipe/tasks/web/vision/face_landmarker/face_landmarks_connections.ts @@ -14,11 +14,7 @@ * limitations under the License. */ -/** A face landmark connection. */ -export interface Connection { - start: number; - end: number; -} +import {Connection} from '../../../../tasks/web/vision/core/types'; // tslint:disable:class-as-namespace Using for easier import by 3P users @@ -27,7 +23,7 @@ export interface Connection { * connections. */ export class FaceLandmarksConnections { - static FACE_LANDMARKS_LIPS = [ + static FACE_LANDMARKS_LIPS: Connection[] = [ {start: 61, end: 146}, {start: 146, end: 91}, {start: 91, end: 181}, {start: 181, end: 84}, {start: 84, end: 17}, {start: 17, end: 314}, {start: 314, end: 405}, {start: 405, end: 321}, {start: 321, end: 375}, @@ -44,7 +40,7 @@ export class FaceLandmarksConnections { {start: 415, end: 308} ]; - static FACE_LANDMARKS_LEFT_EYE = [ + static FACE_LANDMARKS_LEFT_EYE: Connection[] = [ {start: 263, end: 249}, {start: 249, end: 390}, {start: 390, end: 373}, {start: 373, end: 374}, {start: 374, end: 380}, {start: 380, end: 381}, {start: 381, end: 382}, {start: 382, end: 362}, {start: 263, end: 466}, @@ -53,18 +49,18 @@ export class FaceLandmarksConnections { {start: 398, end: 362} ]; - static FACE_LANDMARKS_LEFT_EYEBROW = [ + static FACE_LANDMARKS_LEFT_EYEBROW: Connection[] = [ {start: 276, end: 283}, {start: 283, end: 282}, {start: 282, end: 295}, {start: 295, end: 285}, {start: 300, end: 293}, {start: 293, end: 334}, {start: 334, end: 296}, {start: 296, end: 336} ]; - static FACE_LANDMARKS_LEFT_IRIS = [ + static FACE_LANDMARKS_LEFT_IRIS: Connection[] = [ {start: 474, end: 475}, {start: 475, end: 476}, {start: 476, end: 477}, {start: 477, end: 474} ]; - static FACE_LANDMARKS_RIGHT_EYE = [ + static FACE_LANDMARKS_RIGHT_EYE: Connection[] = [ {start: 33, end: 7}, {start: 7, end: 163}, {start: 163, end: 144}, {start: 144, end: 145}, {start: 145, end: 153}, {start: 153, end: 154}, {start: 154, end: 155}, {start: 155, end: 133}, {start: 33, end: 246}, @@ -73,18 +69,18 @@ export class FaceLandmarksConnections { {start: 173, end: 133} ]; - static FACE_LANDMARKS_RIGHT_EYEBROW = [ + static FACE_LANDMARKS_RIGHT_EYEBROW: Connection[] = [ {start: 46, end: 53}, {start: 53, end: 52}, {start: 52, end: 65}, {start: 65, end: 55}, {start: 70, end: 63}, {start: 63, end: 105}, {start: 105, end: 66}, {start: 66, end: 107} ]; - static FACE_LANDMARKS_RIGHT_IRIS = [ + static FACE_LANDMARKS_RIGHT_IRIS: Connection[] = [ {start: 469, end: 470}, {start: 470, end: 471}, {start: 471, end: 472}, {start: 472, end: 469} ]; - static FACE_LANDMARKS_FACE_OVAL = [ + static FACE_LANDMARKS_FACE_OVAL: Connection[] = [ {start: 10, end: 338}, {start: 338, end: 297}, {start: 297, end: 332}, {start: 332, end: 284}, {start: 284, end: 251}, {start: 251, end: 389}, {start: 389, end: 356}, {start: 356, end: 454}, {start: 454, end: 323}, @@ -99,7 +95,7 @@ export class FaceLandmarksConnections { {start: 103, end: 67}, {start: 67, end: 109}, {start: 109, end: 10} ]; - static FACE_LANDMARKS_CONTOURS = [ + static FACE_LANDMARKS_CONTOURS: Connection[] = [ ...FaceLandmarksConnections.FACE_LANDMARKS_LIPS, ...FaceLandmarksConnections.FACE_LANDMARKS_LEFT_EYE, ...FaceLandmarksConnections.FACE_LANDMARKS_LEFT_EYEBROW, @@ -108,7 +104,7 @@ export class FaceLandmarksConnections { ...FaceLandmarksConnections.FACE_LANDMARKS_FACE_OVAL ]; - static FACE_LANDMARKS_TESSELATION = [ + static FACE_LANDMARKS_TESSELATION: Connection[] = [ {start: 127, end: 34}, {start: 34, end: 139}, {start: 139, end: 127}, {start: 11, end: 0}, {start: 0, end: 37}, {start: 37, end: 11}, {start: 232, end: 231}, {start: 231, end: 120}, {start: 120, end: 232}, diff --git a/mediapipe/tasks/web/vision/index.ts b/mediapipe/tasks/web/vision/index.ts index a9d8afe5e..3b3757bbd 100644 --- a/mediapipe/tasks/web/vision/index.ts +++ b/mediapipe/tasks/web/vision/index.ts @@ -15,6 +15,7 @@ */ import {FilesetResolver as FilesetResolverImpl} from '../../../tasks/web/core/fileset_resolver'; +import {DrawingUtils as DrawingUtilsImpl} from '../../../tasks/web/vision/core/drawing_utils'; import {FaceDetector as FaceDetectorImpl} from '../../../tasks/web/vision/face_detector/face_detector'; import {FaceLandmarker as FaceLandmarkerImpl, FaceLandmarksConnections as FaceLandmarksConnectionsImpl} from '../../../tasks/web/vision/face_landmarker/face_landmarker'; import {FaceStylizer as FaceStylizerImpl} from '../../../tasks/web/vision/face_stylizer/face_stylizer'; @@ -28,6 +29,7 @@ import {ObjectDetector as ObjectDetectorImpl} from '../../../tasks/web/vision/ob // Declare the variables locally so that Rollup in OSS includes them explicitly // as exports. +const DrawingUtils = DrawingUtilsImpl; const FilesetResolver = FilesetResolverImpl; const FaceDetector = FaceDetectorImpl; const FaceLandmarker = FaceLandmarkerImpl; @@ -42,6 +44,7 @@ const InteractiveSegmenter = InteractiveSegmenterImpl; const ObjectDetector = ObjectDetectorImpl; export { + DrawingUtils, FilesetResolver, FaceDetector, FaceLandmarker, diff --git a/mediapipe/tasks/web/vision/types.ts b/mediapipe/tasks/web/vision/types.ts index f49161adf..3db579f59 100644 --- a/mediapipe/tasks/web/vision/types.ts +++ b/mediapipe/tasks/web/vision/types.ts @@ -15,6 +15,7 @@ */ export * from '../../../tasks/web/core/fileset_resolver'; +export * from '../../../tasks/web/vision/core/drawing_utils'; export * from '../../../tasks/web/vision/face_detector/face_detector'; export * from '../../../tasks/web/vision/face_landmarker/face_landmarker'; export * from '../../../tasks/web/vision/face_stylizer/face_stylizer';