Internal change

PiperOrigin-RevId: 523827005

mediapipe/tasks/web/vision/BUILD

@@ -19,6 +19,7 @@ mediapipe_files(srcs = [
 
 VISION_LIBS = [
     "//mediapipe/tasks/web/core:fileset_resolver",
+    "//mediapipe/tasks/web/vision/core:drawing_utils",
     "//mediapipe/tasks/web/vision/face_detector",
     "//mediapipe/tasks/web/vision/face_landmarker",
     "//mediapipe/tasks/web/vision/face_stylizer",

mediapipe/tasks/web/vision/core/BUILD

@@ -29,6 +29,16 @@ mediapipe_ts_declaration(
     ],
 )
 
+mediapipe_ts_library(
+    name = "drawing_utils",
+    srcs = ["drawing_utils.ts"],
+    deps = [
+        ":types",
+        "//mediapipe/tasks/web/components/containers:bounding_box",
+        "//mediapipe/tasks/web/components/containers:landmark",
+    ],
+)
+
 mediapipe_ts_library(
     name = "vision_task_runner",
     srcs = ["vision_task_runner.ts"],

mediapipe/tasks/web/vision/core/drawing_utils.ts

@@ -0,0 +1,218 @@
+/**
+ * Copyright 2023 The MediaPipe Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {BoundingBox} from '../../../../tasks/web/components/containers/bounding_box';
+import {NormalizedLandmark} from '../../../../tasks/web/components/containers/landmark';
+import {Connection} from '../../../../tasks/web/vision/core/types';
+
+/**
+ * A user-defined callback to take input data and map it to a custom output
+ * value.
+ */
+export type Callback<I, O> = (input: I) => O;
+
+/** Data that a user can use to specialize drawing options. */
+export declare interface LandmarkData {
+  index?: number;
+  from?: NormalizedLandmark;
+  to?: NormalizedLandmark;
+}
+
+/**
+ * Options for customizing the drawing routines
+ */
+export declare interface DrawingOptions {
+  /** The color that is used to draw the shape. Defaults to white. */
+  color?: string|CanvasGradient|CanvasPattern|
+      Callback<LandmarkData, string|CanvasGradient|CanvasPattern>;
+  /**
+   * The color that is used to fill the shape. Defaults to `.color` (or black
+   * if color is not set).
+   */
+  fillColor?: string|CanvasGradient|CanvasPattern|
+      Callback<LandmarkData, string|CanvasGradient|CanvasPattern>;
+  /** The width of the line boundary of the shape. Defaults to 4. */
+  lineWidth?: number|Callback<LandmarkData, number>;
+  /** The radius of location marker. Defaults to 6. */
+  radius?: number|Callback<LandmarkData, number>;
+}
+
+/**
+ * This will be merged with user supplied options.
+ */
+const DEFAULT_OPTIONS: DrawingOptions = {
+  color: 'white',
+  lineWidth: 4,
+  radius: 6
+};
+
+/** Merges the user's options with the default options. */
+function addDefaultOptions(style?: DrawingOptions): DrawingOptions {
+  style = style || {};
+  return {
+    ...DEFAULT_OPTIONS,
+    ...{fillColor: style.color},
+    ...style,
+  };
+}
+
+/**
+ * Resolve the value from `value`. Invokes `value` with `data` if it is a
+ * function.
+ */
+function resolve<O, I>(value: O|Callback<I, O>, data: I): O {
+  return value instanceof Function ? value(data) : value;
+}
+
+/** Helper class to visualize the result of a MediaPipe Vision task. */
+export class DrawingUtils {
+  /**
+   * Creates a new DrawingUtils class.
+   *
+   * @param ctx The canvas to render onto.
+   */
+  constructor(private readonly ctx: CanvasRenderingContext2D) {}
+
+  /**
+   * Restricts a number between two endpoints (order doesn't matter).
+   *
+   * @param x The number to clamp.
+   * @param x0 The first boundary.
+   * @param x1 The second boundary.
+   * @return The clamped value.
+   */
+  static clamp(x: number, x0: number, x1: number): number {
+    const lo = Math.min(x0, x1);
+    const hi = Math.max(x0, x1);
+    return Math.max(lo, Math.min(hi, x));
+  }
+
+  /**
+   * Linearly interpolates a value between two points, clamping that value to
+   * the endpoints.
+   *
+   * @param x The number to interpolate.
+   * @param x0 The x coordinate of the start value.
+   * @param x1 The x coordinate of the end value.
+   * @param y0 The y coordinate of the start value.
+   * @param y1 The y coordinate of the end value.
+   * @return The interpolated value.
+   */
+  static lerp(x: number, x0: number, x1: number, y0: number, y1: number):
+      number {
+    const out =
+        y0 * (1 - (x - x0) / (x1 - x0)) + y1 * (1 - (x1 - x) / (x1 - x0));
+    return DrawingUtils.clamp(out, y0, y1);
+  }
+
+  /**
+   * Draws circles onto the provided landmarks.
+   *
+   * @param landmarks The landmarks to draw.
+   * @param style The style to visualize the landmarks.
+   */
+  drawLandmarks(landmarks?: NormalizedLandmark[], style?: DrawingOptions):
+      void {
+    if (!landmarks) {
+      return;
+    }
+    const ctx = this.ctx;
+    const options = addDefaultOptions(style);
+    ctx.save();
+    const canvas = ctx.canvas;
+    let index = 0;
+    for (const landmark of landmarks) {
+      // All of our points are normalized, so we need to scale the unit canvas
+      // to match our actual canvas size.
+      ctx.fillStyle = resolve(options.fillColor!, {index, from: landmark});
+      ctx.strokeStyle = resolve(options.color!, {index, from: landmark});
+      ctx.lineWidth = resolve(options.lineWidth!, {index, from: landmark});
+
+      const circle = new Path2D();
+      // Decrease the size of the arc to compensate for the scale()
+      circle.arc(
+          landmark.x * canvas.width, landmark.y * canvas.height,
+          resolve(options.radius!, {index, from: landmark}), 0, 2 * Math.PI);
+      ctx.fill(circle);
+      ctx.stroke(circle);
+      ++index;
+    }
+    ctx.restore();
+  }
+
+  /**
+   * Draws lines between landmarks (given a connection graph).
+   *
+   * @param landmarks The landmarks to draw.
+   * @param connections The connections array that contains the start and the
+   *     end indices for the connections to draw.
+   * @param style The style to visualize the landmarks.
+   */
+  drawConnectors(
+      landmarks?: NormalizedLandmark[], connections?: Connection[],
+      style?: DrawingOptions): void {
+    if (!landmarks || !connections) {
+      return;
+    }
+    const ctx = this.ctx;
+    const options = addDefaultOptions(style);
+    ctx.save();
+    const canvas = ctx.canvas;
+    let index = 0;
+    for (const connection of connections) {
+      ctx.beginPath();
+      const from = landmarks[connection.start];
+      const to = landmarks[connection.end];
+      if (from && to) {
+        ctx.strokeStyle = resolve(options.color!, {index, from, to});
+        ctx.lineWidth = resolve(options.lineWidth!, {index, from, to});
+        ctx.moveTo(from.x * canvas.width, from.y * canvas.height);
+        ctx.lineTo(to.x * canvas.width, to.y * canvas.height);
+      }
+      ++index;
+      ctx.stroke();
+    }
+    ctx.restore();
+  }
+
+  /**
+   * Draws a bounding box.
+   *
+   * @param boundingBox The bounding box to draw.
+   * @param style The style to visualize the boundin box.
+   */
+  drawBoundingBox(boundingBox: BoundingBox, style?: DrawingOptions): void {
+    const ctx = this.ctx;
+    const options = addDefaultOptions(style);
+    ctx.save();
+    ctx.beginPath();
+    ctx.lineWidth = resolve(options.lineWidth!, {});
+    ctx.strokeStyle = resolve(options.color!, {});
+    ctx.fillStyle = resolve(options.fillColor!, {});
+    ctx.moveTo(boundingBox.originX, boundingBox.originY);
+    ctx.lineTo(boundingBox.originX + boundingBox.width, boundingBox.originY);
+    ctx.lineTo(
+        boundingBox.originX + boundingBox.width,
+        boundingBox.originY + boundingBox.height);
+    ctx.lineTo(boundingBox.originX, boundingBox.originY + boundingBox.height);
+    ctx.lineTo(boundingBox.originX, boundingBox.originY);
+    ctx.stroke();
+    ctx.fill();
+    ctx.restore();
+  }
+}
+
+

mediapipe/tasks/web/vision/core/types.d.ts

@@ -52,3 +52,9 @@ export declare interface RegionOfInterest {
   /** The ROI in keypoint format. */
   keypoint: NormalizedKeypoint;
 }
+
+/** A connection between two landmarks. */
+export declare interface Connection {
+  start: number;
+  end: number;
+}

mediapipe/tasks/web/vision/face_landmarker/BUILD

@@ -40,10 +40,9 @@ mediapipe_ts_library(
 
 mediapipe_ts_library(
     name = "face_landmarks_connections",
-    srcs = [
-        "face_landmarks_connections.ts",
-    ],
+    srcs = ["face_landmarks_connections.ts"],
     visibility = ["//visibility:public"],
+    deps = ["//mediapipe/tasks/web/vision/core:types"],
 )
 
 mediapipe_ts_declaration(

mediapipe/tasks/web/vision/face_landmarker/face_landmarks_connections.ts

@@ -14,11 +14,7 @@
  * limitations under the License.
  */
 
-/** A face landmark connection. */
-export interface Connection {
-  start: number;
-  end: number;
-}
+import {Connection} from '../../../../tasks/web/vision/core/types';
 
 // tslint:disable:class-as-namespace Using for easier import by 3P users
 
@@ -27,7 +23,7 @@ export interface Connection {
  * connections.
  */
 export class FaceLandmarksConnections {
-  static FACE_LANDMARKS_LIPS = [
+  static FACE_LANDMARKS_LIPS: Connection[] = [
     {start: 61, end: 146},  {start: 146, end: 91},  {start: 91, end: 181},
     {start: 181, end: 84},  {start: 84, end: 17},   {start: 17, end: 314},
     {start: 314, end: 405}, {start: 405, end: 321}, {start: 321, end: 375},
@@ -44,7 +40,7 @@ export class FaceLandmarksConnections {
     {start: 415, end: 308}
   ];
 
-  static FACE_LANDMARKS_LEFT_EYE = [
+  static FACE_LANDMARKS_LEFT_EYE: Connection[] = [
     {start: 263, end: 249}, {start: 249, end: 390}, {start: 390, end: 373},
     {start: 373, end: 374}, {start: 374, end: 380}, {start: 380, end: 381},
     {start: 381, end: 382}, {start: 382, end: 362}, {start: 263, end: 466},
@@ -53,18 +49,18 @@ export class FaceLandmarksConnections {
     {start: 398, end: 362}
   ];
 
-  static FACE_LANDMARKS_LEFT_EYEBROW = [
+  static FACE_LANDMARKS_LEFT_EYEBROW: Connection[] = [
     {start: 276, end: 283}, {start: 283, end: 282}, {start: 282, end: 295},
     {start: 295, end: 285}, {start: 300, end: 293}, {start: 293, end: 334},
     {start: 334, end: 296}, {start: 296, end: 336}
   ];
 
-  static FACE_LANDMARKS_LEFT_IRIS = [
+  static FACE_LANDMARKS_LEFT_IRIS: Connection[] = [
     {start: 474, end: 475}, {start: 475, end: 476}, {start: 476, end: 477},
     {start: 477, end: 474}
   ];
 
-  static FACE_LANDMARKS_RIGHT_EYE = [
+  static FACE_LANDMARKS_RIGHT_EYE: Connection[] = [
     {start: 33, end: 7}, {start: 7, end: 163}, {start: 163, end: 144},
     {start: 144, end: 145}, {start: 145, end: 153}, {start: 153, end: 154},
     {start: 154, end: 155}, {start: 155, end: 133}, {start: 33, end: 246},
@@ -73,18 +69,18 @@ export class FaceLandmarksConnections {
     {start: 173, end: 133}
   ];
 
-  static FACE_LANDMARKS_RIGHT_EYEBROW = [
+  static FACE_LANDMARKS_RIGHT_EYEBROW: Connection[] = [
     {start: 46, end: 53}, {start: 53, end: 52}, {start: 52, end: 65},
     {start: 65, end: 55}, {start: 70, end: 63}, {start: 63, end: 105},
     {start: 105, end: 66}, {start: 66, end: 107}
   ];
 
-  static FACE_LANDMARKS_RIGHT_IRIS = [
+  static FACE_LANDMARKS_RIGHT_IRIS: Connection[] = [
     {start: 469, end: 470}, {start: 470, end: 471}, {start: 471, end: 472},
     {start: 472, end: 469}
   ];
 
-  static FACE_LANDMARKS_FACE_OVAL = [
+  static FACE_LANDMARKS_FACE_OVAL: Connection[] = [
     {start: 10, end: 338},  {start: 338, end: 297}, {start: 297, end: 332},
     {start: 332, end: 284}, {start: 284, end: 251}, {start: 251, end: 389},
     {start: 389, end: 356}, {start: 356, end: 454}, {start: 454, end: 323},
@@ -99,7 +95,7 @@ export class FaceLandmarksConnections {
     {start: 103, end: 67},  {start: 67, end: 109},  {start: 109, end: 10}
   ];
 
-  static FACE_LANDMARKS_CONTOURS = [
+  static FACE_LANDMARKS_CONTOURS: Connection[] = [
     ...FaceLandmarksConnections.FACE_LANDMARKS_LIPS,
     ...FaceLandmarksConnections.FACE_LANDMARKS_LEFT_EYE,
     ...FaceLandmarksConnections.FACE_LANDMARKS_LEFT_EYEBROW,
@@ -108,7 +104,7 @@ export class FaceLandmarksConnections {
     ...FaceLandmarksConnections.FACE_LANDMARKS_FACE_OVAL
   ];
 
-  static FACE_LANDMARKS_TESSELATION = [
+  static FACE_LANDMARKS_TESSELATION: Connection[] = [
     {start: 127, end: 34},  {start: 34, end: 139},  {start: 139, end: 127},
     {start: 11, end: 0},    {start: 0, end: 37},    {start: 37, end: 11},
     {start: 232, end: 231}, {start: 231, end: 120}, {start: 120, end: 232},

mediapipe/tasks/web/vision/index.ts

@@ -15,6 +15,7 @@
  */
 
 import {FilesetResolver as FilesetResolverImpl} from '../../../tasks/web/core/fileset_resolver';
+import {DrawingUtils as DrawingUtilsImpl} from '../../../tasks/web/vision/core/drawing_utils';
 import {FaceDetector as FaceDetectorImpl} from '../../../tasks/web/vision/face_detector/face_detector';
 import {FaceLandmarker as FaceLandmarkerImpl, FaceLandmarksConnections as FaceLandmarksConnectionsImpl} from '../../../tasks/web/vision/face_landmarker/face_landmarker';
 import {FaceStylizer as FaceStylizerImpl} from '../../../tasks/web/vision/face_stylizer/face_stylizer';
@@ -28,6 +29,7 @@ import {ObjectDetector as ObjectDetectorImpl} from '../../../tasks/web/vision/ob
 
 // Declare the variables locally so that Rollup in OSS includes them explicitly
 // as exports.
+const DrawingUtils = DrawingUtilsImpl;
 const FilesetResolver = FilesetResolverImpl;
 const FaceDetector = FaceDetectorImpl;
 const FaceLandmarker = FaceLandmarkerImpl;
@@ -42,6 +44,7 @@ const InteractiveSegmenter = InteractiveSegmenterImpl;
 const ObjectDetector = ObjectDetectorImpl;
 
 export {
+  DrawingUtils,
   FilesetResolver,
   FaceDetector,
   FaceLandmarker,

mediapipe/tasks/web/vision/types.ts

@@ -15,6 +15,7 @@
  */
 
 export * from '../../../tasks/web/core/fileset_resolver';
+export * from '../../../tasks/web/vision/core/drawing_utils';
 export * from '../../../tasks/web/vision/face_detector/face_detector';
 export * from '../../../tasks/web/vision/face_landmarker/face_landmarker';
 export * from '../../../tasks/web/vision/face_stylizer/face_stylizer';