Internal change

PiperOrigin-RevId: 523827005
This commit is contained in:
Sebastian Schmidt 2023-04-12 15:47:38 -07:00 committed by Copybara-Service
parent 0179f0c456
commit d0e8a9e09b
8 changed files with 252 additions and 18 deletions

View File

@ -19,6 +19,7 @@ mediapipe_files(srcs = [
VISION_LIBS = [
"//mediapipe/tasks/web/core:fileset_resolver",
"//mediapipe/tasks/web/vision/core:drawing_utils",
"//mediapipe/tasks/web/vision/face_detector",
"//mediapipe/tasks/web/vision/face_landmarker",
"//mediapipe/tasks/web/vision/face_stylizer",

View File

@ -29,6 +29,16 @@ mediapipe_ts_declaration(
],
)
mediapipe_ts_library(
name = "drawing_utils",
srcs = ["drawing_utils.ts"],
deps = [
":types",
"//mediapipe/tasks/web/components/containers:bounding_box",
"//mediapipe/tasks/web/components/containers:landmark",
],
)
mediapipe_ts_library(
name = "vision_task_runner",
srcs = ["vision_task_runner.ts"],

View File

@ -0,0 +1,218 @@
/**
* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {BoundingBox} from '../../../../tasks/web/components/containers/bounding_box';
import {NormalizedLandmark} from '../../../../tasks/web/components/containers/landmark';
import {Connection} from '../../../../tasks/web/vision/core/types';
/**
* A user-defined callback to take input data and map it to a custom output
* value.
*/
export type Callback<I, O> = (input: I) => O;
/** Data that a user can use to specialize drawing options. */
export declare interface LandmarkData {
index?: number;
from?: NormalizedLandmark;
to?: NormalizedLandmark;
}
/**
* Options for customizing the drawing routines
*/
export declare interface DrawingOptions {
/** The color that is used to draw the shape. Defaults to white. */
color?: string|CanvasGradient|CanvasPattern|
Callback<LandmarkData, string|CanvasGradient|CanvasPattern>;
/**
* The color that is used to fill the shape. Defaults to `.color` (or black
* if color is not set).
*/
fillColor?: string|CanvasGradient|CanvasPattern|
Callback<LandmarkData, string|CanvasGradient|CanvasPattern>;
/** The width of the line boundary of the shape. Defaults to 4. */
lineWidth?: number|Callback<LandmarkData, number>;
/** The radius of location marker. Defaults to 6. */
radius?: number|Callback<LandmarkData, number>;
}
/**
* This will be merged with user supplied options.
*/
const DEFAULT_OPTIONS: DrawingOptions = {
color: 'white',
lineWidth: 4,
radius: 6
};
/** Merges the user's options with the default options. */
function addDefaultOptions(style?: DrawingOptions): DrawingOptions {
style = style || {};
return {
...DEFAULT_OPTIONS,
...{fillColor: style.color},
...style,
};
}
/**
* Resolve the value from `value`. Invokes `value` with `data` if it is a
* function.
*/
function resolve<O, I>(value: O|Callback<I, O>, data: I): O {
return value instanceof Function ? value(data) : value;
}
/** Helper class to visualize the result of a MediaPipe Vision task. */
export class DrawingUtils {
/**
* Creates a new DrawingUtils class.
*
* @param ctx The canvas to render onto.
*/
constructor(private readonly ctx: CanvasRenderingContext2D) {}
/**
* Restricts a number between two endpoints (order doesn't matter).
*
* @param x The number to clamp.
* @param x0 The first boundary.
* @param x1 The second boundary.
* @return The clamped value.
*/
static clamp(x: number, x0: number, x1: number): number {
const lo = Math.min(x0, x1);
const hi = Math.max(x0, x1);
return Math.max(lo, Math.min(hi, x));
}
/**
* Linearly interpolates a value between two points, clamping that value to
* the endpoints.
*
* @param x The number to interpolate.
* @param x0 The x coordinate of the start value.
* @param x1 The x coordinate of the end value.
* @param y0 The y coordinate of the start value.
* @param y1 The y coordinate of the end value.
* @return The interpolated value.
*/
static lerp(x: number, x0: number, x1: number, y0: number, y1: number):
number {
const out =
y0 * (1 - (x - x0) / (x1 - x0)) + y1 * (1 - (x1 - x) / (x1 - x0));
return DrawingUtils.clamp(out, y0, y1);
}
/**
* Draws circles onto the provided landmarks.
*
* @param landmarks The landmarks to draw.
* @param style The style to visualize the landmarks.
*/
drawLandmarks(landmarks?: NormalizedLandmark[], style?: DrawingOptions):
void {
if (!landmarks) {
return;
}
const ctx = this.ctx;
const options = addDefaultOptions(style);
ctx.save();
const canvas = ctx.canvas;
let index = 0;
for (const landmark of landmarks) {
// All of our points are normalized, so we need to scale the unit canvas
// to match our actual canvas size.
ctx.fillStyle = resolve(options.fillColor!, {index, from: landmark});
ctx.strokeStyle = resolve(options.color!, {index, from: landmark});
ctx.lineWidth = resolve(options.lineWidth!, {index, from: landmark});
const circle = new Path2D();
// Decrease the size of the arc to compensate for the scale()
circle.arc(
landmark.x * canvas.width, landmark.y * canvas.height,
resolve(options.radius!, {index, from: landmark}), 0, 2 * Math.PI);
ctx.fill(circle);
ctx.stroke(circle);
++index;
}
ctx.restore();
}
/**
* Draws lines between landmarks (given a connection graph).
*
* @param landmarks The landmarks to draw.
* @param connections The connections array that contains the start and the
* end indices for the connections to draw.
* @param style The style to visualize the landmarks.
*/
drawConnectors(
landmarks?: NormalizedLandmark[], connections?: Connection[],
style?: DrawingOptions): void {
if (!landmarks || !connections) {
return;
}
const ctx = this.ctx;
const options = addDefaultOptions(style);
ctx.save();
const canvas = ctx.canvas;
let index = 0;
for (const connection of connections) {
ctx.beginPath();
const from = landmarks[connection.start];
const to = landmarks[connection.end];
if (from && to) {
ctx.strokeStyle = resolve(options.color!, {index, from, to});
ctx.lineWidth = resolve(options.lineWidth!, {index, from, to});
ctx.moveTo(from.x * canvas.width, from.y * canvas.height);
ctx.lineTo(to.x * canvas.width, to.y * canvas.height);
}
++index;
ctx.stroke();
}
ctx.restore();
}
/**
* Draws a bounding box.
*
* @param boundingBox The bounding box to draw.
* @param style The style to visualize the boundin box.
*/
drawBoundingBox(boundingBox: BoundingBox, style?: DrawingOptions): void {
const ctx = this.ctx;
const options = addDefaultOptions(style);
ctx.save();
ctx.beginPath();
ctx.lineWidth = resolve(options.lineWidth!, {});
ctx.strokeStyle = resolve(options.color!, {});
ctx.fillStyle = resolve(options.fillColor!, {});
ctx.moveTo(boundingBox.originX, boundingBox.originY);
ctx.lineTo(boundingBox.originX + boundingBox.width, boundingBox.originY);
ctx.lineTo(
boundingBox.originX + boundingBox.width,
boundingBox.originY + boundingBox.height);
ctx.lineTo(boundingBox.originX, boundingBox.originY + boundingBox.height);
ctx.lineTo(boundingBox.originX, boundingBox.originY);
ctx.stroke();
ctx.fill();
ctx.restore();
}
}

View File

@ -52,3 +52,9 @@ export declare interface RegionOfInterest {
/** The ROI in keypoint format. */
keypoint: NormalizedKeypoint;
}
/** A connection between two landmarks. */
export declare interface Connection {
start: number;
end: number;
}

View File

@ -40,10 +40,9 @@ mediapipe_ts_library(
mediapipe_ts_library(
name = "face_landmarks_connections",
srcs = [
"face_landmarks_connections.ts",
],
srcs = ["face_landmarks_connections.ts"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/tasks/web/vision/core:types"],
)
mediapipe_ts_declaration(

View File

@ -14,11 +14,7 @@
* limitations under the License.
*/
/** A face landmark connection. */
export interface Connection {
start: number;
end: number;
}
import {Connection} from '../../../../tasks/web/vision/core/types';
// tslint:disable:class-as-namespace Using for easier import by 3P users
@ -27,7 +23,7 @@ export interface Connection {
* connections.
*/
export class FaceLandmarksConnections {
static FACE_LANDMARKS_LIPS = [
static FACE_LANDMARKS_LIPS: Connection[] = [
{start: 61, end: 146}, {start: 146, end: 91}, {start: 91, end: 181},
{start: 181, end: 84}, {start: 84, end: 17}, {start: 17, end: 314},
{start: 314, end: 405}, {start: 405, end: 321}, {start: 321, end: 375},
@ -44,7 +40,7 @@ export class FaceLandmarksConnections {
{start: 415, end: 308}
];
static FACE_LANDMARKS_LEFT_EYE = [
static FACE_LANDMARKS_LEFT_EYE: Connection[] = [
{start: 263, end: 249}, {start: 249, end: 390}, {start: 390, end: 373},
{start: 373, end: 374}, {start: 374, end: 380}, {start: 380, end: 381},
{start: 381, end: 382}, {start: 382, end: 362}, {start: 263, end: 466},
@ -53,18 +49,18 @@ export class FaceLandmarksConnections {
{start: 398, end: 362}
];
static FACE_LANDMARKS_LEFT_EYEBROW = [
static FACE_LANDMARKS_LEFT_EYEBROW: Connection[] = [
{start: 276, end: 283}, {start: 283, end: 282}, {start: 282, end: 295},
{start: 295, end: 285}, {start: 300, end: 293}, {start: 293, end: 334},
{start: 334, end: 296}, {start: 296, end: 336}
];
static FACE_LANDMARKS_LEFT_IRIS = [
static FACE_LANDMARKS_LEFT_IRIS: Connection[] = [
{start: 474, end: 475}, {start: 475, end: 476}, {start: 476, end: 477},
{start: 477, end: 474}
];
static FACE_LANDMARKS_RIGHT_EYE = [
static FACE_LANDMARKS_RIGHT_EYE: Connection[] = [
{start: 33, end: 7}, {start: 7, end: 163}, {start: 163, end: 144},
{start: 144, end: 145}, {start: 145, end: 153}, {start: 153, end: 154},
{start: 154, end: 155}, {start: 155, end: 133}, {start: 33, end: 246},
@ -73,18 +69,18 @@ export class FaceLandmarksConnections {
{start: 173, end: 133}
];
static FACE_LANDMARKS_RIGHT_EYEBROW = [
static FACE_LANDMARKS_RIGHT_EYEBROW: Connection[] = [
{start: 46, end: 53}, {start: 53, end: 52}, {start: 52, end: 65},
{start: 65, end: 55}, {start: 70, end: 63}, {start: 63, end: 105},
{start: 105, end: 66}, {start: 66, end: 107}
];
static FACE_LANDMARKS_RIGHT_IRIS = [
static FACE_LANDMARKS_RIGHT_IRIS: Connection[] = [
{start: 469, end: 470}, {start: 470, end: 471}, {start: 471, end: 472},
{start: 472, end: 469}
];
static FACE_LANDMARKS_FACE_OVAL = [
static FACE_LANDMARKS_FACE_OVAL: Connection[] = [
{start: 10, end: 338}, {start: 338, end: 297}, {start: 297, end: 332},
{start: 332, end: 284}, {start: 284, end: 251}, {start: 251, end: 389},
{start: 389, end: 356}, {start: 356, end: 454}, {start: 454, end: 323},
@ -99,7 +95,7 @@ export class FaceLandmarksConnections {
{start: 103, end: 67}, {start: 67, end: 109}, {start: 109, end: 10}
];
static FACE_LANDMARKS_CONTOURS = [
static FACE_LANDMARKS_CONTOURS: Connection[] = [
...FaceLandmarksConnections.FACE_LANDMARKS_LIPS,
...FaceLandmarksConnections.FACE_LANDMARKS_LEFT_EYE,
...FaceLandmarksConnections.FACE_LANDMARKS_LEFT_EYEBROW,
@ -108,7 +104,7 @@ export class FaceLandmarksConnections {
...FaceLandmarksConnections.FACE_LANDMARKS_FACE_OVAL
];
static FACE_LANDMARKS_TESSELATION = [
static FACE_LANDMARKS_TESSELATION: Connection[] = [
{start: 127, end: 34}, {start: 34, end: 139}, {start: 139, end: 127},
{start: 11, end: 0}, {start: 0, end: 37}, {start: 37, end: 11},
{start: 232, end: 231}, {start: 231, end: 120}, {start: 120, end: 232},

View File

@ -15,6 +15,7 @@
*/
import {FilesetResolver as FilesetResolverImpl} from '../../../tasks/web/core/fileset_resolver';
import {DrawingUtils as DrawingUtilsImpl} from '../../../tasks/web/vision/core/drawing_utils';
import {FaceDetector as FaceDetectorImpl} from '../../../tasks/web/vision/face_detector/face_detector';
import {FaceLandmarker as FaceLandmarkerImpl, FaceLandmarksConnections as FaceLandmarksConnectionsImpl} from '../../../tasks/web/vision/face_landmarker/face_landmarker';
import {FaceStylizer as FaceStylizerImpl} from '../../../tasks/web/vision/face_stylizer/face_stylizer';
@ -28,6 +29,7 @@ import {ObjectDetector as ObjectDetectorImpl} from '../../../tasks/web/vision/ob
// Declare the variables locally so that Rollup in OSS includes them explicitly
// as exports.
const DrawingUtils = DrawingUtilsImpl;
const FilesetResolver = FilesetResolverImpl;
const FaceDetector = FaceDetectorImpl;
const FaceLandmarker = FaceLandmarkerImpl;
@ -42,6 +44,7 @@ const InteractiveSegmenter = InteractiveSegmenterImpl;
const ObjectDetector = ObjectDetectorImpl;
export {
DrawingUtils,
FilesetResolver,
FaceDetector,
FaceLandmarker,

View File

@ -15,6 +15,7 @@
*/
export * from '../../../tasks/web/core/fileset_resolver';
export * from '../../../tasks/web/vision/core/drawing_utils';
export * from '../../../tasks/web/vision/face_detector/face_detector';
export * from '../../../tasks/web/vision/face_landmarker/face_landmarker';
export * from '../../../tasks/web/vision/face_stylizer/face_stylizer';