Hand Landmarker Web API
PiperOrigin-RevId: 488434079
This commit is contained in:
parent
34daba4747
commit
b00236e86e
1
mediapipe/tasks/testdata/vision/BUILD
vendored
1
mediapipe/tasks/testdata/vision/BUILD
vendored
|
@ -40,6 +40,7 @@ mediapipe_files(srcs = [
|
|||
"fist.jpg",
|
||||
"hand_landmark_full.tflite",
|
||||
"hand_landmark_lite.tflite",
|
||||
"hand_landmarker.task",
|
||||
"left_hands.jpg",
|
||||
"left_hands_rotated.jpg",
|
||||
"mobilenet_v1_0.25_192_quantized_1_default_1.tflite",
|
||||
|
|
|
@ -9,6 +9,7 @@ mediapipe_ts_library(
|
|||
srcs = ["index.ts"],
|
||||
deps = [
|
||||
"//mediapipe/tasks/web/vision/gesture_recognizer",
|
||||
"//mediapipe/tasks/web/vision/hand_landmarker",
|
||||
"//mediapipe/tasks/web/vision/image_classifier",
|
||||
"//mediapipe/tasks/web/vision/object_detector",
|
||||
],
|
||||
|
|
35
mediapipe/tasks/web/vision/hand_landmarker/BUILD
Normal file
35
mediapipe/tasks/web/vision/hand_landmarker/BUILD
Normal file
|
@ -0,0 +1,35 @@
|
|||
# This contains the MediaPipe Hand Landmarker Task.
|
||||
#
|
||||
# This task takes video frames and outputs synchronized frames along with
|
||||
# the detection results for one or more hand categories, using Hand Landmarker.
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
|
||||
|
||||
package(default_visibility = ["//mediapipe/tasks:internal"])
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
mediapipe_ts_library(
|
||||
name = "hand_landmarker",
|
||||
srcs = [
|
||||
"hand_landmarker.ts",
|
||||
"hand_landmarker_options.ts",
|
||||
"hand_landmarker_result.ts",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_jspb_proto",
|
||||
"//mediapipe/framework:calculator_options_jspb_proto",
|
||||
"//mediapipe/framework/formats:classification_jspb_proto",
|
||||
"//mediapipe/framework/formats:landmark_jspb_proto",
|
||||
"//mediapipe/framework/formats:rect_jspb_proto",
|
||||
"//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_jspb_proto",
|
||||
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_jspb_proto",
|
||||
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_jspb_proto",
|
||||
"//mediapipe/tasks/web/components/containers:category",
|
||||
"//mediapipe/tasks/web/components/containers:landmark",
|
||||
"//mediapipe/tasks/web/components/processors:base_options",
|
||||
"//mediapipe/tasks/web/core",
|
||||
"//mediapipe/tasks/web/core:task_runner",
|
||||
"//mediapipe/web/graph_runner:wasm_mediapipe_lib_ts",
|
||||
],
|
||||
)
|
319
mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts
Normal file
319
mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts
Normal file
|
@ -0,0 +1,319 @@
|
|||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {CalculatorGraphConfig} from '../../../../framework/calculator_pb';
|
||||
import {CalculatorOptions} from '../../../../framework/calculator_options_pb';
|
||||
import {ClassificationList} from '../../../../framework/formats/classification_pb';
|
||||
import {LandmarkList, NormalizedLandmarkList} from '../../../../framework/formats/landmark_pb';
|
||||
import {NormalizedRect} from '../../../../framework/formats/rect_pb';
|
||||
import {HandDetectorGraphOptions} from '../../../../tasks/cc/vision/hand_detector/proto/hand_detector_graph_options_pb';
|
||||
import {HandLandmarkerGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarker_graph_options_pb';
|
||||
import {HandLandmarksDetectorGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options_pb';
|
||||
import {Category} from '../../../../tasks/web/components/containers/category';
|
||||
import {Landmark} from '../../../../tasks/web/components/containers/landmark';
|
||||
import {convertBaseOptionsToProto} from '../../../../tasks/web/components/processors/base_options';
|
||||
import {TaskRunner} from '../../../../tasks/web/core/task_runner';
|
||||
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
|
||||
import {createMediaPipeLib, FileLocator, ImageSource, WasmModule} from '../../../../web/graph_runner/wasm_mediapipe_lib';
|
||||
// Placeholder for internal dependency on trusted resource url
|
||||
|
||||
import {HandLandmarkerOptions} from './hand_landmarker_options';
|
||||
import {HandLandmarkerResult} from './hand_landmarker_result';
|
||||
|
||||
export {ImageSource};
|
||||
|
||||
// The OSS JS API does not support the builder pattern.
|
||||
// tslint:disable:jspb-use-builder-pattern
|
||||
|
||||
const IMAGE_STREAM = 'image_in';
|
||||
const NORM_RECT_STREAM = 'norm_rect';
|
||||
const LANDMARKS_STREAM = 'hand_landmarks';
|
||||
const WORLD_LANDMARKS_STREAM = 'world_hand_landmarks';
|
||||
const HANDEDNESS_STREAM = 'handedness';
|
||||
const HAND_LANDMARKER_GRAPH =
|
||||
'mediapipe.tasks.vision.hand_landmarker.HandLandmarkerGraph';
|
||||
|
||||
const DEFAULT_NUM_HANDS = 1;
|
||||
const DEFAULT_SCORE_THRESHOLD = 0.5;
|
||||
const DEFAULT_CATEGORY_INDEX = -1;
|
||||
const FULL_IMAGE_RECT = new NormalizedRect();
|
||||
FULL_IMAGE_RECT.setXCenter(0.5);
|
||||
FULL_IMAGE_RECT.setYCenter(0.5);
|
||||
FULL_IMAGE_RECT.setWidth(1);
|
||||
FULL_IMAGE_RECT.setHeight(1);
|
||||
|
||||
/** Performs hand landmarks detection on images. */
|
||||
export class HandLandmarker extends TaskRunner {
|
||||
private landmarks: Landmark[][] = [];
|
||||
private worldLandmarks: Landmark[][] = [];
|
||||
private handednesses: Category[][] = [];
|
||||
|
||||
private readonly options: HandLandmarkerGraphOptions;
|
||||
private readonly handLandmarksDetectorGraphOptions:
|
||||
HandLandmarksDetectorGraphOptions;
|
||||
private readonly handDetectorGraphOptions: HandDetectorGraphOptions;
|
||||
|
||||
/**
|
||||
* Initializes the Wasm runtime and creates a new `HandLandmarker` from the
|
||||
* provided options.
|
||||
* @param wasmLoaderOptions A configuration object that provides the location
|
||||
* of the Wasm binary and its loader.
|
||||
* @param handLandmarkerOptions The options for the HandLandmarker.
|
||||
* Note that either a path to the model asset or a model buffer needs to
|
||||
* be provided (via `baseOptions`).
|
||||
*/
|
||||
static async createFromOptions(
|
||||
wasmLoaderOptions: WasmLoaderOptions,
|
||||
handLandmarkerOptions: HandLandmarkerOptions): Promise<HandLandmarker> {
|
||||
// Create a file locator based on the loader options
|
||||
const fileLocator: FileLocator = {
|
||||
locateFile() {
|
||||
// The only file we load via this mechanism is the Wasm binary
|
||||
return wasmLoaderOptions.wasmBinaryPath.toString();
|
||||
}
|
||||
};
|
||||
|
||||
const landmarker = await createMediaPipeLib(
|
||||
HandLandmarker, wasmLoaderOptions.wasmLoaderPath,
|
||||
/* assetLoaderScript= */ undefined,
|
||||
/* glCanvas= */ undefined, fileLocator);
|
||||
await landmarker.setOptions(handLandmarkerOptions);
|
||||
return landmarker;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the Wasm runtime and creates a new `HandLandmarker` based on
|
||||
* the provided model asset buffer.
|
||||
* @param wasmLoaderOptions A configuration object that provides the location
|
||||
* of the Wasm binary and its loader.
|
||||
* @param modelAssetBuffer A binary representation of the model.
|
||||
*/
|
||||
static createFromModelBuffer(
|
||||
wasmLoaderOptions: WasmLoaderOptions,
|
||||
modelAssetBuffer: Uint8Array): Promise<HandLandmarker> {
|
||||
return HandLandmarker.createFromOptions(
|
||||
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the Wasm runtime and creates a new `HandLandmarker` based on
|
||||
* the path to the model asset.
|
||||
* @param wasmLoaderOptions A configuration object that provides the location
|
||||
* of the Wasm binary and its loader.
|
||||
* @param modelAssetPath The path to the model asset.
|
||||
*/
|
||||
static async createFromModelPath(
|
||||
wasmLoaderOptions: WasmLoaderOptions,
|
||||
modelAssetPath: string): Promise<HandLandmarker> {
|
||||
const response = await fetch(modelAssetPath.toString());
|
||||
const graphData = await response.arrayBuffer();
|
||||
return HandLandmarker.createFromModelBuffer(
|
||||
wasmLoaderOptions, new Uint8Array(graphData));
|
||||
}
|
||||
|
||||
constructor(wasmModule: WasmModule) {
|
||||
super(wasmModule);
|
||||
|
||||
this.options = new HandLandmarkerGraphOptions();
|
||||
this.handLandmarksDetectorGraphOptions =
|
||||
new HandLandmarksDetectorGraphOptions();
|
||||
this.options.setHandLandmarksDetectorGraphOptions(
|
||||
this.handLandmarksDetectorGraphOptions);
|
||||
this.handDetectorGraphOptions = new HandDetectorGraphOptions();
|
||||
this.options.setHandDetectorGraphOptions(this.handDetectorGraphOptions);
|
||||
|
||||
this.initDefaults();
|
||||
|
||||
// Disables the automatic render-to-screen code, which allows for pure
|
||||
// CPU processing.
|
||||
this.setAutoRenderToScreen(false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets new options for this `HandLandmarker`.
|
||||
*
|
||||
* Calling `setOptions()` with a subset of options only affects those options.
|
||||
* You can reset an option back to its default value by explicitly setting it
|
||||
* to `undefined`.
|
||||
*
|
||||
* @param options The options for the hand landmarker.
|
||||
*/
|
||||
async setOptions(options: HandLandmarkerOptions): Promise<void> {
|
||||
if (options.baseOptions) {
|
||||
const baseOptionsProto = await convertBaseOptionsToProto(
|
||||
options.baseOptions, this.options.getBaseOptions());
|
||||
this.options.setBaseOptions(baseOptionsProto);
|
||||
}
|
||||
|
||||
// Configure hand detector options.
|
||||
if ('numHands' in options) {
|
||||
this.handDetectorGraphOptions.setNumHands(
|
||||
options.numHands ?? DEFAULT_NUM_HANDS);
|
||||
}
|
||||
if ('minHandDetectionConfidence' in options) {
|
||||
this.handDetectorGraphOptions.setMinDetectionConfidence(
|
||||
options.minHandDetectionConfidence ?? DEFAULT_SCORE_THRESHOLD);
|
||||
}
|
||||
|
||||
// Configure hand landmark detector options.
|
||||
if ('minTrackingConfidence' in options) {
|
||||
this.options.setMinTrackingConfidence(
|
||||
options.minTrackingConfidence ?? DEFAULT_SCORE_THRESHOLD);
|
||||
}
|
||||
if ('minHandPresenceConfidence' in options) {
|
||||
this.handLandmarksDetectorGraphOptions.setMinDetectionConfidence(
|
||||
options.minHandPresenceConfidence ?? DEFAULT_SCORE_THRESHOLD);
|
||||
}
|
||||
|
||||
this.refreshGraph();
|
||||
}
|
||||
|
||||
/**
|
||||
* Performs hand landmarks detection on the provided single image and waits
|
||||
* synchronously for the response.
|
||||
* @param imageSource An image source to process.
|
||||
* @param timestamp The timestamp of the current frame, in ms. If not
|
||||
* provided, defaults to `performance.now()`.
|
||||
* @return The detected hand landmarks.
|
||||
*/
|
||||
detect(imageSource: ImageSource, timestamp: number = performance.now()):
|
||||
HandLandmarkerResult {
|
||||
this.landmarks = [];
|
||||
this.worldLandmarks = [];
|
||||
this.handednesses = [];
|
||||
|
||||
this.addGpuBufferAsImageToStream(imageSource, IMAGE_STREAM, timestamp);
|
||||
this.addProtoToStream(
|
||||
FULL_IMAGE_RECT.serializeBinary(), 'mediapipe.NormalizedRect',
|
||||
NORM_RECT_STREAM, timestamp);
|
||||
this.finishProcessing();
|
||||
|
||||
return {
|
||||
landmarks: this.landmarks,
|
||||
worldLandmarks: this.worldLandmarks,
|
||||
handednesses: this.handednesses
|
||||
};
|
||||
}
|
||||
|
||||
/** Sets the default values for the graph. */
|
||||
private initDefaults(): void {
|
||||
this.handDetectorGraphOptions.setNumHands(DEFAULT_NUM_HANDS);
|
||||
this.handDetectorGraphOptions.setMinDetectionConfidence(
|
||||
DEFAULT_SCORE_THRESHOLD);
|
||||
this.handLandmarksDetectorGraphOptions.setMinDetectionConfidence(
|
||||
DEFAULT_SCORE_THRESHOLD);
|
||||
this.options.setMinTrackingConfidence(DEFAULT_SCORE_THRESHOLD);
|
||||
}
|
||||
|
||||
/** Converts the proto data to a Category[][] structure. */
|
||||
private toJsCategories(data: Uint8Array[]): Category[][] {
|
||||
const result: Category[][] = [];
|
||||
for (const binaryProto of data) {
|
||||
const inputList = ClassificationList.deserializeBinary(binaryProto);
|
||||
const outputList: Category[] = [];
|
||||
for (const classification of inputList.getClassificationList()) {
|
||||
outputList.push({
|
||||
score: classification.getScore() ?? 0,
|
||||
index: classification.getIndex() ?? DEFAULT_CATEGORY_INDEX,
|
||||
categoryName: classification.getLabel() ?? '',
|
||||
displayName: classification.getDisplayName() ?? '',
|
||||
});
|
||||
}
|
||||
result.push(outputList);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Converts raw data into a landmark, and adds it to our landmarks list. */
|
||||
private addJsLandmarks(data: Uint8Array[]): void {
|
||||
for (const binaryProto of data) {
|
||||
const handLandmarksProto =
|
||||
NormalizedLandmarkList.deserializeBinary(binaryProto);
|
||||
const landmarks: Landmark[] = [];
|
||||
for (const handLandmarkProto of handLandmarksProto.getLandmarkList()) {
|
||||
landmarks.push({
|
||||
x: handLandmarkProto.getX() ?? 0,
|
||||
y: handLandmarkProto.getY() ?? 0,
|
||||
z: handLandmarkProto.getZ() ?? 0,
|
||||
normalized: true
|
||||
});
|
||||
}
|
||||
this.landmarks.push(landmarks);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts raw data into a landmark, and adds it to our worldLandmarks
|
||||
* list.
|
||||
*/
|
||||
private adddJsWorldLandmarks(data: Uint8Array[]): void {
|
||||
for (const binaryProto of data) {
|
||||
const handWorldLandmarksProto =
|
||||
LandmarkList.deserializeBinary(binaryProto);
|
||||
const worldLandmarks: Landmark[] = [];
|
||||
for (const handWorldLandmarkProto of
|
||||
handWorldLandmarksProto.getLandmarkList()) {
|
||||
worldLandmarks.push({
|
||||
x: handWorldLandmarkProto.getX() ?? 0,
|
||||
y: handWorldLandmarkProto.getY() ?? 0,
|
||||
z: handWorldLandmarkProto.getZ() ?? 0,
|
||||
normalized: false
|
||||
});
|
||||
}
|
||||
this.worldLandmarks.push(worldLandmarks);
|
||||
}
|
||||
}
|
||||
|
||||
/** Updates the MediaPipe graph configuration. */
|
||||
private refreshGraph(): void {
|
||||
const graphConfig = new CalculatorGraphConfig();
|
||||
graphConfig.addInputStream(IMAGE_STREAM);
|
||||
graphConfig.addInputStream(NORM_RECT_STREAM);
|
||||
graphConfig.addOutputStream(LANDMARKS_STREAM);
|
||||
graphConfig.addOutputStream(WORLD_LANDMARKS_STREAM);
|
||||
graphConfig.addOutputStream(HANDEDNESS_STREAM);
|
||||
|
||||
const calculatorOptions = new CalculatorOptions();
|
||||
calculatorOptions.setExtension(
|
||||
HandLandmarkerGraphOptions.ext, this.options);
|
||||
|
||||
const landmarkerNode = new CalculatorGraphConfig.Node();
|
||||
landmarkerNode.setCalculator(HAND_LANDMARKER_GRAPH);
|
||||
landmarkerNode.addInputStream('IMAGE:' + IMAGE_STREAM);
|
||||
landmarkerNode.addInputStream('NORM_RECT:' + NORM_RECT_STREAM);
|
||||
landmarkerNode.addOutputStream('LANDMARKS:' + LANDMARKS_STREAM);
|
||||
landmarkerNode.addOutputStream('WORLD_LANDMARKS:' + WORLD_LANDMARKS_STREAM);
|
||||
landmarkerNode.addOutputStream('HANDEDNESS:' + HANDEDNESS_STREAM);
|
||||
landmarkerNode.setOptions(calculatorOptions);
|
||||
|
||||
graphConfig.addNode(landmarkerNode);
|
||||
|
||||
this.attachProtoVectorListener(LANDMARKS_STREAM, binaryProto => {
|
||||
this.addJsLandmarks(binaryProto);
|
||||
});
|
||||
this.attachProtoVectorListener(WORLD_LANDMARKS_STREAM, binaryProto => {
|
||||
this.adddJsWorldLandmarks(binaryProto);
|
||||
});
|
||||
this.attachProtoVectorListener(HANDEDNESS_STREAM, binaryProto => {
|
||||
this.handednesses.push(...this.toJsCategories(binaryProto));
|
||||
});
|
||||
|
||||
const binaryGraph = graphConfig.serializeBinary();
|
||||
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {BaseOptions} from '../../../../tasks/web/core/base_options';
|
||||
|
||||
/** Options to configure the MediaPipe HandLandmarker Task */
|
||||
export declare interface HandLandmarkerOptions {
|
||||
/** Options to configure the loading of the model assets. */
|
||||
baseOptions?: BaseOptions;
|
||||
|
||||
/**
|
||||
* The maximum number of hands can be detected by the HandLandmarker.
|
||||
* Defaults to 1.
|
||||
*/
|
||||
numHands?: number|undefined;
|
||||
|
||||
/**
|
||||
* The minimum confidence score for the hand detection to be considered
|
||||
* successful. Defaults to 0.5.
|
||||
*/
|
||||
minHandDetectionConfidence?: number|undefined;
|
||||
|
||||
/**
|
||||
* The minimum confidence score of hand presence score in the hand landmark
|
||||
* detection. Defaults to 0.5.
|
||||
*/
|
||||
minHandPresenceConfidence?: number|undefined;
|
||||
|
||||
/**
|
||||
* The minimum confidence score for the hand tracking to be considered
|
||||
* successful. Defaults to 0.5.
|
||||
*/
|
||||
minTrackingConfidence?: number|undefined;
|
||||
}
|
|
@ -0,0 +1,32 @@
|
|||
/**
|
||||
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import {Category} from '../../../../tasks/web/components/containers/category';
|
||||
import {Landmark} from '../../../../tasks/web/components/containers/landmark';
|
||||
|
||||
/**
|
||||
* Represents the hand landmarks deection results generated by `HandLandmarker`.
|
||||
*/
|
||||
export declare interface HandLandmarkerResult {
|
||||
/** Hand landmarks of detected hands. */
|
||||
landmarks: Landmark[][];
|
||||
|
||||
/** Hand landmarks in world coordniates of detected hands. */
|
||||
worldLandmarks: Landmark[][];
|
||||
|
||||
/** Handedness of detected hands. */
|
||||
handednesses: Category[][];
|
||||
}
|
|
@ -24,6 +24,11 @@ export * from '../../../tasks/web/vision/gesture_recognizer/gesture_recognizer_o
|
|||
export * from '../../../tasks/web/vision/gesture_recognizer/gesture_recognizer_result';
|
||||
export * from '../../../tasks/web/vision/gesture_recognizer/gesture_recognizer';
|
||||
|
||||
// Hand Landmarker
|
||||
export * from '../../../tasks/web/vision/hand_landmarker/hand_landmarker_options';
|
||||
export * from '../../../tasks/web/vision/hand_landmarker/hand_landmarker_result';
|
||||
export * from '../../../tasks/web/vision/hand_landmarker/hand_landmarker';
|
||||
|
||||
// Object Detector
|
||||
export * from '../../../tasks/web/vision/object_detector/object_detector_options';
|
||||
export * from '../../../tasks/web/vision/object_detector/object_detector_result';
|
||||
|
|
Loading…
Reference in New Issue
Block a user