Separate web and java api landmark and world landmark to two classes. This makes the platforms interface consistent.

PiperOrigin-RevId: 492332990
This commit is contained in:
MediaPipe Team 2022-12-01 16:47:05 -08:00 committed by Copybara-Service
parent ead41132a8
commit 768d2dc548
13 changed files with 161 additions and 96 deletions

View File

@ -83,6 +83,15 @@ android_library(
], ],
) )
android_library(
name = "normalized_landmark",
srcs = ["NormalizedLandmark.java"],
deps = [
"//third_party:autovalue",
"@maven//:com_google_guava_guava",
],
)
# Expose the java source files for building mediapipe tasks core AAR. # Expose the java source files for building mediapipe tasks core AAR.
filegroup( filegroup(
name = "java_src", name = "java_src",

View File

@ -18,16 +18,16 @@ import com.google.auto.value.AutoValue;
import java.util.Objects; import java.util.Objects;
/** /**
* Landmark represents a point in 3D space with x, y, z coordinates. If normalized is true, the * Landmark represents a point in 3D space with x, y, z coordinates. The landmark coordinates are in
* landmark coordinates is normalized respect to the dimension of image, and the coordinates values * meters. z represents the landmark depth, and the smaller the value the closer the world landmark
* are in the range of [0,1]. Otherwise, it represenet a point in world coordinates. * is to the camera.
*/ */
@AutoValue @AutoValue
public abstract class Landmark { public abstract class Landmark {
private static final float TOLERANCE = 1e-6f; private static final float TOLERANCE = 1e-6f;
public static Landmark create(float x, float y, float z, boolean normalized) { public static Landmark create(float x, float y, float z) {
return new AutoValue_Landmark(x, y, z, normalized); return new AutoValue_Landmark(x, y, z);
} }
// The x coordinates of the landmark. // The x coordinates of the landmark.
@ -39,28 +39,24 @@ public abstract class Landmark {
// The z coordinates of the landmark. // The z coordinates of the landmark.
public abstract float z(); public abstract float z();
// Whether this landmark is normalized with respect to the image size.
public abstract boolean normalized();
@Override @Override
public final boolean equals(Object o) { public final boolean equals(Object o) {
if (!(o instanceof Landmark)) { if (!(o instanceof Landmark)) {
return false; return false;
} }
Landmark other = (Landmark) o; Landmark other = (Landmark) o;
return other.normalized() == this.normalized() return Math.abs(other.x() - this.x()) < TOLERANCE
&& Math.abs(other.x() - this.x()) < TOLERANCE
&& Math.abs(other.x() - this.y()) < TOLERANCE && Math.abs(other.x() - this.y()) < TOLERANCE
&& Math.abs(other.x() - this.z()) < TOLERANCE; && Math.abs(other.x() - this.z()) < TOLERANCE;
} }
@Override @Override
public final int hashCode() { public final int hashCode() {
return Objects.hash(x(), y(), z(), normalized()); return Objects.hash(x(), y(), z());
} }
@Override @Override
public final String toString() { public final String toString() {
return "<Landmark (x=" + x() + " y=" + y() + " z=" + z() + " normalized=" + normalized() + ")>"; return "<Landmark (x=" + x() + " y=" + y() + " z=" + z() + ")>";
} }
} }

View File

@ -0,0 +1,63 @@
// Copyright 2022 The MediaPipe Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.mediapipe.tasks.components.containers;
import com.google.auto.value.AutoValue;
import java.util.Objects;
/**
* Normalized Landmark represents a point in 3D space with x, y, z coordinates. x and y are
* normalized to [0.0, 1.0] by the image width and height respectively. z represents the landmark
* depth, and the smaller the value the closer the landmark is to the camera. The magnitude of z
* uses roughly the same scale as x.
*/
@AutoValue
public abstract class NormalizedLandmark {
private static final float TOLERANCE = 1e-6f;
public static NormalizedLandmark create(float x, float y, float z) {
return new AutoValue_NormalizedLandmark(x, y, z);
}
// The x coordinates of the normalized landmark.
public abstract float x();
// The y coordinates of the normalized landmark.
public abstract float y();
// The z coordinates of the normalized landmark.
public abstract float z();
@Override
public final boolean equals(Object o) {
if (!(o instanceof NormalizedLandmark)) {
return false;
}
NormalizedLandmark other = (NormalizedLandmark) o;
return Math.abs(other.x() - this.x()) < TOLERANCE
&& Math.abs(other.x() - this.y()) < TOLERANCE
&& Math.abs(other.x() - this.z()) < TOLERANCE;
}
@Override
public final int hashCode() {
return Objects.hash(x(), y(), z());
}
@Override
public final String toString() {
return "<Normalized Landmark (x=" + x() + " y=" + y() + " z=" + z() + ")>";
}
}

View File

@ -135,6 +135,7 @@ android_library(
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category", "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:landmark", "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:landmark",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:normalized_landmark",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors:classifieroptions", "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/processors:classifieroptions",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/core", "//mediapipe/tasks/java/com/google/mediapipe/tasks/core",
"//third_party:autovalue", "//third_party:autovalue",
@ -167,6 +168,7 @@ android_library(
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category", "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:landmark", "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:landmark",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:normalized_landmark",
"//mediapipe/tasks/java/com/google/mediapipe/tasks/core", "//mediapipe/tasks/java/com/google/mediapipe/tasks/core",
"//third_party:autovalue", "//third_party:autovalue",
"@maven//:androidx_annotation_annotation", "@maven//:androidx_annotation_annotation",

View File

@ -15,13 +15,12 @@
package com.google.mediapipe.tasks.vision.gesturerecognizer; package com.google.mediapipe.tasks.vision.gesturerecognizer;
import com.google.auto.value.AutoValue; import com.google.auto.value.AutoValue;
import com.google.mediapipe.formats.proto.LandmarkProto.Landmark; import com.google.mediapipe.formats.proto.LandmarkProto;
import com.google.mediapipe.formats.proto.LandmarkProto.LandmarkList;
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
import com.google.mediapipe.formats.proto.ClassificationProto.Classification; import com.google.mediapipe.formats.proto.ClassificationProto.Classification;
import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList; import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList;
import com.google.mediapipe.tasks.components.containers.Category; import com.google.mediapipe.tasks.components.containers.Category;
import com.google.mediapipe.tasks.components.containers.Landmark;
import com.google.mediapipe.tasks.components.containers.NormalizedLandmark;
import com.google.mediapipe.tasks.core.TaskResult; import com.google.mediapipe.tasks.core.TaskResult;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
@ -43,41 +42,36 @@ public abstract class GestureRecognizerResult implements TaskResult {
* @param gesturesProto a List of {@link ClassificationList} * @param gesturesProto a List of {@link ClassificationList}
*/ */
static GestureRecognizerResult create( static GestureRecognizerResult create(
List<NormalizedLandmarkList> landmarksProto, List<LandmarkProto.NormalizedLandmarkList> landmarksProto,
List<LandmarkList> worldLandmarksProto, List<LandmarkProto.LandmarkList> worldLandmarksProto,
List<ClassificationList> handednessesProto, List<ClassificationList> handednessesProto,
List<ClassificationList> gesturesProto, List<ClassificationList> gesturesProto,
long timestampMs) { long timestampMs) {
List<List<com.google.mediapipe.tasks.components.containers.Landmark>> multiHandLandmarks = List<List<NormalizedLandmark>> multiHandLandmarks = new ArrayList<>();
new ArrayList<>(); List<List<Landmark>> multiHandWorldLandmarks = new ArrayList<>();
List<List<com.google.mediapipe.tasks.components.containers.Landmark>> multiHandWorldLandmarks =
new ArrayList<>();
List<List<Category>> multiHandHandednesses = new ArrayList<>(); List<List<Category>> multiHandHandednesses = new ArrayList<>();
List<List<Category>> multiHandGestures = new ArrayList<>(); List<List<Category>> multiHandGestures = new ArrayList<>();
for (NormalizedLandmarkList handLandmarksProto : landmarksProto) { for (LandmarkProto.NormalizedLandmarkList handLandmarksProto : landmarksProto) {
List<com.google.mediapipe.tasks.components.containers.Landmark> handLandmarks = List<NormalizedLandmark> handLandmarks = new ArrayList<>();
new ArrayList<>();
multiHandLandmarks.add(handLandmarks); multiHandLandmarks.add(handLandmarks);
for (NormalizedLandmark handLandmarkProto : handLandmarksProto.getLandmarkList()) { for (LandmarkProto.NormalizedLandmark handLandmarkProto :
handLandmarksProto.getLandmarkList()) {
handLandmarks.add( handLandmarks.add(
com.google.mediapipe.tasks.components.containers.Landmark.create( com.google.mediapipe.tasks.components.containers.NormalizedLandmark.create(
handLandmarkProto.getX(), handLandmarkProto.getX(), handLandmarkProto.getY(), handLandmarkProto.getZ()));
handLandmarkProto.getY(),
handLandmarkProto.getZ(),
true));
} }
} }
for (LandmarkList handWorldLandmarksProto : worldLandmarksProto) { for (LandmarkProto.LandmarkList handWorldLandmarksProto : worldLandmarksProto) {
List<com.google.mediapipe.tasks.components.containers.Landmark> handWorldLandmarks = List<com.google.mediapipe.tasks.components.containers.Landmark> handWorldLandmarks =
new ArrayList<>(); new ArrayList<>();
multiHandWorldLandmarks.add(handWorldLandmarks); multiHandWorldLandmarks.add(handWorldLandmarks);
for (Landmark handWorldLandmarkProto : handWorldLandmarksProto.getLandmarkList()) { for (LandmarkProto.Landmark handWorldLandmarkProto :
handWorldLandmarksProto.getLandmarkList()) {
handWorldLandmarks.add( handWorldLandmarks.add(
com.google.mediapipe.tasks.components.containers.Landmark.create( com.google.mediapipe.tasks.components.containers.Landmark.create(
handWorldLandmarkProto.getX(), handWorldLandmarkProto.getX(),
handWorldLandmarkProto.getY(), handWorldLandmarkProto.getY(),
handWorldLandmarkProto.getZ(), handWorldLandmarkProto.getZ()));
false));
} }
} }
for (ClassificationList handednessProto : handednessesProto) { for (ClassificationList handednessProto : handednessesProto) {
@ -118,11 +112,10 @@ public abstract class GestureRecognizerResult implements TaskResult {
public abstract long timestampMs(); public abstract long timestampMs();
/** Hand landmarks of detected hands. */ /** Hand landmarks of detected hands. */
public abstract List<List<com.google.mediapipe.tasks.components.containers.Landmark>> landmarks(); public abstract List<List<NormalizedLandmark>> landmarks();
/** Hand landmarks in world coordniates of detected hands. */ /** Hand landmarks in world coordniates of detected hands. */
public abstract List<List<com.google.mediapipe.tasks.components.containers.Landmark>> public abstract List<List<Landmark>> worldLandmarks();
worldLandmarks();
/** Handedness of detected hands. */ /** Handedness of detected hands. */
public abstract List<List<Category>> handednesses(); public abstract List<List<Category>> handednesses();

View File

@ -15,13 +15,12 @@
package com.google.mediapipe.tasks.vision.handlandmarker; package com.google.mediapipe.tasks.vision.handlandmarker;
import com.google.auto.value.AutoValue; import com.google.auto.value.AutoValue;
import com.google.mediapipe.formats.proto.LandmarkProto.Landmark; import com.google.mediapipe.formats.proto.LandmarkProto;
import com.google.mediapipe.formats.proto.LandmarkProto.LandmarkList;
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
import com.google.mediapipe.formats.proto.ClassificationProto.Classification; import com.google.mediapipe.formats.proto.ClassificationProto.Classification;
import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList; import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList;
import com.google.mediapipe.tasks.components.containers.Category; import com.google.mediapipe.tasks.components.containers.Category;
import com.google.mediapipe.tasks.components.containers.Landmark;
import com.google.mediapipe.tasks.components.containers.NormalizedLandmark;
import com.google.mediapipe.tasks.core.TaskResult; import com.google.mediapipe.tasks.core.TaskResult;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
@ -32,47 +31,41 @@ import java.util.List;
public abstract class HandLandmarkerResult implements TaskResult { public abstract class HandLandmarkerResult implements TaskResult {
/** /**
* Creates a {@link HandLandmarkerResult} instance from the lists of landmarks and * Creates a {@link HandLandmarkerResult} instance from the lists of landmarks and handedness
* handedness protobuf messages. * protobuf messages.
* *
* @param landmarksProto a List of {@link NormalizedLandmarkList} * @param landmarksProto a List of {@link NormalizedLandmarkList}
* @param worldLandmarksProto a List of {@link LandmarkList} * @param worldLandmarksProto a List of {@link LandmarkList}
* @param handednessesProto a List of {@link ClassificationList} * @param handednessesProto a List of {@link ClassificationList}
*/ */
static HandLandmarkerResult create( static HandLandmarkerResult create(
List<NormalizedLandmarkList> landmarksProto, List<LandmarkProto.NormalizedLandmarkList> landmarksProto,
List<LandmarkList> worldLandmarksProto, List<LandmarkProto.LandmarkList> worldLandmarksProto,
List<ClassificationList> handednessesProto, List<ClassificationList> handednessesProto,
long timestampMs) { long timestampMs) {
List<List<com.google.mediapipe.tasks.components.containers.Landmark>> multiHandLandmarks = List<List<NormalizedLandmark>> multiHandLandmarks = new ArrayList<>();
new ArrayList<>(); List<List<Landmark>> multiHandWorldLandmarks = new ArrayList<>();
List<List<com.google.mediapipe.tasks.components.containers.Landmark>> multiHandWorldLandmarks =
new ArrayList<>();
List<List<Category>> multiHandHandednesses = new ArrayList<>(); List<List<Category>> multiHandHandednesses = new ArrayList<>();
for (NormalizedLandmarkList handLandmarksProto : landmarksProto) { for (LandmarkProto.NormalizedLandmarkList handLandmarksProto : landmarksProto) {
List<com.google.mediapipe.tasks.components.containers.Landmark> handLandmarks = List<NormalizedLandmark> handLandmarks = new ArrayList<>();
new ArrayList<>();
multiHandLandmarks.add(handLandmarks); multiHandLandmarks.add(handLandmarks);
for (NormalizedLandmark handLandmarkProto : handLandmarksProto.getLandmarkList()) { for (LandmarkProto.NormalizedLandmark handLandmarkProto :
handLandmarksProto.getLandmarkList()) {
handLandmarks.add( handLandmarks.add(
com.google.mediapipe.tasks.components.containers.Landmark.create( NormalizedLandmark.create(
handLandmarkProto.getX(), handLandmarkProto.getX(), handLandmarkProto.getY(), handLandmarkProto.getZ()));
handLandmarkProto.getY(),
handLandmarkProto.getZ(),
true));
} }
} }
for (LandmarkList handWorldLandmarksProto : worldLandmarksProto) { for (LandmarkProto.LandmarkList handWorldLandmarksProto : worldLandmarksProto) {
List<com.google.mediapipe.tasks.components.containers.Landmark> handWorldLandmarks = List<Landmark> handWorldLandmarks = new ArrayList<>();
new ArrayList<>();
multiHandWorldLandmarks.add(handWorldLandmarks); multiHandWorldLandmarks.add(handWorldLandmarks);
for (Landmark handWorldLandmarkProto : handWorldLandmarksProto.getLandmarkList()) { for (LandmarkProto.Landmark handWorldLandmarkProto :
handWorldLandmarksProto.getLandmarkList()) {
handWorldLandmarks.add( handWorldLandmarks.add(
com.google.mediapipe.tasks.components.containers.Landmark.create( com.google.mediapipe.tasks.components.containers.Landmark.create(
handWorldLandmarkProto.getX(), handWorldLandmarkProto.getX(),
handWorldLandmarkProto.getY(), handWorldLandmarkProto.getY(),
handWorldLandmarkProto.getZ(), handWorldLandmarkProto.getZ()));
false));
} }
} }
for (ClassificationList handednessProto : handednessesProto) { for (ClassificationList handednessProto : handednessesProto) {
@ -98,11 +91,10 @@ public abstract class HandLandmarkerResult implements TaskResult {
public abstract long timestampMs(); public abstract long timestampMs();
/** Hand landmarks of detected hands. */ /** Hand landmarks of detected hands. */
public abstract List<List<com.google.mediapipe.tasks.components.containers.Landmark>> landmarks(); public abstract List<List<NormalizedLandmark>> landmarks();
/** Hand landmarks in world coordniates of detected hands. */ /** Hand landmarks in world coordniates of detected hands. */
public abstract List<List<com.google.mediapipe.tasks.components.containers.Landmark>> public abstract List<List<Landmark>> worldLandmarks();
worldLandmarks();
/** Handedness of detected hands. */ /** Handedness of detected hands. */
public abstract List<List<Category>> handednesses(); public abstract List<List<Category>> handednesses();

View File

@ -28,7 +28,7 @@ import com.google.mediapipe.framework.MediaPipeException;
import com.google.mediapipe.framework.image.BitmapImageBuilder; import com.google.mediapipe.framework.image.BitmapImageBuilder;
import com.google.mediapipe.framework.image.MPImage; import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.components.containers.Category; import com.google.mediapipe.tasks.components.containers.Category;
import com.google.mediapipe.tasks.components.containers.Landmark; import com.google.mediapipe.tasks.components.containers.NormalizedLandmark;
import com.google.mediapipe.tasks.components.containers.proto.LandmarksDetectionResultProto.LandmarksDetectionResult; import com.google.mediapipe.tasks.components.containers.proto.LandmarksDetectionResultProto.LandmarksDetectionResult;
import com.google.mediapipe.tasks.components.processors.ClassifierOptions; import com.google.mediapipe.tasks.components.processors.ClassifierOptions;
import com.google.mediapipe.tasks.core.BaseOptions; import com.google.mediapipe.tasks.core.BaseOptions;
@ -603,7 +603,7 @@ public class GestureRecognizerTest {
assertThat(actualResult.landmarks().get(0)) assertThat(actualResult.landmarks().get(0))
.comparingElementsUsing( .comparingElementsUsing(
Correspondence.from( Correspondence.from(
(Correspondence.BinaryPredicate<Landmark, Landmark>) (Correspondence.BinaryPredicate<NormalizedLandmark, NormalizedLandmark>)
(actual, expected) -> { (actual, expected) -> {
return Correspondence.tolerance(LANDMARKS_ERROR_TOLERANCE) return Correspondence.tolerance(LANDMARKS_ERROR_TOLERANCE)
.compare(actual.x(), expected.x()) .compare(actual.x(), expected.x())

View File

@ -27,7 +27,7 @@ import com.google.mediapipe.framework.MediaPipeException;
import com.google.mediapipe.framework.image.BitmapImageBuilder; import com.google.mediapipe.framework.image.BitmapImageBuilder;
import com.google.mediapipe.framework.image.MPImage; import com.google.mediapipe.framework.image.MPImage;
import com.google.mediapipe.tasks.components.containers.Category; import com.google.mediapipe.tasks.components.containers.Category;
import com.google.mediapipe.tasks.components.containers.Landmark; import com.google.mediapipe.tasks.components.containers.NormalizedLandmark;
import com.google.mediapipe.tasks.components.containers.proto.LandmarksDetectionResultProto.LandmarksDetectionResult; import com.google.mediapipe.tasks.components.containers.proto.LandmarksDetectionResultProto.LandmarksDetectionResult;
import com.google.mediapipe.tasks.core.BaseOptions; import com.google.mediapipe.tasks.core.BaseOptions;
import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions; import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions;
@ -399,7 +399,7 @@ public class HandLandmarkerTest {
assertThat(actualResult.landmarks().get(0)) assertThat(actualResult.landmarks().get(0))
.comparingElementsUsing( .comparingElementsUsing(
Correspondence.from( Correspondence.from(
(Correspondence.BinaryPredicate<Landmark, Landmark>) (Correspondence.BinaryPredicate<NormalizedLandmark, NormalizedLandmark>)
(actual, expected) -> { (actual, expected) -> {
return Correspondence.tolerance(LANDMARKS_ERROR_TOLERANCE) return Correspondence.tolerance(LANDMARKS_ERROR_TOLERANCE)
.compare(actual.x(), expected.x()) .compare(actual.x(), expected.x())

View File

@ -15,10 +15,27 @@
*/ */
/** /**
* Landmark represents a point in 3D space with x, y, z coordinates. If * Normalized Landmark represents a point in 3D space with x, y, z coordinates.
* normalized is true, the landmark coordinates is normalized respect to the * x and y are normalized to [0.0, 1.0] by the image width and height
* dimension of image, and the coordinates values are in the range of [0,1]. * respectively. z represents the landmark depth, and the smaller the value the
* Otherwise, it represenet a point in world coordinates. * closer the landmark is to the camera. The magnitude of z uses roughly the
* same scale as x.
*/
export declare interface NormalizedLandmark {
/** The x coordinates of the normalized landmark. */
x: number;
/** The y coordinates of the normalized landmark. */
y: number;
/** The z coordinates of the normalized landmark. */
z: number;
}
/**
* Landmark represents a point in 3D space with x, y, z coordinates. The
* landmark coordinates are in meters. z represents the landmark depth,
* and the smaller the value the closer the world landmark is to the camera.
*/ */
export declare interface Landmark { export declare interface Landmark {
/** The x coordinates of the landmark. */ /** The x coordinates of the landmark. */
@ -29,7 +46,4 @@ export declare interface Landmark {
/** The z coordinates of the landmark. */ /** The z coordinates of the landmark. */
z: number; z: number;
/** Whether this landmark is normalized with respect to the image size. */
normalized: boolean;
} }

View File

@ -27,7 +27,7 @@ import {HandDetectorGraphOptions} from '../../../../tasks/cc/vision/hand_detecto
import {HandLandmarkerGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarker_graph_options_pb'; import {HandLandmarkerGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarker_graph_options_pb';
import {HandLandmarksDetectorGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options_pb'; import {HandLandmarksDetectorGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options_pb';
import {Category} from '../../../../tasks/web/components/containers/category'; import {Category} from '../../../../tasks/web/components/containers/category';
import {Landmark} from '../../../../tasks/web/components/containers/landmark'; import {Landmark, NormalizedLandmark} from '../../../../tasks/web/components/containers/landmark';
import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options'; import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options';
import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset'; import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner'; import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner';
@ -67,7 +67,7 @@ FULL_IMAGE_RECT.setHeight(1);
export class GestureRecognizer extends export class GestureRecognizer extends
VisionTaskRunner<GestureRecognizerResult> { VisionTaskRunner<GestureRecognizerResult> {
private gestures: Category[][] = []; private gestures: Category[][] = [];
private landmarks: Landmark[][] = []; private landmarks: NormalizedLandmark[][] = [];
private worldLandmarks: Landmark[][] = []; private worldLandmarks: Landmark[][] = [];
private handednesses: Category[][] = []; private handednesses: Category[][] = [];
@ -306,13 +306,12 @@ export class GestureRecognizer extends
for (const binaryProto of data) { for (const binaryProto of data) {
const handLandmarksProto = const handLandmarksProto =
NormalizedLandmarkList.deserializeBinary(binaryProto); NormalizedLandmarkList.deserializeBinary(binaryProto);
const landmarks: Landmark[] = []; const landmarks: NormalizedLandmark[] = [];
for (const handLandmarkProto of handLandmarksProto.getLandmarkList()) { for (const handLandmarkProto of handLandmarksProto.getLandmarkList()) {
landmarks.push({ landmarks.push({
x: handLandmarkProto.getX() ?? 0, x: handLandmarkProto.getX() ?? 0,
y: handLandmarkProto.getY() ?? 0, y: handLandmarkProto.getY() ?? 0,
z: handLandmarkProto.getZ() ?? 0, z: handLandmarkProto.getZ() ?? 0
normalized: true
}); });
} }
this.landmarks.push(landmarks); this.landmarks.push(landmarks);
@ -333,8 +332,7 @@ export class GestureRecognizer extends
worldLandmarks.push({ worldLandmarks.push({
x: handWorldLandmarkProto.getX() ?? 0, x: handWorldLandmarkProto.getX() ?? 0,
y: handWorldLandmarkProto.getY() ?? 0, y: handWorldLandmarkProto.getY() ?? 0,
z: handWorldLandmarkProto.getZ() ?? 0, z: handWorldLandmarkProto.getZ() ?? 0
normalized: false
}); });
} }
this.worldLandmarks.push(worldLandmarks); this.worldLandmarks.push(worldLandmarks);

View File

@ -15,14 +15,14 @@
*/ */
import {Category} from '../../../../tasks/web/components/containers/category'; import {Category} from '../../../../tasks/web/components/containers/category';
import {Landmark} from '../../../../tasks/web/components/containers/landmark'; import {Landmark, NormalizedLandmark} from '../../../../tasks/web/components/containers/landmark';
/** /**
* Represents the gesture recognition results generated by `GestureRecognizer`. * Represents the gesture recognition results generated by `GestureRecognizer`.
*/ */
export declare interface GestureRecognizerResult { export declare interface GestureRecognizerResult {
/** Hand landmarks of detected hands. */ /** Hand landmarks of detected hands. */
landmarks: Landmark[][]; landmarks: NormalizedLandmark[][];
/** Hand landmarks in world coordniates of detected hands. */ /** Hand landmarks in world coordniates of detected hands. */
worldLandmarks: Landmark[][]; worldLandmarks: Landmark[][];

View File

@ -24,7 +24,7 @@ import {HandDetectorGraphOptions} from '../../../../tasks/cc/vision/hand_detecto
import {HandLandmarkerGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarker_graph_options_pb'; import {HandLandmarkerGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarker_graph_options_pb';
import {HandLandmarksDetectorGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options_pb'; import {HandLandmarksDetectorGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options_pb';
import {Category} from '../../../../tasks/web/components/containers/category'; import {Category} from '../../../../tasks/web/components/containers/category';
import {Landmark} from '../../../../tasks/web/components/containers/landmark'; import {Landmark, NormalizedLandmark} from '../../../../tasks/web/components/containers/landmark';
import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset'; import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner'; import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner';
import {ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner'; import {ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner';
@ -59,7 +59,7 @@ FULL_IMAGE_RECT.setHeight(1);
/** Performs hand landmarks detection on images. */ /** Performs hand landmarks detection on images. */
export class HandLandmarker extends VisionTaskRunner<HandLandmarkerResult> { export class HandLandmarker extends VisionTaskRunner<HandLandmarkerResult> {
private landmarks: Landmark[][] = []; private landmarks: NormalizedLandmark[][] = [];
private worldLandmarks: Landmark[][] = []; private worldLandmarks: Landmark[][] = [];
private handednesses: Category[][] = []; private handednesses: Category[][] = [];
@ -255,13 +255,12 @@ export class HandLandmarker extends VisionTaskRunner<HandLandmarkerResult> {
for (const binaryProto of data) { for (const binaryProto of data) {
const handLandmarksProto = const handLandmarksProto =
NormalizedLandmarkList.deserializeBinary(binaryProto); NormalizedLandmarkList.deserializeBinary(binaryProto);
const landmarks: Landmark[] = []; const landmarks: NormalizedLandmark[] = [];
for (const handLandmarkProto of handLandmarksProto.getLandmarkList()) { for (const handLandmarkProto of handLandmarksProto.getLandmarkList()) {
landmarks.push({ landmarks.push({
x: handLandmarkProto.getX() ?? 0, x: handLandmarkProto.getX() ?? 0,
y: handLandmarkProto.getY() ?? 0, y: handLandmarkProto.getY() ?? 0,
z: handLandmarkProto.getZ() ?? 0, z: handLandmarkProto.getZ() ?? 0,
normalized: true
}); });
} }
this.landmarks.push(landmarks); this.landmarks.push(landmarks);
@ -269,7 +268,7 @@ export class HandLandmarker extends VisionTaskRunner<HandLandmarkerResult> {
} }
/** /**
* Converts raw data into a landmark, and adds it to our worldLandmarks * Converts raw data into a world landmark, and adds it to our worldLandmarks
* list. * list.
*/ */
private adddJsWorldLandmarks(data: Uint8Array[]): void { private adddJsWorldLandmarks(data: Uint8Array[]): void {
@ -283,7 +282,6 @@ export class HandLandmarker extends VisionTaskRunner<HandLandmarkerResult> {
x: handWorldLandmarkProto.getX() ?? 0, x: handWorldLandmarkProto.getX() ?? 0,
y: handWorldLandmarkProto.getY() ?? 0, y: handWorldLandmarkProto.getY() ?? 0,
z: handWorldLandmarkProto.getZ() ?? 0, z: handWorldLandmarkProto.getZ() ?? 0,
normalized: false
}); });
} }
this.worldLandmarks.push(worldLandmarks); this.worldLandmarks.push(worldLandmarks);

View File

@ -15,14 +15,14 @@
*/ */
import {Category} from '../../../../tasks/web/components/containers/category'; import {Category} from '../../../../tasks/web/components/containers/category';
import {Landmark} from '../../../../tasks/web/components/containers/landmark'; import {Landmark, NormalizedLandmark} from '../../../../tasks/web/components/containers/landmark';
/** /**
* Represents the hand landmarks deection results generated by `HandLandmarker`. * Represents the hand landmarks deection results generated by `HandLandmarker`.
*/ */
export declare interface HandLandmarkerResult { export declare interface HandLandmarkerResult {
/** Hand landmarks of detected hands. */ /** Hand landmarks of detected hands. */
landmarks: Landmark[][]; landmarks: NormalizedLandmark[][];
/** Hand landmarks in world coordniates of detected hands. */ /** Hand landmarks in world coordniates of detected hands. */
worldLandmarks: Landmark[][]; worldLandmarks: Landmark[][];