Internal change
PiperOrigin-RevId: 499283559
This commit is contained in:
parent
68f247a5c7
commit
75b87e0e32
|
@ -263,12 +263,22 @@ export class GestureRecognizer extends
|
|||
NORM_RECT_STREAM, timestamp);
|
||||
this.finishProcessing();
|
||||
|
||||
return {
|
||||
gestures: this.gestures,
|
||||
landmarks: this.landmarks,
|
||||
worldLandmarks: this.worldLandmarks,
|
||||
handednesses: this.handednesses
|
||||
};
|
||||
if (this.gestures.length === 0) {
|
||||
// If no gestures are detected in the image, just return an empty list
|
||||
return {
|
||||
gestures: [],
|
||||
landmarks: [],
|
||||
worldLandmarks: [],
|
||||
handednesses: [],
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
gestures: this.gestures,
|
||||
landmarks: this.landmarks,
|
||||
worldLandmarks: this.worldLandmarks,
|
||||
handednesses: this.handednesses
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Sets the default values for the graph. */
|
||||
|
@ -283,15 +293,19 @@ export class GestureRecognizer extends
|
|||
}
|
||||
|
||||
/** Converts the proto data to a Category[][] structure. */
|
||||
private toJsCategories(data: Uint8Array[]): Category[][] {
|
||||
private toJsCategories(data: Uint8Array[], populateIndex = true):
|
||||
Category[][] {
|
||||
const result: Category[][] = [];
|
||||
for (const binaryProto of data) {
|
||||
const inputList = ClassificationList.deserializeBinary(binaryProto);
|
||||
const outputList: Category[] = [];
|
||||
for (const classification of inputList.getClassificationList()) {
|
||||
const index = populateIndex && classification.hasIndex() ?
|
||||
classification.getIndex()! :
|
||||
DEFAULT_CATEGORY_INDEX;
|
||||
outputList.push({
|
||||
score: classification.getScore() ?? 0,
|
||||
index: classification.getIndex() ?? DEFAULT_CATEGORY_INDEX,
|
||||
index,
|
||||
categoryName: classification.getLabel() ?? '',
|
||||
displayName: classification.getDisplayName() ?? '',
|
||||
});
|
||||
|
@ -375,7 +389,10 @@ export class GestureRecognizer extends
|
|||
});
|
||||
this.graphRunner.attachProtoVectorListener(
|
||||
HAND_GESTURES_STREAM, binaryProto => {
|
||||
this.gestures.push(...this.toJsCategories(binaryProto));
|
||||
// Gesture index is not used, because the final gesture result comes
|
||||
// from multiple classifiers.
|
||||
this.gestures.push(
|
||||
...this.toJsCategories(binaryProto, /* populateIndex= */ false));
|
||||
});
|
||||
this.graphRunner.attachProtoVectorListener(
|
||||
HANDEDNESS_STREAM, binaryProto => {
|
||||
|
|
|
@ -17,6 +17,8 @@
|
|||
import {Category} from '../../../../tasks/web/components/containers/category';
|
||||
import {Landmark, NormalizedLandmark} from '../../../../tasks/web/components/containers/landmark';
|
||||
|
||||
export {Category, Landmark, NormalizedLandmark};
|
||||
|
||||
/**
|
||||
* Represents the gesture recognition results generated by `GestureRecognizer`.
|
||||
*/
|
||||
|
@ -30,6 +32,10 @@ export declare interface GestureRecognizerResult {
|
|||
/** Handedness of detected hands. */
|
||||
handednesses: Category[][];
|
||||
|
||||
/** Recognized hand gestures of detected hands */
|
||||
/**
|
||||
* Recognized hand gestures of detected hands. Note that the index of the
|
||||
* gesture is always -1, because the raw indices from multiple gesture
|
||||
* classifiers cannot consolidate to a meaningful index.
|
||||
*/
|
||||
gestures: Category[][];
|
||||
}
|
||||
|
|
|
@ -272,7 +272,7 @@ describe('GestureRecognizer', () => {
|
|||
expect(gestures).toEqual({
|
||||
'gestures': [[{
|
||||
'score': 0.2,
|
||||
'index': 2,
|
||||
'index': -1,
|
||||
'categoryName': 'gesture_label',
|
||||
'displayName': 'gesture_display_name'
|
||||
}]],
|
||||
|
@ -305,4 +305,25 @@ describe('GestureRecognizer', () => {
|
|||
// gestures.
|
||||
expect(gestures2).toEqual(gestures1);
|
||||
});
|
||||
|
||||
it('returns empty results when no gestures are detected', async () => {
|
||||
// Pass the test data to our listener
|
||||
gestureRecognizer.fakeWasmModule._waitUntilIdle.and.callFake(() => {
|
||||
verifyListenersRegistered(gestureRecognizer);
|
||||
gestureRecognizer.listeners.get('hand_landmarks')!(createLandmarks());
|
||||
gestureRecognizer.listeners.get('world_hand_landmarks')!
|
||||
(createWorldLandmarks());
|
||||
gestureRecognizer.listeners.get('handedness')!(createHandednesses());
|
||||
gestureRecognizer.listeners.get('hand_gestures')!([]);
|
||||
});
|
||||
|
||||
// Invoke the gesture recognizer
|
||||
const gestures = gestureRecognizer.recognize({} as HTMLImageElement);
|
||||
expect(gestures).toEqual({
|
||||
'gestures': [],
|
||||
'landmarks': [],
|
||||
'worldLandmarks': [],
|
||||
'handednesses': []
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
Loading…
Reference in New Issue
Block a user