From 9483ac4651ae1aa4a9f05752eab4536a3f0f6d91 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 25 May 2023 20:42:24 +0530 Subject: [PATCH] Updated iOS gesture recognizer results to initialize hand gestures with a default index --- .../utils/sources/MPPCategory+Helpers.h | 22 ++++++++++++++++++ .../utils/sources/MPPCategory+Helpers.mm | 9 ++++++-- .../MPPGestureRecognizerResult+Helpers.h | 12 ++++++---- .../MPPGestureRecognizerResult+Helpers.mm | 23 +++++++++++++------ 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h index 9a11d1e29..9ad958479 100644 --- a/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h @@ -19,8 +19,30 @@ NS_ASSUME_NONNULL_BEGIN @interface MPPCategory (Helpers) +/** + * Creates an `MPPCategory` with the given MediaPipe `Classification` proto. + * + * @param classificationProto A MediaPipe `Classification` proto. + * @return An `MPPCategory` object that with the given MediaPipe `Classification` proto. + */ + (MPPCategory *)categoryWithProto:(const ::mediapipe::Classification &)classificationProto; +/** + * Creates an `MPPCategory` with the given MediaPipe `Classification` proto and the given category + * index. The resulting `MPPCategory` is created with the given category index instead of the + * category index specified in the `Classification` proto. This method is useful for tasks like + * gesture recognizer which always returns a default index for the recognized gestures. + * + * @param classificationProto A MediaPipe `Classification` proto. + * @param index The index to be used for creating the `MPPCategory` instead of the category index + * specified in the `Classification` proto. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ ++ (MPPCategory *)categoryWithProto:(const ::mediapipe::Classification &)classificationProto + index:(NSInteger)index; + @end NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm index 12cfa5627..542b8c41d 100644 --- a/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm @@ -21,7 +21,8 @@ using ClassificationProto = ::mediapipe::Classification; @implementation MPPCategory (Helpers) -+ (MPPCategory *)categoryWithProto:(const ClassificationProto &)classificationProto { ++ (MPPCategory *)categoryWithProto:(const ClassificationProto &)classificationProto + index:(NSInteger)index { NSString *categoryName; NSString *displayName; @@ -33,10 +34,14 @@ using ClassificationProto = ::mediapipe::Classification; displayName = [NSString stringWithCppString:classificationProto.display_name()]; } - return [[MPPCategory alloc] initWithIndex:classificationProto.index() + return [[MPPCategory alloc] initWithIndex:index score:classificationProto.score() categoryName:categoryName displayName:displayName]; } ++ (MPPCategory *)categoryWithProto:(const ClassificationProto &)classificationProto { + return [MPPCategory categoryWithProto:classificationProto index:classificationProto.index()]; +} + @end diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h index 649c11c8a..6b0f8bf81 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h @@ -14,6 +14,8 @@ #import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/packet.h" NS_ASSUME_NONNULL_BEGIN @@ -23,14 +25,14 @@ static const int kMicroSecondsPerMilliSecond = 1000; @interface MPPGestureRecognizerResult (Helpers) /** - * Creates an `MPPGestureRecognizerResult` from hand gestures, handedness, hand landmarks and world + * Creates an `MPPGestureRecognizerResult` from hand gestures, handedness, hand landmarks and world * landmarks packets. * - * @param handGesturesPacket a MediaPipe packet wrapping a`std::vector`. - * @param handednessPacket a MediaPipe packet wrapping a`std::vector`. - * @param handLandmarksPacket a MediaPipe packet wrapping + * @param handGesturesPacket A MediaPipe packet wrapping a`std::vector`. + * @param handednessPacket A MediaPipe packet wrapping a`std::vector`. + * @param handLandmarksPacket A MediaPipe packet wrapping * a`std::vector`. - * @param handLandmarksPacket a MediaPipe packet wrapping a`std::vector`. + * @param worldLandmarksPacket A MediaPipe packet wrapping a`std::vector`. * * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition * results. diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm index 70773a940..b2e7ccded 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm @@ -21,6 +21,8 @@ #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/packet.h" +static const NSInteger kDefaultGestureIndex = -1; + namespace { using ClassificationListProto = ::mediapipe::ClassificationList; using LandmarkListProto = ::mediapipe::LandmarkList; @@ -30,6 +32,15 @@ using ::mediapipe::Packet; @implementation MPPGestureRecognizerResult (Helpers) ++ (MPPGestureRecognizerResult *)emptyGestureRecognizerResultWithTimestampInMilliseconds: + (NSInteger)timestampInMilliseconds { + return [[MPPGestureRecognizerResult alloc] initWithGestures:@[] + handedness:@[] + landmarks:@[] + worldLandmarks:@[] + timestampInMilliseconds:timestampInMilliseconds]; +} + + (MPPGestureRecognizerResult *) gestureRecognizerResultWithHandGesturesPacket:(const Packet &)handGesturesPacket handednessPacket:(const Packet &)handednessPacket @@ -39,18 +50,16 @@ using ::mediapipe::Packet; (NSInteger)(handGesturesPacket.Timestamp().Value() / kMicroSecondsPerMilliSecond); if (handGesturesPacket.IsEmpty()) { - return [[MPPGestureRecognizerResult alloc] initWithGestures:@[] - handedness:@[] - landmarks:@[] - worldLandmarks:@[] - timestampInMilliseconds:timestampInMilliseconds]; + return [MPPGestureRecognizerResult + emptyGestureRecognizerResultWithTimestampInMilliseconds:timestampInMilliseconds]; } if (!handGesturesPacket.ValidateAsType>().ok() || !handednessPacket.ValidateAsType>().ok() || !handLandmarksPacket.ValidateAsType>().ok() || !worldLandmarksPacket.ValidateAsType>().ok()) { - return nil; + return [MPPGestureRecognizerResult + emptyGestureRecognizerResultWithTimestampInMilliseconds:timestampInMilliseconds]; } const std::vector &handGesturesClassificationListProtos = @@ -62,7 +71,7 @@ using ::mediapipe::Packet; NSMutableArray *gestures = [NSMutableArray arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; for (const auto &classificationProto : classificationListProto.classification()) { - MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; + MPPCategory *category = [MPPCategory categoryWithProto:classificationProto index:kDefaultGestureIndex]; [gestures addObject:category]; } [multiHandGestures addObject:gestures];