From e16f5f8f084a64fe313add198f4cda5a325c56e5 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 24 May 2023 19:09:48 +0530 Subject: [PATCH] Added a method for creating ios hand gesture result from protos --- .../MPPGestureRecognizerResult+Helpers.h | 36 +++++- .../MPPGestureRecognizerResult+Helpers.mm | 113 ++++++++++-------- 2 files changed, 97 insertions(+), 52 deletions(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h index 649c11c8a..5e75febf3 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h @@ -14,6 +14,8 @@ #import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/packet.h" NS_ASSUME_NONNULL_BEGIN @@ -23,14 +25,14 @@ static const int kMicroSecondsPerMilliSecond = 1000; @interface MPPGestureRecognizerResult (Helpers) /** - * Creates an `MPPGestureRecognizerResult` from hand gestures, handedness, hand landmarks and world + * Creates an `MPPGestureRecognizerResult` from hand gestures, handedness, hand landmarks and world * landmarks packets. * - * @param handGesturesPacket a MediaPipe packet wrapping a`std::vector`. - * @param handednessPacket a MediaPipe packet wrapping a`std::vector`. - * @param handLandmarksPacket a MediaPipe packet wrapping + * @param handGesturesPacket A MediaPipe packet wrapping a`std::vector`. + * @param handednessPacket A MediaPipe packet wrapping a`std::vector`. + * @param handLandmarksPacket A MediaPipe packet wrapping * a`std::vector`. - * @param handLandmarksPacket a MediaPipe packet wrapping a`std::vector`. + * @param worldLandmarksPacket A MediaPipe packet wrapping a`std::vector`. * * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition * results. @@ -41,6 +43,30 @@ static const int kMicroSecondsPerMilliSecond = 1000; handLandmarksPacket:(const mediapipe::Packet &)handLandmarksPacket worldLandmarksPacket:(const mediapipe::Packet &)worldLandmarksPacket; +/** + * Creates an `MPPGestureRecognizerResult` from hand gestures, handedness, hand landmarks and world + * landmarks proto vectors. + * + * @param handGesturesProto A vector of protos of type `std::vector`. + * @param handednessPacket A vector of protos of type `std::vector`. + * @param handLandmarksPacket A vector of protos of type `std::vector`. + * @param handLandmarksPacket A vector of protos of type `std::vector`. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultWithHandGesturesProto: + (const std::vector &)handGesturesProto + handednessroto: + (const std::vector &) + handednessProto + handLandmarksPacket: + (const std::vector &) + handLandmarksProto + worldLandmarksPacket: + (const std::vector &)worldLandmarksProto + timestampInMilliSeconds:(NSInteger)timestampInMilliseconds; @end NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm index ecb0e7831..5e5b301cc 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm @@ -17,10 +17,6 @@ #import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h" #import "mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/packet.h" - namespace { using ClassificationListProto = ::mediapipe::ClassificationList; using LandmarkListProto = ::mediapipe::LandmarkList; @@ -28,52 +24,47 @@ using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList; using ::mediapipe::Packet; } // namespace +static const NSInteger kDefaultGestureIndex = -1; + @implementation MPPGestureRecognizerResult (Helpers) ++ (MPPGestureRecognizerResult *)emptyGestureRecognizerResultWithTimestampInMilliseconds: + (NSInteger)timestampInMilliseconds { + return [[MPPGestureRecognizerResult alloc] initWithGestures:@[] + handedness:@[] + landmarks:@[] + worldLandmarks:@[] + timestampInMilliseconds:timestampInMilliseconds]; +} + + (MPPGestureRecognizerResult *) - gestureRecognizerResultWithHandGesturesPacket:(const Packet &)handGesturesPacket - handednessPacket:(const Packet &)handednessPacket - handLandmarksPacket:(const Packet &)handLandmarksPacket - worldLandmarksPacket:(const Packet &)worldLandmarksPacket { - NSInteger timestampInMilliseconds = - (NSInteger)(handGesturesPacket.Timestamp().Value() / kMicroSecondsPerMilliSecond); - - if (handGesturesPacket.IsEmpty()) { - return [[MPPGestureRecognizerResult alloc] initWithGestures:@[] - handedness:@[] - landmarks:@[] - worldLandmarks:@[] - timestampInMilliseconds:timestampInMilliseconds]; - } - - if (!handGesturesPacket.ValidateAsType>().ok() || - !handednessPacket.ValidateAsType>().ok() || - !handLandmarksPacket.ValidateAsType>().ok() || - !worldLandmarksPacket.ValidateAsType>().ok()) { - return nil; - } - - const std::vector &handGesturesClassificationListProtos = - handGesturesPacket.Get>(); + gestureRecognizerResultWithHandGesturesProto: + (const std::vector &)handGesturesProto + handednessroto: + (const std::vector &)handednessProto + handLandmarksPacket:(const std::vector &) + handLandmarksProto + worldLandmarksPacket: + (const std::vector &)worldLandmarksProto + timestampInMilliSeconds:(NSInteger)timestampInMilliseconds { NSMutableArray *> *multiHandGestures = - [NSMutableArray arrayWithCapacity:(NSUInteger)handGesturesClassificationListProtos.size()]; + [NSMutableArray arrayWithCapacity:(NSUInteger)handGesturesProto.size()]; - for (const auto &classificationListProto : handGesturesClassificationListProtos) { + for (const auto &classificationListProto : handGesturesProto) { NSMutableArray *gestures = [NSMutableArray arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; for (const auto &classificationProto : classificationListProto.classification()) { - MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; + MPPCategory *category = [MPPCategory categoryWithProto:classificationProto + andIndex:kDefaultGestureIndex]; [gestures addObject:category]; } [multiHandGestures addObject:gestures]; } - const std::vector &handednessClassificationListProtos = - handednessPacket.Get>(); NSMutableArray *> *multiHandHandedness = - [NSMutableArray arrayWithCapacity:(NSUInteger)handednessClassificationListProtos.size()]; + [NSMutableArray arrayWithCapacity:(NSUInteger)handednessProto.size()]; - for (const auto &classificationListProto : handednessClassificationListProtos) { + for (const auto &classificationListProto : handednessProto) { NSMutableArray *handedness = [NSMutableArray arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; for (const auto &classificationProto : classificationListProto.classification()) { @@ -83,36 +74,31 @@ using ::mediapipe::Packet; [multiHandHandedness addObject:handedness]; } - const std::vector &handLandmarkListProtos = - handLandmarksPacket.Get>(); NSMutableArray *> *multiHandLandmarks = - [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProtos.size()]; + [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarksProto.size()]; - for (const auto &handLandmarkListProto : handLandmarkListProtos) { + for (const auto &handLandmarkListProto : handLandmarksProto) { NSMutableArray *handLandmarks = [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProto.landmark().size()]; for (const auto &normalizedLandmarkProto : handLandmarkListProto.landmark()) { MPPNormalizedLandmark *normalizedLandmark = - [MPPNormalizedLandmark normalizedLandmarkWithProto:normalizedLandmarkProto]; + [MPPNormalizedLandmark normalizedLandmarkWithProto:normalizedLandmarkProto]; [handLandmarks addObject:normalizedLandmark]; } [multiHandLandmarks addObject:handLandmarks]; } - const std::vector &worldLandmarkListProtos = - worldLandmarksPacket.Get>(); NSMutableArray *> *multiHandWorldLandmarks = - [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProtos.size()]; + [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarksProto.size()]; - for (const auto &worldLandmarkListProto : worldLandmarkListProtos) { + for (const auto &worldLandmarkListProto : worldLandmarksProto) { NSMutableArray *worldLandmarks = [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProto.landmark().size()]; for (const auto &landmarkProto : worldLandmarkListProto.landmark()) { - MPPLandmark *landmark = - [MPPLandmark landmarkWithProto:landmarkProto]; + MPPLandmark *landmark = [MPPLandmark landmarkWithProto:landmarkProto]; [worldLandmarks addObject:landmark]; } - [multiHandWorldLandmarks addObject:worldLandmarks]; + [multiHandWorldLandmarks addObject:worldLandmarks]; } MPPGestureRecognizerResult *gestureRecognizerResult = @@ -125,4 +111,37 @@ using ::mediapipe::Packet; return gestureRecognizerResult; } ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultWithHandGesturesPacket:(const Packet &)handGesturesPacket + handednessPacket:(const Packet &)handednessPacket + handLandmarksPacket:(const Packet &)handLandmarksPacket + worldLandmarksPacket:(const Packet &)worldLandmarksPacket { + NSInteger timestampInMilliseconds = + (NSInteger)(handGesturesPacket.Timestamp().Value() / kMicroSecondsPerMilliSecond); + + if (handGesturesPacket.IsEmpty()) { + return [MPPGestureRecognizerResult + emptyGestureRecognizerResultWithTimestampInMilliseconds:timestampInMilliseconds]; + } + + if (!handGesturesPacket.ValidateAsType>().ok() || + !handednessPacket.ValidateAsType>().ok() || + !handLandmarksPacket.ValidateAsType>().ok() || + !worldLandmarksPacket.ValidateAsType>().ok()) { + return [MPPGestureRecognizerResult + emptyGestureRecognizerResultWithTimestampInMilliseconds:timestampInMilliseconds]; + } + + return [MPPGestureRecognizerResult + gestureRecognizerResultWithHandGesturesProto:handGesturesPacket + .Get>() + handednessroto:handednessPacket + .Get>() + handLandmarksPacket:handednessPacket + .Get>() + worldLandmarksPacket:worldLandmarksPacket + .Get>() + timestampInMilliSeconds:timestampInMilliseconds]; +} + @end