From 501b4bbc7b46d50139d269c00db0014f0f99b1de Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 17 May 2023 21:52:05 +0530 Subject: [PATCH] Added MPPGestureRecognizerResultHelpers --- .../ios/vision/gesture_recognizer/utils/BUILD | 13 ++ .../MPPGestureRecognizerResult+Helpers.h | 46 +++++++ .../MPPGestureRecognizerResult+Helpers.mm | 128 ++++++++++++++++++ 3 files changed, 187 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD index f643c83f5..8db5a63db 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD @@ -31,3 +31,16 @@ objc_library( ], ) +objc_library( + name = "MPPGestureRecognizerResultHelpers", + srcs = ["sources/MPPGestureRecognizerResult+Helpers.mm"], + hdrs = ["sources/MPPGestureRecognizerResult+Helpers.h"], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/tasks/ios/components/containers/utils:MPPCategoryHelpers", + "//mediapipe/tasks/ios/components/containers/utils:MPPLandmarkHelpers", + "//mediapipe/tasks/ios/vision/gesture_recognizer:MPPGestureRecognizerResult", + ], +) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h new file mode 100644 index 000000000..649c11c8a --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h @@ -0,0 +1,46 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" + +#include "mediapipe/framework/packet.h" + +NS_ASSUME_NONNULL_BEGIN + +static const int kMicroSecondsPerMilliSecond = 1000; + +@interface MPPGestureRecognizerResult (Helpers) + +/** + * Creates an `MPPGestureRecognizerResult` from hand gestures, handedness, hand landmarks and world + * landmarks packets. + * + * @param handGesturesPacket a MediaPipe packet wrapping a`std::vector`. + * @param handednessPacket a MediaPipe packet wrapping a`std::vector`. + * @param handLandmarksPacket a MediaPipe packet wrapping + * a`std::vector`. + * @param handLandmarksPacket a MediaPipe packet wrapping a`std::vector`. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultWithHandGesturesPacket:(const mediapipe::Packet &)handGesturesPacket + handednessPacket:(const mediapipe::Packet &)handednessPacket + handLandmarksPacket:(const mediapipe::Packet &)handLandmarksPacket + worldLandmarksPacket:(const mediapipe::Packet &)worldLandmarksPacket; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm new file mode 100644 index 000000000..d125ad26f --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm @@ -0,0 +1,128 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h" + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h" + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/packet.h" + +namespace { +using ClassificationListProto = ::mediapipe::ClassificationList; +using LandmarkListProto = ::mediapipe::LandmarkList; +using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList; +using ::mediapipe::Packet; +} // namespace + +@implementation MPPGestureRecognizerResult (Helpers) + ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultWithHandGesturesPacket:(const Packet &)handGesturesPacket + handednessPacket:(const Packet &)handednessPacket + handLandmarksPacket:(const Packet &)handLandmarksPacket + worldLandmarksPacket:(const Packet &)worldLandmarksPacket { + NSInteger timestampInMilliseconds = + (NSInteger)(handGesturesPacket.Timestamp().Value() / kMicroSecondsPerMilliSecond); + + if (handGesturesPacket.IsEmpty()) { + return [[MPPGestureRecognizerResult alloc] initWithGestures:@[] + handedness:@[] + landmarks:@[] + worldLandmarks:@[] + timestampInMilliseconds:timestampInMilliseconds]; + } + + if (!handGesturesPacket.ValidateAsType>().ok() || + !handednessPacket.ValidateAsType>().ok() || + !handLandmarksPacket.ValidateAsType>().ok() || + !worldLandmarksPacket.ValidateAsType>().ok()) { + return nil; + } + + const std::vector &handGesturesClassificationListProtos = + handGesturesPacket.Get>(); + NSMutableArray *> *multiHandGestures = + [NSMutableArray arrayWithCapacity:(NSUInteger)handGesturesClassificationListProtos.size()]; + + for (const auto &classificationListProto : handGesturesClassificationListProtos) { + NSMutableArray *gestures = [NSMutableArray + arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; + for (const auto &classificationProto : classificationListProto.classification()) { + MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; + [gestures addObject:category]; + } + [multiHandGestures addObject:[gestures copy]]; + } + + const std::vector &handednessClassificationListProtos = + handednessPacket.Get>(); + NSMutableArray *> *multiHandHandedness = + [NSMutableArray arrayWithCapacity:(NSUInteger)handednessClassificationListProtos.size()]; + + for (const auto &classificationListProto : handednessClassificationListProtos) { + NSMutableArray *handedness = [NSMutableArray + arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; + for (const auto &classificationProto : classificationListProto.classification()) { + MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; + [handedness addObject:category]; + } + [multiHandHandedness addObject:[handedness copy]]; + } + + const std::vector &handLandmarkListProtos = + handLandmarksPacket.Get>(); + NSMutableArray *> *multiHandLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProtos.size()]; + + for (const auto &handLandmarkListProto : handLandmarkListProtos) { + NSMutableArray *handLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProto.landmark().size()]; + for (const auto &normalizedLandmarkProto : handLandmarkListProto.landmark()) { + MPPNormalizedLandmark *normalizedLandmark = + [MPPNormalizedLandmark normalizedLandmarkWithProto:normalizedLandmarkProto]; + [handLandmarks addObject:normalizedLandmark]; + } + [multiHandLandmarks addObject:[handLandmarks copy]]; + } + + const std::vector &worldLandmarkListProtos = + worldLandmarksPacket.Get>(); + NSMutableArray *> *multiHandWorldLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProtos.size()]; + + for (const auto &worldLandmarkListProto : worldLandmarkListProtos) { + NSMutableArray *worldLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProto.landmark().size()]; + for (const auto &landmarkProto : worldLandmarkListProto.landmark()) { + MPPLandmark *landmark = + [MPPLandmark landmarkWithProto:landmarkProto]; + [worldLandmarks addObject:landmark]; + } + [multiHandWorldLandmarks addObject:[worldLandmarks copy]]; + } + + MPPGestureRecognizerResult *gestureRecognizerResult = + [[MPPGestureRecognizerResult alloc] initWithGestures:[multiHandGestures copy] + handedness:[multiHandHandedness copy] + landmarks:[multiHandLandmarks copy] + worldLandmarks:[multiHandWorldLandmarks copy] + timestampInMilliseconds:timestampInMilliseconds]; + + return gestureRecognizerResult; +} + +@end