diff --git a/mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen b/mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen index d99a41cba..cbb494369 100644 --- a/mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen +++ b/mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen @@ -23,6 +23,8 @@ "mediapipe/objc/testing/app/BUILD" ], "buildTargets" : [ + "//mediapipe/examples/ios/common:CommonMediaPipeAppLibrary", + "//mediapipe/examples/ios/posetrackingsolutiongpu:CommonMediaPipeAppLibrary", "//mediapipe/examples/ios/posetrackingsolutiongpu:PoseTrackingSolutionGpuApp" ], "optionSet" : { @@ -92,6 +94,7 @@ "mediapipe/examples/ios/objectdetectiongpu", "mediapipe/examples/ios/posetrackinggpu", "mediapipe/examples/ios/posetrackingsolutiongpu", + "mediapipe/examples/ios/posetrackingsolutiongpu/Base.lproj", "mediapipe/examples/ios/selfiesegmentationgpu", "mediapipe/framework", "mediapipe/framework/deps", diff --git a/mediapipe/MediaPipe.tulsiproj/project.tulsiconf b/mediapipe/MediaPipe.tulsiproj/project.tulsiconf index 8a6ebbc7d..6393f863b 100644 --- a/mediapipe/MediaPipe.tulsiproj/project.tulsiconf +++ b/mediapipe/MediaPipe.tulsiproj/project.tulsiconf @@ -2,7 +2,7 @@ "configDefaults" : { "optionSet" : { "BazelBuildOptionsDebug" : { - "p" : "--config=debug" + "p" : "--config=debug --strip=never --features=oso_prefix_is_pwd --apple_generate_dsym" }, "CLANG_CXX_LANGUAGE_STANDARD" : { "p" : "c++14" @@ -13,6 +13,7 @@ "", "mediapipe", "mediapipe/examples/ios", + "mediapipe/examples/ios/common", "mediapipe/examples/ios/facedetectioncpu", "mediapipe/examples/ios/facedetectiongpu", "mediapipe/examples/ios/faceeffect", diff --git a/mediapipe/examples/ios/posetrackingsolutiongpu/AppDelegate.mm b/mediapipe/examples/ios/posetrackingsolutiongpu/AppDelegate.mm index 3e08df40f..f46968e12 100644 --- a/mediapipe/examples/ios/posetrackingsolutiongpu/AppDelegate.mm +++ b/mediapipe/examples/ios/posetrackingsolutiongpu/AppDelegate.mm @@ -13,8 +13,7 @@ // limitations under the License. #import "AppDelegate.h" - -#import "CommonViewController.h" +#import "PoseTrackingViewController.h" @interface AppDelegate () diff --git a/mediapipe/examples/ios/posetrackingsolutiongpu/BUILD b/mediapipe/examples/ios/posetrackingsolutiongpu/BUILD index dbd69fad7..f1598415b 100644 --- a/mediapipe/examples/ios/posetrackingsolutiongpu/BUILD +++ b/mediapipe/examples/ios/posetrackingsolutiongpu/BUILD @@ -55,6 +55,10 @@ objc_library( name = "PoseTrackingGpuAppLibrary", srcs = [ "PoseTrackingViewController.mm", + ] + [ + "AppDelegate.mm", + "main.mm", + "AppDelegate.h", ], hdrs = [ "PoseTrackingViewController.h", @@ -64,10 +68,21 @@ objc_library( "//mediapipe/graphs/pose_tracking:pose_tracking_gpu.binarypb", "//mediapipe/modules/pose_detection:pose_detection.tflite", "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite", + ] + [ + "Base.lproj/LaunchScreen.storyboard", + "Base.lproj/Main.storyboard", + ], + sdk_frameworks = [ + "AVFoundation", + "CoreGraphics", + "CoreMedia", + "UIKit", ], deps = [ - ":CommonMediaPipeAppLibrary", "//mediapipe/objc/solutions/posetracking_gpu:posetracking_gpu_solution", + "//mediapipe/objc:mediapipe_framework_ios", + "//mediapipe/objc:mediapipe_input_sources_ios", + "//mediapipe/objc:mediapipe_layer_renderer", ] + select({ "//mediapipe:ios_i386": [], "//mediapipe:ios_x86_64": [], @@ -78,38 +93,34 @@ objc_library( }), ) -objc_library( - name = "CommonMediaPipeAppLibrary", - srcs = [ - "AppDelegate.mm", - "CommonViewController.mm", - "main.m", - ], - hdrs = [ - "AppDelegate.h", - "CommonViewController.h", - ], - data = [ - "Base.lproj/LaunchScreen.storyboard", - "Base.lproj/Main.storyboard", - ], - sdk_frameworks = [ - "AVFoundation", - "CoreGraphics", - "CoreMedia", - "UIKit", - ], - visibility = [ - "//mediapipe:__subpackages__", - ], - deps = [ - "//mediapipe/objc:mediapipe_framework_ios", - "//mediapipe/objc:mediapipe_input_sources_ios", - "//mediapipe/objc:mediapipe_layer_renderer", - ], -) - -exports_files(["Info.plist"]) +#objc_library( +# name = "CommonMediaPipeAppLibrary", +# srcs = [ +# "AppDelegate.mm", +# "main.m", +# ], +# hdrs = [ +# "AppDelegate.h", +# ], +# data = [ +# "Base.lproj/LaunchScreen.storyboard", +# "Base.lproj/Main.storyboard", +# ], +# sdk_frameworks = [ +# "AVFoundation", +# "CoreGraphics", +# "CoreMedia", +# "UIKit", +# ], +# visibility = [ +# "//mediapipe:__subpackages__", +# ], +# deps = [ +# "//mediapipe/objc:mediapipe_framework_ios", +# "//mediapipe/objc:mediapipe_input_sources_ios", +# "//mediapipe/objc:mediapipe_layer_renderer", +# ], +#) filegroup( name = "AppIcon", diff --git a/mediapipe/examples/ios/posetrackingsolutiongpu/Base.lproj/Main.storyboard b/mediapipe/examples/ios/posetrackingsolutiongpu/Base.lproj/Main.storyboard index fcf71c0e2..29a268b41 100644 --- a/mediapipe/examples/ios/posetrackingsolutiongpu/Base.lproj/Main.storyboard +++ b/mediapipe/examples/ios/posetrackingsolutiongpu/Base.lproj/Main.storyboard @@ -1,16 +1,17 @@ - + - + - + - + + @@ -18,30 +19,19 @@ - - - - + - - diff --git a/mediapipe/examples/ios/posetrackingsolutiongpu/CommonViewController.h b/mediapipe/examples/ios/posetrackingsolutiongpu/CommonViewController.h deleted file mode 100644 index d7cb1121a..000000000 --- a/mediapipe/examples/ios/posetrackingsolutiongpu/CommonViewController.h +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import - -#import "mediapipe/objc/MPPCameraInputSource.h" -#import "mediapipe/objc/MPPGraph.h" -#import "mediapipe/objc/MPPLayerRenderer.h" -#import "mediapipe/objc/MPPPlayerInputSource.h" -#import "mediapipe/objc/MPPTimestampConverter.h" - -typedef NS_ENUM(NSInteger, MediaPipeDemoSourceMode) { - MediaPipeDemoSourceCamera, - MediaPipeDemoSourceVideo -}; - -@interface CommonViewController : UIViewController - -// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in -// viewWillAppear: and sent video frames on videoQueue. -@property(nonatomic) MPPGraph* mediapipeGraph; - -// Handles camera access via AVCaptureSession library. -@property(nonatomic) MPPCameraInputSource* cameraSource; - -// Provides data from a video. -@property(nonatomic) MPPPlayerInputSource* videoSource; - -// Helps to convert timestamp. -@property(nonatomic) MPPTimestampConverter* timestampConverter; - -// The data source for the demo. -@property(nonatomic) MediaPipeDemoSourceMode sourceMode; - -// Inform the user when camera is unavailable. -@property(nonatomic) IBOutlet UILabel* noCameraLabel; - -// Display the camera preview frames. -@property(strong, nonatomic) IBOutlet UIView* liveView; - -// Render frames in a layer. -@property(nonatomic) MPPLayerRenderer* renderer; - -// Process camera frames on this queue. -@property(nonatomic) dispatch_queue_t videoQueue; - -// Graph name. -@property(nonatomic) NSString* graphName; - -// Graph input stream. -@property(nonatomic) const char* graphInputStream; - -// Graph output stream. -@property(nonatomic) const char* graphOutputStream; - -@end diff --git a/mediapipe/examples/ios/posetrackingsolutiongpu/CommonViewController.mm b/mediapipe/examples/ios/posetrackingsolutiongpu/CommonViewController.mm deleted file mode 100644 index f6c47eacf..000000000 --- a/mediapipe/examples/ios/posetrackingsolutiongpu/CommonViewController.mm +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import "CommonViewController.h" - -static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue"; - -@implementation CommonViewController - -// This provides a hook to replace the basic ViewController with a subclass when it's created from a -// storyboard, without having to change the storyboard itself. -+ (instancetype)allocWithZone:(struct _NSZone*)zone { - NSString* subclassName = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"MainViewController"]; - if (subclassName.length > 0) { - Class customClass = NSClassFromString(subclassName); - Class baseClass = [CommonViewController class]; - NSAssert([customClass isSubclassOfClass:baseClass], @"%@ must be a subclass of %@", customClass, - baseClass); - if (self == baseClass) return [customClass allocWithZone:zone]; - } - return [super allocWithZone:zone]; -} - -#pragma mark - Cleanup methods - -- (void)dealloc { - self.mediapipeGraph.delegate = nil; - [self.mediapipeGraph cancel]; - // Ignore errors since we're cleaning up. - [self.mediapipeGraph closeAllInputStreamsWithError:nil]; - [self.mediapipeGraph waitUntilDoneWithError:nil]; -} - -#pragma mark - MediaPipe graph methods - -+ (MPPGraph*)loadGraphFromResource:(NSString*)resource { - // Load the graph config resource. - NSError* configLoadError = nil; - NSBundle* bundle = [NSBundle bundleForClass:[self class]]; - if (!resource || resource.length == 0) { - return nil; - } - NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"]; - NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError]; - if (!data) { - NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError); - return nil; - } - - // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object. - mediapipe::CalculatorGraphConfig config; - config.ParseFromArray(data.bytes, data.length); - - // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object. - MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config]; - return newGraph; -} - -#pragma mark - UIViewController methods - -- (void)viewDidLoad { - [super viewDidLoad]; - - self.renderer = [[MPPLayerRenderer alloc] init]; - self.renderer.layer.frame = self.liveView.layer.bounds; - [self.liveView.layer addSublayer:self.renderer.layer]; - self.renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop; - - self.timestampConverter = [[MPPTimestampConverter alloc] init]; - - dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class( - DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0); - self.videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute); - - self.graphName = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"GraphName"]; - self.graphInputStream = - [[[NSBundle mainBundle] objectForInfoDictionaryKey:@"GraphInputStream"] UTF8String]; - self.graphOutputStream = - [[[NSBundle mainBundle] objectForInfoDictionaryKey:@"GraphOutputStream"] UTF8String]; - - self.mediapipeGraph = [[self class] loadGraphFromResource:self.graphName]; - [self.mediapipeGraph addFrameOutputStream:self.graphOutputStream - outputPacketType:MPPPacketTypePixelBuffer]; - - self.mediapipeGraph.delegate = self; -} - -// In this application, there is only one ViewController which has no navigation to other view -// controllers, and there is only one View with live display showing the result of running the -// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph -// setup/teardown and camera start/stop logic should be updated appropriately in response to the -// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times -// depending on the application navigation flow in that case. -- (void)viewWillAppear:(BOOL)animated { - [super viewWillAppear:animated]; - - switch (self.sourceMode) { - case MediaPipeDemoSourceVideo: { - NSString* videoName = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"VideoName"]; - AVAsset* video = [AVAsset assetWithURL:[[NSBundle mainBundle] URLForResource:videoName - withExtension:@"mov"]]; - self.videoSource = [[MPPPlayerInputSource alloc] initWithAVAsset:video]; - [self.videoSource setDelegate:self queue:self.videoQueue]; - dispatch_async(self.videoQueue, ^{ - [self.videoSource start]; - }); - break; - } - case MediaPipeDemoSourceCamera: { - self.cameraSource = [[MPPCameraInputSource alloc] init]; - [self.cameraSource setDelegate:self queue:self.videoQueue]; - self.cameraSource.sessionPreset = AVCaptureSessionPresetHigh; - - NSString* cameraPosition = - [[NSBundle mainBundle] objectForInfoDictionaryKey:@"CameraPosition"]; - if (cameraPosition.length > 0 && [cameraPosition isEqualToString:@"back"]) { - self.cameraSource.cameraPosition = AVCaptureDevicePositionBack; - } else { - self.cameraSource.cameraPosition = AVCaptureDevicePositionFront; - // When using the front camera, mirror the input for a more natural look. - _cameraSource.videoMirrored = YES; - } - - // The frame's native format is rotated with respect to the portrait orientation. - _cameraSource.orientation = AVCaptureVideoOrientationPortrait; - - [self.cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) { - if (granted) { - dispatch_async(dispatch_get_main_queue(), ^{ - self.noCameraLabel.hidden = YES; - }); - [self startGraphAndCamera]; - } - }]; - - break; - } - } -} - -- (void)startGraphAndCamera { - // Start running self.mediapipeGraph. - NSError* error; - if (![self.mediapipeGraph startWithError:&error]) { - NSLog(@"Failed to start graph: %@", error); - } - else if (![self.mediapipeGraph waitUntilIdleWithError:&error]) { - NSLog(@"Failed to complete graph initial run: %@", error); - } - - // Start fetching frames from the camera. - dispatch_async(self.videoQueue, ^{ - [self.cameraSource start]; - }); -} - -#pragma mark - MPPInputSourceDelegate methods - -// Must be invoked on self.videoQueue. -- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer - timestamp:(CMTime)timestamp - fromSource:(MPPInputSource*)source { - if (source != self.cameraSource && source != self.videoSource) { - NSLog(@"Unknown source: %@", source); - return; - } - - [self.mediapipeGraph sendPixelBuffer:imageBuffer - intoStream:self.graphInputStream - packetType:MPPPacketTypePixelBuffer - timestamp:[self.timestampConverter timestampForMediaTime:timestamp]]; -} - -#pragma mark - MPPGraphDelegate methods - -// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread. -- (void)mediapipeGraph:(MPPGraph*)graph - didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer - fromStream:(const std::string&)streamName { - if (streamName == self.graphOutputStream) { - // Display the captured image on the screen. - CVPixelBufferRetain(pixelBuffer); - dispatch_async(dispatch_get_main_queue(), ^{ - [self.renderer renderPixelBuffer:pixelBuffer]; - CVPixelBufferRelease(pixelBuffer); - }); - } -} - -@end diff --git a/mediapipe/examples/ios/posetrackingsolutiongpu/PoseTrackingViewController.h b/mediapipe/examples/ios/posetrackingsolutiongpu/PoseTrackingViewController.h index 3bcfa7391..649bbd9df 100644 --- a/mediapipe/examples/ios/posetrackingsolutiongpu/PoseTrackingViewController.h +++ b/mediapipe/examples/ios/posetrackingsolutiongpu/PoseTrackingViewController.h @@ -13,9 +13,13 @@ // limitations under the License. #import +#import "mediapipe/objc/MPPCameraInputSource.h" +#import "mediapipe/objc/solutions/posetracking_gpu/PoseTracking.h" -#import "CommonViewController.h" - -@interface PoseTrackingViewController : CommonViewController +@interface PoseTrackingViewController : UIViewController +@property(strong, nonatomic) IBOutlet UIView* liveView; +// Handles camera access via AVCaptureSession library. +@property(nonatomic) MPPCameraInputSource* cameraSource; +@property(nonatomic) PoseTracking* poseTracking; @end diff --git a/mediapipe/examples/ios/posetrackingsolutiongpu/PoseTrackingViewController.mm b/mediapipe/examples/ios/posetrackingsolutiongpu/PoseTrackingViewController.mm index c80f19283..a368fa68f 100644 --- a/mediapipe/examples/ios/posetrackingsolutiongpu/PoseTrackingViewController.mm +++ b/mediapipe/examples/ios/posetrackingsolutiongpu/PoseTrackingViewController.mm @@ -16,6 +16,7 @@ #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/objc/solutions/posetracking_gpu/PoseTrackingOptions.h" +#include "mediapipe/objc/solutions/posetracking_gpu/PoseTracking.h" static const char* kLandmarksOutputStream = "pose_landmarks"; @@ -23,41 +24,64 @@ static const char* kLandmarksOutputStream = "pose_landmarks"; #pragma mark - UIViewController methods + - (void)viewDidLoad { [super viewDidLoad]; PoseTrackingOptions* options = [ [PoseTrackingOptions alloc] initWithShowLandmarks:true cameraRotation:0]; - [self.mediapipeGraph addFrameOutputStream:kLandmarksOutputStream - outputPacketType:MPPPacketTypeRaw]; - [self.mediapipeGraph addFrameOutputStream:"throttled_input_video" - outputPacketType:MPPPacketTypePixelBuffer]; - if (options.showLandmarks){ - self.graphOutputStream = "output_video"; - }else{ - self.graphOutputStream = "throttled_input_video"; - } + self.poseTracking = [[PoseTracking alloc] initWithPoseTrackingOptions:options]; + + self.poseTracking.renderer.layer.frame = self.liveView.layer.bounds; + [self.liveView.layer addSublayer:self.poseTracking.renderer.layer]; + + + + + + + + } -#pragma mark - MPPGraphDelegate methods -// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread. -- (void)mediapipeGraph:(MPPGraph*)graph - didOutputPacket:(const ::mediapipe::Packet&)packet - fromStream:(const std::string&)streamName { - if (streamName == kLandmarksOutputStream) { - if (packet.IsEmpty()) { - NSLog(@"[TS:%lld] No pose landmarks", packet.Timestamp().Value()); - return; - } - const auto& landmarks = packet.Get<::mediapipe::NormalizedLandmarkList>(); - NSLog(@"[TS:%lld] Number of pose landmarks: %d", packet.Timestamp().Value(), - landmarks.landmark_size()); - for (int i = 0; i < landmarks.landmark_size(); ++i) { - NSLog(@"\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(), - landmarks.landmark(i).y(), landmarks.landmark(i).z()); - } - } +// In this application, there is only one ViewController which has no navigation to other view +// controllers, and there is only one View with live display showing the result of running the +// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph +// setup/teardown and camera start/stop logic should be updated appropriately in response to the +// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times +// depending on the application navigation flow in that case. +- (void)viewWillAppear:(BOOL)animated { + [super viewWillAppear:animated]; + + self.cameraSource = [[MPPCameraInputSource alloc] init]; + [self.cameraSource setDelegate:self.poseTracking queue:self.poseTracking.videoQueue]; + self.cameraSource.sessionPreset = AVCaptureSessionPresetHigh; + + + self.cameraSource.cameraPosition = AVCaptureDevicePositionBack; + +// self.cameraSource.cameraPosition = AVCaptureDevicePositionFront; +// // When using the front camera, mirror the input for a more natural look. +// _cameraSource.videoMirrored = YES; + + + // The frame's native format is rotated with respect to the portrait orientation. + _cameraSource.orientation = AVCaptureVideoOrientationPortrait; + + [self.cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) { + if (granted) { + + [self.poseTracking startWithCamera:self.cameraSource]; + } + }]; + } +//#pragma mark - MPPGraphDelegate methods +// + + + + @end diff --git a/mediapipe/examples/ios/posetrackingsolutiongpu/main.m b/mediapipe/examples/ios/posetrackingsolutiongpu/main.mm similarity index 100% rename from mediapipe/examples/ios/posetrackingsolutiongpu/main.m rename to mediapipe/examples/ios/posetrackingsolutiongpu/main.mm diff --git a/mediapipe/objc/solutions/posetracking_gpu/BUILD b/mediapipe/objc/solutions/posetracking_gpu/BUILD index 1bc83e979..4f2841cb5 100644 --- a/mediapipe/objc/solutions/posetracking_gpu/BUILD +++ b/mediapipe/objc/solutions/posetracking_gpu/BUILD @@ -4,11 +4,25 @@ objc_library( "*.h", "*.mm", ]), - hdrs = ["PoseTrackingOptions.h"], + hdrs = [ + "PoseTracking.h", + "PoseTrackingOptions.h", + ], copts = [ "-Wno-shorten-64-to-32", ], sdk_frameworks = ["Accelerate"], - # This build rule is public to allow external customers to build their own iOS apps. visibility = ["//visibility:public"], + deps = [ + "//mediapipe/objc:mediapipe_framework_ios", + "//mediapipe/objc:mediapipe_input_sources_ios", + "//mediapipe/objc:mediapipe_layer_renderer", + ] + select({ + "//mediapipe:ios_i386": [], + "//mediapipe:ios_x86_64": [], + "//conditions:default": [ + "//mediapipe/graphs/pose_tracking:pose_tracking_gpu_deps", + "//mediapipe/framework/formats:landmark_cc_proto", + ], + }), ) diff --git a/mediapipe/objc/solutions/posetracking_gpu/PoseTracking.h b/mediapipe/objc/solutions/posetracking_gpu/PoseTracking.h new file mode 100644 index 000000000..9cfe546b6 --- /dev/null +++ b/mediapipe/objc/solutions/posetracking_gpu/PoseTracking.h @@ -0,0 +1,49 @@ +// +// Created by Mautisim Munir on 05/10/2022. +// + +#ifndef MEDIAPIPE_POSETRACKING_H +#define MEDIAPIPE_POSETRACKING_H +#import +#import "mediapipe/objc/MPPCameraInputSource.h" +#import "mediapipe/objc/MPPGraph.h" +#import "mediapipe/objc/MPPLayerRenderer.h" +#import "mediapipe/objc/MPPPlayerInputSource.h" +#import "mediapipe/objc/MPPTimestampConverter.h" +#import "PoseTrackingOptions.h" +@interface PoseTracking : NSObject + +// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in +// viewWillAppear: and sent video frames on videoQueue. +@property(nonatomic) MPPGraph* mediapipeGraph; + + +// Helps to convert timestamp. +@property(nonatomic) MPPTimestampConverter* timestampConverter; + +// Render frames in a layer. +@property(nonatomic) MPPLayerRenderer* renderer; + + +// Graph name. +@property(nonatomic) NSString* graphName; + +// Graph input stream. +@property(nonatomic) const char* graphInputStream; + +// Graph output stream. +@property(nonatomic) const char* graphOutputStream; + +// Modify graph options +@property(nonatomic) PoseTrackingOptions* poseTrackingOptions; + + +// Process camera frames on this queue. +@property(nonatomic) dispatch_queue_t videoQueue; + +- (instancetype) initWithPoseTrackingOptions: (PoseTrackingOptions*) poseTrackingOptions; +- (void) startWithCamera: (MPPCameraInputSource*) cameraSource; +@end + + +#endif //MEDIAPIPE_POSETRACKING_H diff --git a/mediapipe/objc/solutions/posetracking_gpu/PoseTracking.mm b/mediapipe/objc/solutions/posetracking_gpu/PoseTracking.mm new file mode 100644 index 000000000..8403c1187 --- /dev/null +++ b/mediapipe/objc/solutions/posetracking_gpu/PoseTracking.mm @@ -0,0 +1,141 @@ +#include "PoseTracking.h" +#include "mediapipe/framework/formats/landmark.pb.h" + +static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue"; +static const char* kLandmarksOutputStream = "pose_landmarks"; + +@implementation PoseTracking + +#pragma mark - MediaPipe graph methods + ++ (MPPGraph*)loadGraphFromResource:(NSString*)resource { + // Load the graph config resource. + NSError* configLoadError = nil; + NSBundle* bundle = [NSBundle bundleForClass:[self class]]; + if (!resource || resource.length == 0) { + return nil; + } + NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"]; + NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError]; + if (!data) { + NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError); + return nil; + } + + // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object. + mediapipe::CalculatorGraphConfig config; + config.ParseFromArray(data.bytes, data.length); + + // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object. + MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config]; + return newGraph; +} + +- (instancetype) initWithPoseTrackingOptions: (PoseTrackingOptions*) poseTrackingOptions{ + self.renderer = [[MPPLayerRenderer alloc] init]; + self.renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop; + + self.timestampConverter = [[MPPTimestampConverter alloc] init]; + + dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class( + DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0); + self.videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute); + + self.poseTrackingOptions = poseTrackingOptions; + self.graphName = @"pose_tracking_gpu"; + self.mediapipeGraph = [[self class] loadGraphFromResource: self.graphName]; + self.graphInputStream = "input_video"; + + if (poseTrackingOptions.showLandmarks){ + self.graphOutputStream = "output_video"; + }else{ + self.graphOutputStream = "throttled_input_video"; + } + + [self.mediapipeGraph addFrameOutputStream:self.graphOutputStream + outputPacketType:MPPPacketTypePixelBuffer]; + + + + [self.mediapipeGraph addFrameOutputStream:"pose_landmarks" + outputPacketType:MPPPacketTypeRaw]; + + self.mediapipeGraph.delegate = self; + + + + + return self; +} + + + +- (void)startGraph { + // Start running self.mediapipeGraph. + NSError* error; + if (![self.mediapipeGraph startWithError:&error]) { + NSLog(@"Failed to start graph: %@", error); + } + else if (![self.mediapipeGraph waitUntilIdleWithError:&error]) { + NSLog(@"Failed to complete graph initial run: %@", error); + } +} + +- (void) startWithCamera: (MPPCameraInputSource*) cameraSource { + [self startGraph]; + // Start fetching frames from the camera. + dispatch_async(self.videoQueue, ^{ + [cameraSource start]; + }); +} + + +#pragma mark - MPPInputSourceDelegate methods + +// Must be invoked on self.videoQueue. +- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer + timestamp:(CMTime)timestamp + fromSource:(MPPInputSource*)source { + + [self.mediapipeGraph sendPixelBuffer:imageBuffer + intoStream:self.graphInputStream + packetType:MPPPacketTypePixelBuffer + timestamp:[self.timestampConverter timestampForMediaTime:timestamp]]; +} + +#pragma mark - MPPGraphDelegate methods + +// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread. +- (void)mediapipeGraph:(MPPGraph*)graph + didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer + fromStream:(const std::string&)streamName { + if (streamName == self.graphOutputStream) { + // Display the captured image on the screen. + CVPixelBufferRetain(pixelBuffer); + dispatch_async(dispatch_get_main_queue(), ^{ + [self.renderer renderPixelBuffer:pixelBuffer]; + CVPixelBufferRelease(pixelBuffer); + }); + } +} + + +// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread. +- (void)mediapipeGraph:(MPPGraph*)graph + didOutputPacket:(const ::mediapipe::Packet&)packet + fromStream:(const std::string&)streamName { + if (streamName == kLandmarksOutputStream) { + if (packet.IsEmpty()) { + NSLog(@"[TS:%lld] No pose landmarks", packet.Timestamp().Value()); + return; + } + const auto& landmarks = packet.Get<::mediapipe::NormalizedLandmarkList>(); + NSLog(@"[TS:%lld] Number of pose landmarks: %d", packet.Timestamp().Value(), + landmarks.landmark_size()); + for (int i = 0; i < landmarks.landmark_size(); ++i) { + NSLog(@"\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(), + landmarks.landmark(i).y(), landmarks.landmark(i).z()); + } + } +} +@end diff --git a/mediapipe/objc/solutions/posetracking_gpu/PoseTrackingOptions.h b/mediapipe/objc/solutions/posetracking_gpu/PoseTrackingOptions.h index 204640aee..34a181097 100644 --- a/mediapipe/objc/solutions/posetracking_gpu/PoseTrackingOptions.h +++ b/mediapipe/objc/solutions/posetracking_gpu/PoseTrackingOptions.h @@ -4,8 +4,8 @@ #ifndef MEDIAPIPE_POSETRACKINGOPTIONS_H #define MEDIAPIPE_POSETRACKINGOPTIONS_H - -@interface PoseTrackingOptions +#import +@interface PoseTrackingOptions: NSObject @property(nonatomic) bool showLandmarks; @property(nonatomic) int cameraRotation;