cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

posetracking objective c api update

Browse Source
This commit is contained in:
Mautisim Munir 2022-10-16 20:06:04 +05:00
parent caf4256e00
commit fc2bcf67ec
7 changed files with 127 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
WORKSPACE
View File

@ -527,11 +527,11 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_ndk_repository",
android_sdk_repository(
name = "androidsdk",
build_tools_version = "30.0.3",
# path = "/Users/tj/Library/Android/sdk", # Path to Android SDK, optional if $ANDROID_HOME is set
path = "/Users/tj/Library/Android/sdk", # Path to Android SDK, optional if $ANDROID_HOME is set
)
android_ndk_repository(
name = "androidndk", # Required. Name *must* be "androidndk".
api_level = 21,
# path = "/Users/tj/Library/Android/sdk/ndk/21.4.7075529", # Optional. Can be omitted if `ANDROID_NDK_HOME` environment variable is set.
path = "/Users/tj/Library/Android/sdk/ndk/21.4.7075529", # Optional. Can be omitted if `ANDROID_NDK_HOME` environment variable is set.
)

12
mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen
View File

@ -23,7 +23,10 @@
"mediapipe/objc/testing/app/BUILD"
],
"buildTargets" : [
"//mediapipe/examples/ios/posetrackingsolutiongpu:PoseTrackingSolutionGpuApp"
"//mediapipe/examples/ios/posetrackingsolution-swift:app_lib",
"//mediapipe/examples/ios/posetrackingsolution-swift:posetracking-solution-swift",
"//mediapipe/examples/ios/posetrackingsolutiongpu:PoseTrackingSolutionGpuApp",
"//mediapipe/swift/solutions/lindera:Lindera"
],
"optionSet" : {
"BazelBuildOptionsDebug" : {
@ -91,6 +94,10 @@
"mediapipe/examples/ios/objectdetectioncpu",
"mediapipe/examples/ios/objectdetectiongpu",
"mediapipe/examples/ios/posetrackinggpu",
"mediapipe/examples/ios/posetrackingsolution-swift",
"mediapipe/examples/ios/posetrackingsolution-swift/Camera",
"mediapipe/examples/ios/posetrackingsolution-swift/ViewModels",
"mediapipe/examples/ios/posetrackingsolution-swift/Views",
"mediapipe/examples/ios/posetrackingsolutiongpu",
"mediapipe/examples/ios/posetrackingsolutiongpu/Base.lproj",
"mediapipe/examples/ios/selfiesegmentationgpu",
@ -117,6 +124,9 @@
"mediapipe/objc",
"mediapipe/objc/solutions",
"mediapipe/objc/solutions/posetracking_gpu",
"mediapipe/swift",
"mediapipe/swift/solutions",
"mediapipe/swift/solutions/lindera",
"mediapipe/util",
"mediapipe/util/android",
"mediapipe/util/android/file",

4
mediapipe/MediaPipe.tulsiproj/project.tulsiconf
View File

@ -25,10 +25,12 @@
"mediapipe/examples/ios/objectdetectiongpu",
"mediapipe/examples/ios/objectdetectiontrackinggpu",
"mediapipe/examples/ios/posetrackinggpu",
"mediapipe/examples/ios/posetrackingsolution-swift",
"mediapipe/examples/ios/posetrackingsolutiongpu",
"mediapipe/examples/ios/selfiesegmentationgpu",
"mediapipe/objc",
"mediapipe/objc/solutions/posetracking_gpu"
"mediapipe/objc/solutions/posetracking_gpu",
"mediapipe/swift/solutions/lindera"
],
"projectName" : "Mediapipe",
"workspaceRoot" : "../.."

1
mediapipe/objc/solutions/posetracking_gpu/BUILD
View File

@ -4,6 +4,7 @@ objc_library(
"*.h",
"*.mm",
]),
module_name = "MPPoseTracking",
hdrs = [
"PoseTracking.h",
"PoseTrackingOptions.h",

5
mediapipe/objc/solutions/posetracking_gpu/PoseTracking.h
View File

@ -6,17 +6,16 @@
#define MEDIAPIPE_POSETRACKING_H
#import <Foundation/Foundation.h>
#import "mediapipe/objc/MPPCameraInputSource.h"
#import "mediapipe/objc/MPPGraph.h"
#import "mediapipe/objc/MPPLayerRenderer.h"
#import "mediapipe/objc/MPPPlayerInputSource.h"
#import "mediapipe/objc/MPPTimestampConverter.h"
#import "PoseTrackingOptions.h"
#import "PoseTrackingResults.h"
@interface PoseTracking : NSObject<MPPGraphDelegate,MPPInputSourceDelegate>
@interface PoseTracking : NSObject<MPPInputSourceDelegate>
// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in
// viewWillAppear: and sent video frames on videoQueue.
@property(nonatomic) MPPGraph* mediapipeGraph;
//@property(nonatomic) MPPGraph* mediapipeGraph;
// Helps to convert timestamp.

154
mediapipe/objc/solutions/posetracking_gpu/PoseTracking.mm
View File

@ -1,9 +1,96 @@
#include "PoseTracking.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#import "mediapipe/objc/MPPGraph.h"
static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
static const char* kLandmarksOutputStream = "pose_landmarks";
# pragma mark - PoseTrackingGraphDelegate Interface
@interface PoseTrackingGraphDelegate : NSObject<MPPGraphDelegate>
// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread.
@property (nonatomic) MPPGraph* mediapipeGraph;
@property (nonatomic) const char* graphOutputStream;
@property (nonatomic) MPPLayerRenderer* renderer;
@property(nonatomic) void(^poseTrackingResultsListener)(PoseTrackingResults*);
-(id) initWithMediapipeGraph: (MPPGraph*) graph graphOutputStream: (const char*) graphOutputStream
renderer: (MPPLayerRenderer*) renderer;
- (void)mediapipeGraph:(MPPGraph*)graph
didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
fromStream:(const std::string&)streamName ;
- (void)mediapipeGraph:(MPPGraph*)graph
didOutputPacket:(const ::mediapipe::Packet&)packet
fromStream:(const std::string&)streamName ;
@end
# pragma mark - PoseTrackingGraphDelegate Implementation
@implementation PoseTrackingGraphDelegate
-(id) initWithMediapipeGraph: (MPPGraph*) graph graphOutputStream: (const char*) graphOutputStream
renderer: (MPPLayerRenderer*) renderer
{
self.mediapipeGraph = graph;
self.graphOutputStream =graphOutputStream;
self.renderer = renderer;
return self;
}
// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread.
- (void)mediapipeGraph:(MPPGraph*)graph
didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
fromStream:(const std::string&)streamName {
if (streamName == self.graphOutputStream) {
// Display the captured image on the screen.
CVPixelBufferRetain(pixelBuffer);
dispatch_async(dispatch_get_main_queue(), ^{
[self.renderer renderPixelBuffer:pixelBuffer];
CVPixelBufferRelease(pixelBuffer);
});
}
}
// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread.
- (void)mediapipeGraph:(MPPGraph*)graph
didOutputPacket:(const ::mediapipe::Packet&)packet
fromStream:(const std::string&)streamName {
if (streamName == kLandmarksOutputStream) {
if (packet.IsEmpty()) {
self.poseTrackingResultsListener(nil);
return;
}
const auto& landmarks = packet.Get<::mediapipe::NormalizedLandmarkList>();
NSMutableArray<PoseLandmark*>* poseLandmarks = [[NSMutableArray<PoseLandmark*> alloc] init];
for (int i = 0; i < landmarks.landmark_size(); ++i) {
[poseLandmarks addObject: [[PoseLandmark alloc] initWithX:landmarks.landmark(i).x() y:landmarks.landmark(i).y() z:landmarks.landmark(i).z() presence:landmarks.landmark(i).presence() visibility:landmarks.landmark(i).visibility()] ];
}
PoseTrackingResults* results = [[PoseTrackingResults alloc] initWithLandmarks:poseLandmarks];
self.poseTrackingResultsListener(results);
}
}
@end
@interface PoseTracking(){
// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in
// viewWillAppear: and sent video frames on videoQueue.
MPPGraph* mediapipeGraph;
PoseTrackingGraphDelegate* poseTrackingGraphDelegate;
}
@end
@implementation PoseTracking
#pragma mark - MediaPipe graph methods
@ -43,7 +130,7 @@ static const char* kLandmarksOutputStream = "pose_landmarks";
self.poseTrackingOptions = poseTrackingOptions;
self.graphName = @"pose_tracking_gpu";
self.mediapipeGraph = [[self class] loadGraphFromResource: self.graphName];
self->mediapipeGraph = [[self class] loadGraphFromResource: self.graphName];
self.graphInputStream = "input_video";
@ -53,19 +140,25 @@ static const char* kLandmarksOutputStream = "pose_landmarks";
self.graphOutputStream = "throttled_input_video";
}
[self.mediapipeGraph addFrameOutputStream:self.graphOutputStream
[self->mediapipeGraph addFrameOutputStream:self.graphOutputStream
outputPacketType:MPPPacketTypePixelBuffer];
[self.mediapipeGraph addFrameOutputStream:"pose_landmarks"
outputPacketType:MPPPacketTypeRaw];
self.mediapipeGraph.delegate = self;
self.poseTrackingResultsListener = ^(PoseTrackingResults*){};
[self->mediapipeGraph addFrameOutputStream:"pose_landmarks"
outputPacketType:MPPPacketTypeRaw];
self-> poseTrackingGraphDelegate = [[PoseTrackingGraphDelegate alloc] initWithMediapipeGraph:self->mediapipeGraph graphOutputStream:self.graphOutputStream renderer:self.renderer];
// To prevent ARC from causing an accidental memory leak in the next block
__weak PoseTracking* weakSelf = self;
self -> poseTrackingGraphDelegate.poseTrackingResultsListener = ^(PoseTrackingResults* results){
weakSelf.poseTrackingResultsListener(results);
};
self->mediapipeGraph.delegate = self->poseTrackingGraphDelegate;
return self;
}
@ -74,10 +167,10 @@ static const char* kLandmarksOutputStream = "pose_landmarks";
- (void)startGraph {
// Start running self.mediapipeGraph.
NSError* error;
if (![self.mediapipeGraph startWithError:&error]) {
if (![self->mediapipeGraph startWithError:&error]) {
NSLog(@"Failed to start graph: %@", error);
}
else if (![self.mediapipeGraph waitUntilIdleWithError:&error]) {
else if (![self->mediapipeGraph waitUntilIdleWithError:&error]) {
NSLog(@"Failed to complete graph initial run: %@", error);
}
}
@ -100,7 +193,7 @@ static const char* kLandmarksOutputStream = "pose_landmarks";
timestamp:(CMTime)timestamp
fromSource:(MPPInputSource*)source {
[self.mediapipeGraph sendPixelBuffer:imageBuffer
[self->mediapipeGraph sendPixelBuffer:imageBuffer
intoStream:self.graphInputStream
packetType:MPPPacketTypePixelBuffer
timestamp:[self.timestampConverter timestampForMediaTime:timestamp]];
@ -108,42 +201,5 @@ static const char* kLandmarksOutputStream = "pose_landmarks";
#pragma mark - MPPGraphDelegate methods
// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread.
- (void)mediapipeGraph:(MPPGraph*)graph
didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
fromStream:(const std::string&)streamName {
if (streamName == self.graphOutputStream) {
// Display the captured image on the screen.
CVPixelBufferRetain(pixelBuffer);
dispatch_async(dispatch_get_main_queue(), ^{
[self.renderer renderPixelBuffer:pixelBuffer];
CVPixelBufferRelease(pixelBuffer);
});
}
}
// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread.
- (void)mediapipeGraph:(MPPGraph*)graph
didOutputPacket:(const ::mediapipe::Packet&)packet
fromStream:(const std::string&)streamName {
if (streamName == kLandmarksOutputStream) {
if (packet.IsEmpty()) {
self.poseTrackingResultsListener(nil);
return;
}
const auto& landmarks = packet.Get<::mediapipe::NormalizedLandmarkList>();
NSMutableArray<PoseLandmark*>* poseLandmarks = [[NSMutableArray<PoseLandmark*> alloc] init];
for (int i = 0; i < landmarks.landmark_size(); ++i) {
[poseLandmarks addObject: [[PoseLandmark alloc] initWithX:landmarks.landmark(i).x() y:landmarks.landmark(i).y() z:landmarks.landmark(i).z() presence:landmarks.landmark(i).presence() visibility:landmarks.landmark(i).visibility()] ];
}
PoseTrackingResults* results = [[PoseTrackingResults alloc] initWithLandmarks:poseLandmarks];
self.poseTrackingResultsListener(results);
}
}
@end

3
mediapipe/swift/solutions/lindera/BUILD
View File

@ -3,4 +3,7 @@ load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
swift_library(
name = "Lindera",
srcs = glob(["*.swift"]),
deps = [
"//mediapipe/objc/solutions/posetracking_gpu:posetracking_gpu_solution",
]
)