From dbeea8069e6a5354e614240b8f21a75a97c664da Mon Sep 17 00:00:00 2001 From: "vladimir.borisov" Date: Thu, 9 Jun 2022 20:10:19 +0400 Subject: [PATCH] add framework builder --- mediapipe/examples/ios/facemeshgpu/BUILD | 74 +++++- .../ios/facemeshgpu/MediaPipeController.h | 44 ++++ .../ios/facemeshgpu/MediaPipeController.mm | 241 ++++++++++++++++++ .../ios/facemeshgpu/build_mediapipe.sh | 19 ++ .../ios/facemeshgpu/patch_ios_framework.sh | 66 +++++ mediapipe/util/annotation_renderer.cc | 4 +- mediapipe/util/annotation_renderer.h | 2 +- 7 files changed, 446 insertions(+), 4 deletions(-) create mode 100644 mediapipe/examples/ios/facemeshgpu/MediaPipeController.h create mode 100644 mediapipe/examples/ios/facemeshgpu/MediaPipeController.mm create mode 100755 mediapipe/examples/ios/facemeshgpu/build_mediapipe.sh create mode 100755 mediapipe/examples/ios/facemeshgpu/patch_ios_framework.sh diff --git a/mediapipe/examples/ios/facemeshgpu/BUILD b/mediapipe/examples/ios/facemeshgpu/BUILD index 02103ce2f..456b11546 100644 --- a/mediapipe/examples/ios/facemeshgpu/BUILD +++ b/mediapipe/examples/ios/facemeshgpu/BUILD @@ -15,13 +15,23 @@ load( "@build_bazel_rules_apple//apple:ios.bzl", "ios_application", -) + "ios_framework") +load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library") load( "//mediapipe/examples/ios:bundle_id.bzl", "BUNDLE_ID_PREFIX", "example_provisioning", ) +FRAMEWORK_HEADERS = [ + "MediaPipeController.h", +] + +IOS_FAMILIES = [ + "iphone", + "ipad", +] + licenses(["notice"]) MIN_IOS_VERSION = "11.0" @@ -51,6 +61,68 @@ ios_application( ], ) +ios_framework( + name = "MediaPipeFramework", + hdrs = FRAMEWORK_HEADERS, + bundle_id = "dh.MediaPipeFramework", + bundle_name = "MediaPipeFramework", + families = IOS_FAMILIES, + infoplists = [ + "//mediapipe/examples/ios/common:Info.plist", +# "Info.plist", + ], + minimum_os_version = MIN_IOS_VERSION, + visibility = ["//visibility:public"], + deps = [ + ":MediaPipeLib", + "@ios_opencv//:OpencvFramework", + ], +) + + +objc_library( + name = "MediaPipeLib", + srcs = [ + "MediaPipeController.mm", + ], + hdrs = FRAMEWORK_HEADERS, + copts = ["-std=c++17"], # https://github.com/google/mediapipe/issues/2275#issuecomment-877145926 + data = [ + "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", + "//mediapipe/graphs/face_effect/data:axis.binarypb", + "//mediapipe/graphs/face_effect/data:axis.pngblob", + "//mediapipe/graphs/face_effect/data:facepaint.pngblob", + "//mediapipe/graphs/face_effect/data:glasses.binarypb", + "//mediapipe/graphs/face_effect/data:glasses.pngblob", + "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb", + "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_detection.binarypb", + "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_landmarks.binarypb", + "//mediapipe/modules/face_landmark:face_landmark.tflite", + "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb", + "//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite", + ], + deps = [ + "//mediapipe/objc:mediapipe_framework_ios", + "//mediapipe/objc:mediapipe_input_sources_ios", + "//mediapipe/calculators/core:packet_presence_calculator", +# "//mediapipe/objc:mediapipe_layer_renderer", # no need for layer renderer since I don't render + ] + select({ +# "//mediapipe:ios_i386": [], +# "//mediapipe:ios_x86_64": [], + "//conditions:default": [ + "//mediapipe/framework/formats:matrix_data_cc_proto", + "//mediapipe/graphs/face_effect:face_effect_gpu_deps", + "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto", + "//mediapipe/graphs/face_mesh:mobile_calculators", + "//mediapipe/framework/formats:landmark_cc_proto", +# "//mediapipe/examples/facial_search/graphs:gpu_calculators", +# "//mediapipe/examples/facial_search:embeddings_database", + ], + }), +) + + objc_library( name = "FaceMeshGpuAppLibrary", srcs = [ diff --git a/mediapipe/examples/ios/facemeshgpu/MediaPipeController.h b/mediapipe/examples/ios/facemeshgpu/MediaPipeController.h new file mode 100644 index 000000000..f76ec2719 --- /dev/null +++ b/mediapipe/examples/ios/facemeshgpu/MediaPipeController.h @@ -0,0 +1,44 @@ +#import +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +@class MediaPipeController; +@class MediaPipeFaceLandmarkPoint; +@class MediaPipeNormalizedRect; + +typedef void (^MediaPipeCompletionBlock)(CVPixelBufferRef pixelBuffer); + +@protocol MediaPipeControllerDelegate +@optional +- (void)mediaPipeController:(MediaPipeController *)controller didReceiveFaces:(NSArray*>*)faces; +- (void)mediaPipeController:(MediaPipeController *)controller didReceiveFaceBoxes:(NSArray*)faces; +- (void)mediaPipeController:(MediaPipeController *)controller didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer; +@end + +@interface MediaPipeController : NSObject + ++ (instancetype)facemesh; ++ (instancetype)effects; + +- (void)startGraph; +- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer timestamp:(CMTime)timestamp completion:(nullable MediaPipeCompletionBlock)completion; +@property (nullable, weak, nonatomic) id delegate; +@end + +@interface MediaPipeFaceLandmarkPoint : NSObject +@property (nonatomic) float x; +@property (nonatomic) float y; +@property (nonatomic) float z; +@end + +@interface MediaPipeNormalizedRect : NSObject +@property (nonatomic) float centerX; +@property (nonatomic) float centerY; +@property (nonatomic) float height; +@property (nonatomic) float width; +@property (nonatomic) float rotation; +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/examples/ios/facemeshgpu/MediaPipeController.mm b/mediapipe/examples/ios/facemeshgpu/MediaPipeController.mm new file mode 100644 index 000000000..5ef99d52f --- /dev/null +++ b/mediapipe/examples/ios/facemeshgpu/MediaPipeController.mm @@ -0,0 +1,241 @@ +#import "MediaPipeController.h" +#import "mediapipe/objc/MPPCameraInputSource.h" +#import "mediapipe/objc/MPPGraph.h" + +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/formats/detection.pb.h" + +//#import "mediapipe/objc/MPPLayerRenderer.h" + +static NSString* const kMeshGraphName = @"face_mesh_mobile_gpu"; +static NSString* const kEffectsGraphName = @"face_effect_gpu"; + +static const char *kInputStream = "input_video"; +static const char *kOutputStream = "output_video"; +static const char *kNumFacesInputSidePacket = "num_faces"; +static const char *kLandmarksOutputStream = "multi_face_landmarks"; +static const char *kFaceRectsOutputStream = "face_rects_from_landmarks"; +static const char *kLandmarkPresenceOutputStream = "landmark_presence"; +static const char *kSelectedEffectIdInputStream = "selected_effect_id"; +static const char *kMultiFaceGeometryStream = "multi_face_geometry"; +static const char* kUseFaceDetectionInputSourceInputSidePacket = "use_face_detection_input_source"; +static const BOOL kUseFaceDetectionInputSource = NO; + +// Max number of faces to detect/process. +static const int kNumFaces = 2; + +@interface MediaPipeController () +@property (nonatomic) MPPGraph* mediapipeGraph; +@property (nonatomic, copy) MediaPipeCompletionBlock completion; +@property (nonatomic) size_t timestamp; +@end + +@implementation MediaPipeController + +#pragma mark - Cleanup methods + +- (void)dealloc { + self.mediapipeGraph.delegate = nil; + [self.mediapipeGraph cancel]; + // Ignore errors since we're cleaning up. + [self.mediapipeGraph closeAllInputStreamsWithError:nil]; + [self.mediapipeGraph waitUntilDoneWithError:nil]; + + NSLog(@"dealloc MediaPipeController"); +} + +#pragma mark - MediaPipe graph methods + ++ (MPPGraph*)loadGraphFromResource:(NSString*)resource { + // Load the graph config resource. + NSError* configLoadError = nil; + NSBundle* bundle = [NSBundle bundleForClass:[self class]]; + if (!resource || resource.length == 0) { + bundle = NSBundle.mainBundle; + } + NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"]; + NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError]; + if (!data) { + NSLog(@"MediaPipe: Failed to load graph config: %@", configLoadError); + return nil; + } + + mediapipe::CalculatorGraphConfig config; + config.ParseFromArray(data.bytes, data.length); + + MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config]; + [newGraph setSidePacket:(mediapipe::MakePacket(kUseFaceDetectionInputSource)) named:kUseFaceDetectionInputSourceInputSidePacket]; + [newGraph setSidePacket:(mediapipe::MakePacket(kNumFaces)) named:kNumFacesInputSidePacket]; + + [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer]; + //[newGraph addFrameOutputStream:kMultiFaceGeometryStream outputPacketType:MPPPacketTypeRaw]; + +// [newGraph addFrameOutputStream:kLandmarksOutputStream outputPacketType:MPPPacketTypeRaw]; +// [newGraph addFrameOutputStream:kFaceRectsOutputStream outputPacketType:MPPPacketTypeRaw]; +// [newGraph addFrameOutputStream:kLandmarkPresenceOutputStream outputPacketType:MPPPacketTypeRaw]; + return newGraph; +} + +- (instancetype)initWithGraphName:(NSString *)graphName { + self = [super init]; + if (self) { + self.mediapipeGraph = [[self class] loadGraphFromResource:graphName]; + self.mediapipeGraph.delegate = self; + + // Set maxFramesInFlight to a small value to avoid memory contention for real-time processing. + self.mediapipeGraph.maxFramesInFlight = 2; + NSLog(@"MediaPipe: Inited graph %@", graphName); + NSLog(@"alloc MediaPipeController"); + } + return self; +} + ++ (instancetype)facemesh { + return [[MediaPipeController alloc] initWithGraphName:kMeshGraphName]; +} + ++ (instancetype)effects { + return [[MediaPipeController alloc] initWithGraphName:kEffectsGraphName]; +} + +- (void)startGraph { + NSError* error; + if (![self.mediapipeGraph startWithError:&error]) { + NSLog(@"MediaPipe: Failed to start graph: %@", error); + } + NSLog(@"MediaPipe: Started graph"); +} + +#pragma mark - MPPGraphDelegate methods + +- (void)mediapipeGraph:(MPPGraph*)graph + didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer + fromStream:(const std::string&)streamName { + //NSLog(@"MediaPipe: didOutputPixelBuffer %s %@", streamName.c_str(), self.completion); + if (streamName == kOutputStream) { + if([self.delegate respondsToSelector:@selector(mediaPipeController:didOutputPixelBuffer:)]) { + [_delegate mediaPipeController:self didOutputPixelBuffer:pixelBuffer]; + } + if (self.completion) { + self.completion(pixelBuffer); + } + } +} + +- (void)mediapipeGraph:(MPPGraph*)graph + didOutputPacket:(const ::mediapipe::Packet&)packet + fromStream:(const std::string&)streamName { + if (streamName == kLandmarksOutputStream) { + if (packet.IsEmpty()) { + return; + } + if(![self.delegate respondsToSelector:@selector(mediaPipeController:didReceiveFaces:)]) { + return; + } + const auto& multi_face_landmarks = packet.Get>(); + // NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(), + // multi_face_landmarks.size()); + NSMutableArray *>*faceLandmarks = [NSMutableArray new]; + + for (int face_index = 0; face_index < multi_face_landmarks.size(); ++face_index) { + NSMutableArray *thisFaceLandmarks = [NSMutableArray new]; + const auto& landmarks = multi_face_landmarks[face_index]; + // NSLog(@"\tNumber of landmarks for face[%d]: %d", face_index, landmarks.landmark_size()); + for (int i = 0; i < landmarks.landmark_size(); ++i) { + // NSLog(@"\t\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(), + // landmarks.landmark(i).y(), landmarks.landmark(i).z()); + MediaPipeFaceLandmarkPoint *obj_landmark = [MediaPipeFaceLandmarkPoint new]; + obj_landmark.x = landmarks.landmark(i).x(); + obj_landmark.y = landmarks.landmark(i).y(); + obj_landmark.z = landmarks.landmark(i).z(); + [thisFaceLandmarks addObject:obj_landmark]; + } + [faceLandmarks addObject:thisFaceLandmarks]; + } + [self.delegate mediaPipeController:self didReceiveFaces:faceLandmarks]; + } + + else if (streamName == kFaceRectsOutputStream) { + if (packet.IsEmpty()) { // This condition never gets called because FaceLandmarkFrontGpu does not process when there are no detections + // NSLog(@"[TS:%lld] No face rects", packet.Timestamp().Value()); + if([self.delegate respondsToSelector:@selector(mediaPipeController:didReceiveFaceBoxes:)]) { + [self.delegate mediaPipeController:self didReceiveFaceBoxes:@[]]; + } + return; + } + if(![self.delegate respondsToSelector:@selector(mediaPipeController:didReceiveFaceBoxes:)]) { + return; + } + const auto& face_rects_from_landmarks = packet.Get>(); + NSMutableArray *outRects = [NSMutableArray new]; + for (int face_index = 0; face_index < face_rects_from_landmarks.size(); ++face_index) { + const auto& face = face_rects_from_landmarks[face_index]; + float centerX = face.x_center(); + float centerY = face.y_center(); + float height = face.height(); + float width = face.width(); + float rotation = face.rotation(); + MediaPipeNormalizedRect *rect = [MediaPipeNormalizedRect new]; + rect.centerX = centerX; rect.centerY = centerY; rect.height = height; rect.width = width; rect.rotation = rotation; + [outRects addObject:rect]; + } + [self.delegate mediaPipeController:self didReceiveFaceBoxes:outRects]; + } else if (streamName == kLandmarkPresenceOutputStream) { + bool is_landmark_present = true; + if (packet.IsEmpty()) { + is_landmark_present = false; + } else { + is_landmark_present = packet.Get(); + } + + if (is_landmark_present) { + } else { + // NSLog(@"Landmarks not present"); + if([self.delegate respondsToSelector:@selector(mediaPipeController:didReceiveFaceBoxes:)]) { + [self.delegate mediaPipeController:self didReceiveFaceBoxes:@[]]; + } + if([self.delegate respondsToSelector:@selector(mediaPipeController:didReceiveFaces:)]) { + [self.delegate mediaPipeController:self didReceiveFaces:@[]]; + } + } + } else { + //NSLog(@"MediaPipe: Unknown %@ packet with stream name %s", packet.IsEmpty() ? @"EMPTY" : @"NON-EMPTY",streamName.c_str()); + } +} + + +#pragma mark - MPPInputSourceDelegate methods + +- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer timestamp:(CMTime)timestamp completion:(MediaPipeCompletionBlock)completion { + const auto ts = mediapipe::Timestamp(self.timestamp++ * mediapipe::Timestamp::kTimestampUnitsPerSecond); + self.completion = completion; + + NSError* err = nil; + //NSLog(@"sending imageBuffer @%@ to %s", @(ts.DebugString().c_str()), kInputStream); + auto sent = [self.mediapipeGraph sendPixelBuffer:imageBuffer + intoStream:kInputStream + packetType:MPPPacketTypePixelBuffer + timestamp:ts + allowOverwrite:NO + error:&err + ]; + //NSLog(@"imageBuffer %s", sent ? "sent!" : "not sent."); + if (err) { + NSLog(@"MediaPipe: sendPixelBuffer error: %@", err); + } + + mediapipe::Packet selectedEffectIdPacket = mediapipe::MakePacket(2).At(ts); + [self.mediapipeGraph movePacket:std::move(selectedEffectIdPacket) + intoStream:kSelectedEffectIdInputStream + error:nil]; +} + +@end + + +@implementation MediaPipeFaceLandmarkPoint +@end + +@implementation MediaPipeNormalizedRect +@end diff --git a/mediapipe/examples/ios/facemeshgpu/build_mediapipe.sh b/mediapipe/examples/ios/facemeshgpu/build_mediapipe.sh new file mode 100755 index 000000000..8a11c4048 --- /dev/null +++ b/mediapipe/examples/ios/facemeshgpu/build_mediapipe.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +mkdir -p ./frameworkbuild/MediaPipeFramework/arm64 +mkdir -p ./frameworkbuild/MediaPipeFramework/x86_64 +mkdir -p ./frameworkbuild/MediaPipeFramework/xcframework + +bazel build --config=ios_arm64 mediapipe/examples/ios/facemeshgpu:MediaPipeFramework +./mediapipe/examples/ios/facemeshgpu/patch_ios_framework.sh ./bazel-out/applebin_ios-ios_arm64-fastbuild-ST-2967bd56a867/bin/mediapipe/examples/ios/facemeshgpu/MediaPipeFramework.zip MediaPipeController.h +cp -a ./bazel-out/applebin_ios-ios_arm64-fastbuild-ST-2967bd56a867/bin/mediapipe/examples/ios/facemeshgpu/MediaPipeFramework.framework ./frameworkbuild/MediaPipeFramework/arm64 + +bazel build --config=ios_x86_64 mediapipe/examples/ios/facemeshgpu:MediaPipeFramework +./mediapipe/examples/ios/facemeshgpu/patch_ios_framework.sh ./bazel-out/applebin_ios-ios_x86_64-fastbuild-ST-2967bd56a867/bin/mediapipe/examples/ios/facemeshgpu/MediaPipeFramework.zip MediaPipeController.h +cp -a ./bazel-out/applebin_ios-ios_x86_64-fastbuild-ST-2967bd56a867/bin/mediapipe/examples/ios/facemeshgpu/MediaPipeFramework.framework ./frameworkbuild/MediaPipeFramework/x86_64 + +xcodebuild -create-xcframework \ + -framework ./frameworkbuild/MediaPipeFramework/x86_64/MediaPipeFramework.framework \ + -framework ./frameworkbuild/MediaPipeFramework/arm64/MediaPipeFramework.framework \ + -output ./frameworkbuild/MediaPipeFramework/xcframework/MediaPipeFramework.xcframework + diff --git a/mediapipe/examples/ios/facemeshgpu/patch_ios_framework.sh b/mediapipe/examples/ios/facemeshgpu/patch_ios_framework.sh new file mode 100755 index 000000000..6ebcb4cea --- /dev/null +++ b/mediapipe/examples/ios/facemeshgpu/patch_ios_framework.sh @@ -0,0 +1,66 @@ +#!/bin/bash +set -eu +set -o pipefail + +# Adds modulemap & header files to an iOS Framework +# generated with bazel and encapsulating Mediapipe. +# +# This makes it so that the patched .framework can be imported into Xcode. +# For a long term solution track the following issue: +# https://github.com/bazelbuild/rules_apple/issues/355 + +[[ $# -lt 2 ]] && echo "Usage: $0 ..." && exit 1 +zipped=$(python -c "import os; print(os.path.realpath('$1'))"); shift +name=$(basename "$zipped" .zip) +parent=$(dirname "$zipped") +named="$parent"/"$name".framework + +unzip "$zipped" -d "$parent" + +mkdir "$named"/Modules +cat << EOF >"$named"/Modules/module.modulemap +framework module $name { + umbrella header "$name.h" + + export * + module * { export * } + + link framework "AVFoundation" + link framework "Accelerate" + link framework "AssetsLibrary" + link framework "CoreFoundation" + link framework "CoreGraphics" + link framework "CoreImage" + link framework "CoreMedia" + link framework "CoreVideo" + link framework "GLKit" + link framework "Metal" + link framework "MetalKit" + link framework "OpenGLES" + link framework "QuartzCore" + link framework "UIKit" +} +EOF +# NOTE: All these linked frameworks are required by mediapipe/objc. + +cat << EOF >"$named"/Headers/$name.h +// +// $name.h +// $name +// + +#import + +//! Project version number for $name. +FOUNDATION_EXPORT double ${name}VersionNumber; + +//! Project version string for $name. +FOUNDATION_EXPORT const unsigned char ${name}VersionString[]; + +// In this header, you should import all the public headers of your framework using statements like #import <$name/PublicHeader.h> + +EOF +until [[ $# -eq 0 ]]; do + printf '#import "'"$1"'"\n' "$1" >>"$named"/Headers/$name.h + shift +done diff --git a/mediapipe/util/annotation_renderer.cc b/mediapipe/util/annotation_renderer.cc index bbaa3fa26..b7bbfd550 100644 --- a/mediapipe/util/annotation_renderer.cc +++ b/mediapipe/util/annotation_renderer.cc @@ -108,7 +108,7 @@ void AnnotationRenderer::RenderDataOnImage(const RenderData &render_data) if (render_data.render_annotations().size()){ DrawLipstick(render_data); WhitenTeeth(render_data); -// smooth_face(render_data); +// SmoothFace(render_data); } else { @@ -288,7 +288,7 @@ cv::Mat AnnotationRenderer::predict_forehead_mask(const RenderData &render_data, return new_skin_mask; } -void AnnotationRenderer::smooth_face(const RenderData &render_data) +void AnnotationRenderer::SmoothFace(const RenderData &render_data) { cv::Mat not_full_face = cv::Mat(FormFacePartMask(FACE_OVAL, render_data)) + diff --git a/mediapipe/util/annotation_renderer.h b/mediapipe/util/annotation_renderer.h index e812720b0..970980721 100644 --- a/mediapipe/util/annotation_renderer.h +++ b/mediapipe/util/annotation_renderer.h @@ -132,7 +132,7 @@ class AnnotationRenderer { void WhitenTeeth(const RenderData &render_data); // - void smooth_face(const RenderData &render_data); + void SmoothFace(const RenderData &render_data); // cv::Mat FormFacePartMask(std::vector orderList, const RenderData &render_data);