add framework builder

mediapipe/examples/ios/facemeshgpu/BUILD

@@ -15,13 +15,23 @@
 load(
     "@build_bazel_rules_apple//apple:ios.bzl",
     "ios_application",
-)
+    "ios_framework")
+load("@build_bazel_rules_swift//swift:swift.bzl", "swift_library")
 load(
     "//mediapipe/examples/ios:bundle_id.bzl",
     "BUNDLE_ID_PREFIX",
     "example_provisioning",
 )
 
+FRAMEWORK_HEADERS = [
+    "MediaPipeController.h",
+]
+
+IOS_FAMILIES = [
+    "iphone",
+    "ipad",
+]
+
 licenses(["notice"])
 
 MIN_IOS_VERSION = "11.0"
@@ -51,6 +61,68 @@ ios_application(
     ],
 )
 
+ios_framework(
+    name = "MediaPipeFramework",
+    hdrs = FRAMEWORK_HEADERS,
+    bundle_id = "dh.MediaPipeFramework",
+    bundle_name = "MediaPipeFramework",
+    families = IOS_FAMILIES,
+    infoplists = [
+        "//mediapipe/examples/ios/common:Info.plist",
+#        "Info.plist",
+    ],
+    minimum_os_version = MIN_IOS_VERSION,
+    visibility = ["//visibility:public"],
+    deps = [
+        ":MediaPipeLib",
+        "@ios_opencv//:OpencvFramework",
+    ],
+)
+
+
+objc_library(
+    name = "MediaPipeLib",
+    srcs = [
+        "MediaPipeController.mm",
+    ],
+    hdrs = FRAMEWORK_HEADERS,
+    copts = ["-std=c++17"],  # https://github.com/google/mediapipe/issues/2275#issuecomment-877145926
+    data = [
+        "//mediapipe/graphs/face_effect:face_effect_gpu.binarypb",
+        "//mediapipe/modules/face_detection:face_detection_short_range.tflite",
+        "//mediapipe/graphs/face_effect/data:axis.binarypb",
+        "//mediapipe/graphs/face_effect/data:axis.pngblob",
+        "//mediapipe/graphs/face_effect/data:facepaint.pngblob",
+        "//mediapipe/graphs/face_effect/data:glasses.binarypb",
+        "//mediapipe/graphs/face_effect/data:glasses.pngblob",
+        "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata.binarypb",
+        "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_detection.binarypb",
+        "//mediapipe/modules/face_geometry/data:geometry_pipeline_metadata_landmarks.binarypb",
+        "//mediapipe/modules/face_landmark:face_landmark.tflite",
+        "//mediapipe/graphs/face_mesh:face_mesh_mobile_gpu.binarypb",
+        "//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite",
+    ],
+    deps = [
+        "//mediapipe/objc:mediapipe_framework_ios",
+        "//mediapipe/objc:mediapipe_input_sources_ios",
+        "//mediapipe/calculators/core:packet_presence_calculator",
+#        "//mediapipe/objc:mediapipe_layer_renderer", # no need for layer renderer since I don't render
+    ] + select({
+#        "//mediapipe:ios_i386": [],
+#        "//mediapipe:ios_x86_64": [],
+        "//conditions:default": [
+            "//mediapipe/framework/formats:matrix_data_cc_proto",
+            "//mediapipe/graphs/face_effect:face_effect_gpu_deps",
+            "//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
+            "//mediapipe/graphs/face_mesh:mobile_calculators",
+            "//mediapipe/framework/formats:landmark_cc_proto",
+#            "//mediapipe/examples/facial_search/graphs:gpu_calculators",
+#            "//mediapipe/examples/facial_search:embeddings_database",
+        ],
+    }),
+)
+
+
 objc_library(
     name = "FaceMeshGpuAppLibrary",
     srcs = [

mediapipe/examples/ios/facemeshgpu/MediaPipeController.h

@@ -0,0 +1,44 @@
+#import <CoreVideo/CoreVideo.h>
+#import <Foundation/Foundation.h>
+#import <AVFoundation/AVFoundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@class MediaPipeController;
+@class MediaPipeFaceLandmarkPoint;
+@class MediaPipeNormalizedRect;
+
+typedef void (^MediaPipeCompletionBlock)(CVPixelBufferRef pixelBuffer);
+
+@protocol MediaPipeControllerDelegate <NSObject>
+@optional
+- (void)mediaPipeController:(MediaPipeController *)controller didReceiveFaces:(NSArray<NSArray<MediaPipeFaceLandmarkPoint *>*>*)faces;
+- (void)mediaPipeController:(MediaPipeController *)controller didReceiveFaceBoxes:(NSArray<MediaPipeNormalizedRect *>*)faces;
+- (void)mediaPipeController:(MediaPipeController *)controller didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer;
+@end
+
+@interface MediaPipeController : NSObject
+
++ (instancetype)facemesh;
++ (instancetype)effects;
+
+- (void)startGraph;
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer timestamp:(CMTime)timestamp completion:(nullable MediaPipeCompletionBlock)completion;
+@property (nullable, weak, nonatomic) id<MediaPipeControllerDelegate> delegate;
+@end
+
+@interface MediaPipeFaceLandmarkPoint : NSObject
+@property (nonatomic) float x;
+@property (nonatomic) float y;
+@property (nonatomic) float z;
+@end
+
+@interface MediaPipeNormalizedRect : NSObject
+@property (nonatomic) float centerX;
+@property (nonatomic) float centerY;
+@property (nonatomic) float height;
+@property (nonatomic) float width;
+@property (nonatomic) float rotation;
+@end
+
+NS_ASSUME_NONNULL_END

mediapipe/examples/ios/facemeshgpu/MediaPipeController.mm

@@ -0,0 +1,241 @@
+#import "MediaPipeController.h"
+#import "mediapipe/objc/MPPCameraInputSource.h"
+#import "mediapipe/objc/MPPGraph.h"
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+
+//#import "mediapipe/objc/MPPLayerRenderer.h"
+
+static NSString* const kMeshGraphName = @"face_mesh_mobile_gpu";
+static NSString* const kEffectsGraphName = @"face_effect_gpu";
+
+static const char *kInputStream = "input_video";
+static const char *kOutputStream = "output_video";
+static const char *kNumFacesInputSidePacket = "num_faces";
+static const char *kLandmarksOutputStream = "multi_face_landmarks";
+static const char *kFaceRectsOutputStream = "face_rects_from_landmarks";
+static const char *kLandmarkPresenceOutputStream = "landmark_presence";
+static const char *kSelectedEffectIdInputStream = "selected_effect_id";
+static const char *kMultiFaceGeometryStream = "multi_face_geometry";
+static const char* kUseFaceDetectionInputSourceInputSidePacket = "use_face_detection_input_source";
+static const BOOL kUseFaceDetectionInputSource = NO;
+
+// Max number of faces to detect/process.
+static const int kNumFaces = 2;
+
+@interface MediaPipeController () <MPPGraphDelegate>
+@property (nonatomic) MPPGraph* mediapipeGraph;
+@property (nonatomic, copy) MediaPipeCompletionBlock completion;
+@property (nonatomic) size_t timestamp;
+@end
+
+@implementation MediaPipeController
+
+#pragma mark - Cleanup methods
+
+- (void)dealloc {
+    self.mediapipeGraph.delegate = nil;
+    [self.mediapipeGraph cancel];
+    // Ignore errors since we're cleaning up.
+    [self.mediapipeGraph closeAllInputStreamsWithError:nil];
+    [self.mediapipeGraph waitUntilDoneWithError:nil];
+    
+    NSLog(@"dealloc MediaPipeController");
+}
+
+#pragma mark - MediaPipe graph methods
+
++ (MPPGraph*)loadGraphFromResource:(NSString*)resource {
+    // Load the graph config resource.
+    NSError* configLoadError = nil;
+    NSBundle* bundle = [NSBundle bundleForClass:[self class]];
+    if (!resource || resource.length == 0) {
+        bundle = NSBundle.mainBundle;
+    }
+    NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"];
+    NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError];
+    if (!data) {
+        NSLog(@"MediaPipe: Failed to load graph config: %@", configLoadError);
+        return nil;
+    }
+    
+    mediapipe::CalculatorGraphConfig config;
+    config.ParseFromArray(data.bytes, data.length);
+    
+    MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
+    [newGraph setSidePacket:(mediapipe::MakePacket<bool>(kUseFaceDetectionInputSource)) named:kUseFaceDetectionInputSourceInputSidePacket];
+    [newGraph setSidePacket:(mediapipe::MakePacket<int>(kNumFaces)) named:kNumFacesInputSidePacket];
+
+    [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
+    //[newGraph addFrameOutputStream:kMultiFaceGeometryStream outputPacketType:MPPPacketTypeRaw];
+    
+//    [newGraph addFrameOutputStream:kLandmarksOutputStream outputPacketType:MPPPacketTypeRaw];
+//    [newGraph addFrameOutputStream:kFaceRectsOutputStream outputPacketType:MPPPacketTypeRaw];
+//    [newGraph addFrameOutputStream:kLandmarkPresenceOutputStream outputPacketType:MPPPacketTypeRaw];
+    return newGraph;
+}
+
+- (instancetype)initWithGraphName:(NSString *)graphName {
+    self = [super init];
+    if (self) {
+        self.mediapipeGraph = [[self class] loadGraphFromResource:graphName];
+        self.mediapipeGraph.delegate = self;
+        
+        // Set maxFramesInFlight to a small value to avoid memory contention for real-time processing.
+        self.mediapipeGraph.maxFramesInFlight = 2;
+        NSLog(@"MediaPipe: Inited graph %@", graphName);
+        NSLog(@"alloc MediaPipeController");
+    }
+    return self;
+}
+
++ (instancetype)facemesh {
+    return [[MediaPipeController alloc] initWithGraphName:kMeshGraphName];
+}
+
++ (instancetype)effects {
+    return [[MediaPipeController alloc] initWithGraphName:kEffectsGraphName];
+}
+
+- (void)startGraph {
+    NSError* error;
+    if (![self.mediapipeGraph startWithError:&error]) {
+        NSLog(@"MediaPipe: Failed to start graph: %@", error);
+    }
+    NSLog(@"MediaPipe: Started graph");
+}
+
+#pragma mark - MPPGraphDelegate methods
+
+- (void)mediapipeGraph:(MPPGraph*)graph
+  didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
+            fromStream:(const std::string&)streamName {
+    //NSLog(@"MediaPipe: didOutputPixelBuffer %s %@", streamName.c_str(), self.completion);
+    if (streamName == kOutputStream) {
+        if([self.delegate respondsToSelector:@selector(mediaPipeController:didOutputPixelBuffer:)]) {
+            [_delegate mediaPipeController:self didOutputPixelBuffer:pixelBuffer];
+        }
+        if (self.completion) {
+            self.completion(pixelBuffer);
+        }
+    }
+}
+
+- (void)mediapipeGraph:(MPPGraph*)graph
+       didOutputPacket:(const ::mediapipe::Packet&)packet
+            fromStream:(const std::string&)streamName {
+    if (streamName == kLandmarksOutputStream) {
+        if (packet.IsEmpty()) {
+            return;
+        }
+        if(![self.delegate respondsToSelector:@selector(mediaPipeController:didReceiveFaces:)]) {
+            return;
+        }
+        const auto& multi_face_landmarks = packet.Get<std::vector<::mediapipe::NormalizedLandmarkList>>();
+        // NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(),
+        // multi_face_landmarks.size());
+        NSMutableArray <NSArray <MediaPipeFaceLandmarkPoint *>*>*faceLandmarks = [NSMutableArray new];
+        
+        for (int face_index = 0; face_index < multi_face_landmarks.size(); ++face_index) {
+            NSMutableArray *thisFaceLandmarks = [NSMutableArray new];
+            const auto& landmarks = multi_face_landmarks[face_index];
+            //      NSLog(@"\tNumber of landmarks for face[%d]: %d", face_index, landmarks.landmark_size());
+            for (int i = 0; i < landmarks.landmark_size(); ++i) {
+                //        NSLog(@"\t\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(),
+                //              landmarks.landmark(i).y(), landmarks.landmark(i).z());
+                MediaPipeFaceLandmarkPoint *obj_landmark = [MediaPipeFaceLandmarkPoint new];
+                obj_landmark.x = landmarks.landmark(i).x();
+                obj_landmark.y = landmarks.landmark(i).y();
+                obj_landmark.z = landmarks.landmark(i).z();
+                [thisFaceLandmarks addObject:obj_landmark];
+            }
+            [faceLandmarks addObject:thisFaceLandmarks];
+        }
+        [self.delegate mediaPipeController:self didReceiveFaces:faceLandmarks];
+    }
+    
+    else if (streamName == kFaceRectsOutputStream) {
+        if (packet.IsEmpty()) { // This condition never gets called because FaceLandmarkFrontGpu does not process when there are no detections
+            // NSLog(@"[TS:%lld] No face rects", packet.Timestamp().Value());
+            if([self.delegate respondsToSelector:@selector(mediaPipeController:didReceiveFaceBoxes:)]) {
+                [self.delegate mediaPipeController:self didReceiveFaceBoxes:@[]];
+            }
+            return;
+        }
+        if(![self.delegate respondsToSelector:@selector(mediaPipeController:didReceiveFaceBoxes:)]) {
+            return;
+        }
+        const auto& face_rects_from_landmarks = packet.Get<std::vector<::mediapipe::NormalizedRect>>();
+        NSMutableArray <MediaPipeNormalizedRect *>*outRects = [NSMutableArray new];
+        for (int face_index = 0; face_index < face_rects_from_landmarks.size(); ++face_index) {
+            const auto& face = face_rects_from_landmarks[face_index];
+            float centerX = face.x_center();
+            float centerY = face.y_center();
+            float height = face.height();
+            float width = face.width();
+            float rotation = face.rotation();
+            MediaPipeNormalizedRect *rect = [MediaPipeNormalizedRect new];
+            rect.centerX = centerX; rect.centerY = centerY; rect.height = height; rect.width = width; rect.rotation = rotation;
+            [outRects addObject:rect];
+        }
+        [self.delegate mediaPipeController:self didReceiveFaceBoxes:outRects];
+    } else if (streamName == kLandmarkPresenceOutputStream) {
+        bool is_landmark_present = true;
+        if (packet.IsEmpty()) {
+            is_landmark_present = false;
+        } else {
+            is_landmark_present = packet.Get<bool>();
+        }
+        
+        if (is_landmark_present) {
+        } else {
+            // NSLog(@"Landmarks not present");
+            if([self.delegate respondsToSelector:@selector(mediaPipeController:didReceiveFaceBoxes:)]) {
+                [self.delegate mediaPipeController:self didReceiveFaceBoxes:@[]];
+            }
+            if([self.delegate respondsToSelector:@selector(mediaPipeController:didReceiveFaces:)]) {
+                [self.delegate mediaPipeController:self didReceiveFaces:@[]];
+            }
+        }
+    } else {
+        //NSLog(@"MediaPipe: Unknown %@ packet with stream name %s", packet.IsEmpty() ? @"EMPTY" : @"NON-EMPTY",streamName.c_str());
+    }
+}
+
+
+#pragma mark - MPPInputSourceDelegate methods
+
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer timestamp:(CMTime)timestamp completion:(MediaPipeCompletionBlock)completion {
+    const auto ts = mediapipe::Timestamp(self.timestamp++ * mediapipe::Timestamp::kTimestampUnitsPerSecond);
+    self.completion = completion;
+    
+    NSError* err = nil;
+    //NSLog(@"sending imageBuffer @%@ to %s", @(ts.DebugString().c_str()), kInputStream);
+    auto sent = [self.mediapipeGraph sendPixelBuffer:imageBuffer
+                                          intoStream:kInputStream
+                                          packetType:MPPPacketTypePixelBuffer
+                                           timestamp:ts
+                                      allowOverwrite:NO
+                                               error:&err
+                 ];
+    //NSLog(@"imageBuffer %s", sent ? "sent!" : "not sent.");
+    if (err) {
+        NSLog(@"MediaPipe: sendPixelBuffer error: %@", err);
+    }
+    
+    mediapipe::Packet selectedEffectIdPacket = mediapipe::MakePacket<int>(2).At(ts);
+    [self.mediapipeGraph movePacket:std::move(selectedEffectIdPacket)
+                         intoStream:kSelectedEffectIdInputStream
+                              error:nil];
+}
+
+@end
+
+
+@implementation MediaPipeFaceLandmarkPoint
+@end
+
+@implementation MediaPipeNormalizedRect
+@end

mediapipe/examples/ios/facemeshgpu/build_mediapipe.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+mkdir -p ./frameworkbuild/MediaPipeFramework/arm64
+mkdir -p ./frameworkbuild/MediaPipeFramework/x86_64
+mkdir -p ./frameworkbuild/MediaPipeFramework/xcframework
+
+bazel build --config=ios_arm64 mediapipe/examples/ios/facemeshgpu:MediaPipeFramework
+./mediapipe/examples/ios/facemeshgpu/patch_ios_framework.sh ./bazel-out/applebin_ios-ios_arm64-fastbuild-ST-2967bd56a867/bin/mediapipe/examples/ios/facemeshgpu/MediaPipeFramework.zip MediaPipeController.h
+cp -a ./bazel-out/applebin_ios-ios_arm64-fastbuild-ST-2967bd56a867/bin/mediapipe/examples/ios/facemeshgpu/MediaPipeFramework.framework ./frameworkbuild/MediaPipeFramework/arm64
+
+bazel build --config=ios_x86_64 mediapipe/examples/ios/facemeshgpu:MediaPipeFramework
+./mediapipe/examples/ios/facemeshgpu/patch_ios_framework.sh ./bazel-out/applebin_ios-ios_x86_64-fastbuild-ST-2967bd56a867/bin/mediapipe/examples/ios/facemeshgpu/MediaPipeFramework.zip MediaPipeController.h
+cp -a ./bazel-out/applebin_ios-ios_x86_64-fastbuild-ST-2967bd56a867/bin/mediapipe/examples/ios/facemeshgpu/MediaPipeFramework.framework ./frameworkbuild/MediaPipeFramework/x86_64
+
+xcodebuild -create-xcframework \
+  -framework ./frameworkbuild/MediaPipeFramework/x86_64/MediaPipeFramework.framework \
+  -framework ./frameworkbuild/MediaPipeFramework/arm64/MediaPipeFramework.framework \
+  -output ./frameworkbuild/MediaPipeFramework/xcframework/MediaPipeFramework.xcframework
+

mediapipe/examples/ios/facemeshgpu/patch_ios_framework.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+set -eu
+set -o pipefail
+
+# Adds modulemap & header files to an iOS Framework
+# generated with bazel and encapsulating Mediapipe.
+#
+# This makes it so that the patched .framework can be imported into Xcode.
+# For a long term solution track the following issue:
+#   https://github.com/bazelbuild/rules_apple/issues/355
+
+[[ $# -lt 2 ]] && echo "Usage: $0  <path/to/zipped .framework> <hdrs>..." && exit 1
+zipped=$(python -c "import os; print(os.path.realpath('$1'))"); shift
+name=$(basename "$zipped" .zip)
+parent=$(dirname "$zipped")
+named="$parent"/"$name".framework
+
+unzip "$zipped" -d "$parent"
+
+mkdir "$named"/Modules
+cat << EOF >"$named"/Modules/module.modulemap
+framework module $name {
+  umbrella header "$name.h"
+
+  export *
+  module * { export * }
+
+  link framework "AVFoundation"
+  link framework "Accelerate"
+  link framework "AssetsLibrary"
+  link framework "CoreFoundation"
+  link framework "CoreGraphics"
+  link framework "CoreImage"
+  link framework "CoreMedia"
+  link framework "CoreVideo"
+  link framework "GLKit"
+  link framework "Metal"
+  link framework "MetalKit"
+  link framework "OpenGLES"
+  link framework "QuartzCore"
+  link framework "UIKit"
+}
+EOF
+# NOTE: All these linked frameworks are required by mediapipe/objc.
+
+cat << EOF >"$named"/Headers/$name.h
+//
+//  $name.h
+//  $name
+//
+
+#import <Foundation/Foundation.h>
+
+//! Project version number for $name.
+FOUNDATION_EXPORT double ${name}VersionNumber;
+
+//! Project version string for $name.
+FOUNDATION_EXPORT const unsigned char ${name}VersionString[];
+
+// In this header, you should import all the public headers of your framework using statements like #import <$name/PublicHeader.h>
+
+EOF
+until [[ $# -eq 0 ]]; do
+  printf '#import "'"$1"'"\n' "$1" >>"$named"/Headers/$name.h
+  shift
+done

mediapipe/util/annotation_renderer.cc

@@ -108,7 +108,7 @@ void AnnotationRenderer::RenderDataOnImage(const RenderData &render_data)
   if (render_data.render_annotations().size()){
     DrawLipstick(render_data);
     WhitenTeeth(render_data);
-//    smooth_face(render_data);
+//    SmoothFace(render_data);
   }
   else
   {
@@ -288,7 +288,7 @@ cv::Mat AnnotationRenderer::predict_forehead_mask(const RenderData &render_data,
   return new_skin_mask;
 }
 
-void AnnotationRenderer::smooth_face(const RenderData &render_data)
+void AnnotationRenderer::SmoothFace(const RenderData &render_data)
 {
 
   cv::Mat not_full_face = cv::Mat(FormFacePartMask(FACE_OVAL, render_data)) +

mediapipe/util/annotation_renderer.h

@@ -132,7 +132,7 @@ class AnnotationRenderer {
   void WhitenTeeth(const RenderData &render_data);
 
   //
-  void smooth_face(const RenderData &render_data);
+  void SmoothFace(const RenderData &render_data);
 
   //
   cv::Mat FormFacePartMask(std::vector<int> orderList, const RenderData &render_data);