Merge 1001ead358 into ec43bea176

WORKSPACE

@@ -390,6 +390,14 @@ http_archive(
     url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
 )
 
+# http_archive(
+#     name = "ios_opencv",
+#     sha256 = "c23e92c4a0cd343f73d4056e66c961cdf68e73ca699b1129638537a931c4ffc8",
+#     build_file = "@//third_party:opencv_ios.BUILD",
+#     type = "zip",
+#     url = "https://github.com/opencv/opencv/releases/download/4.7.0/opencv-4.7.0-ios-framework.zip",
+# )
+
 # Building an opencv.xcframework from the OpenCV 4.5.3 sources is necessary for
 # MediaPipe iOS Task Libraries to be supported on arm64(M1) Macs. An
 # `opencv.xcframework` archive has not been released and it is recommended to

build_face_mesh_ios_framework.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+sudo rm -rf frameworkbuild
+
+# Create output directories~
+mkdir -p ./frameworkbuild/FaceMeshSDK/arm64
+# XCFramework is how we're going to use it.
+mkdir -p ./frameworkbuild/FaceMeshSDK/xcframework
+
+# Interesting fact. Bazel `build` command stores cached files in `/private/var/tmp/...` folders
+# and when you run build, if it finds cached files, it kind of symlinks the files/folders
+# into the `bazel-bin` folder found in the project root. So don't be afraid of re-running builds
+# because the files are cached.
+
+# build the arm64 binary framework
+# bazel build --copt=-fembed-bitcode --apple_bitcode=embedded --config=ios_arm64 mediapipe/examples/ios/facemeshioslib:FaceMeshIOSLibFramework
+bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/facemeshioslib:FaceMeshSDK
+# use --cxxopt=-O3 to reduce framework size
+# bazel build --copt=-O3 --cxxopt=-O3 --config=ios_arm64 mediapipe/examples/ios/facemeshioslib:FaceMeshIOSLibFramework
+
+# The arm64 framework zip will be located at //bazel-bin/mediapipe/examples/ios/facemeshioslib/FaceMeshIOSLibFramework.zip
+
+# Call the framework patcher (First argument = compressed framework.zip, Second argument = header file's name(in this case FaceMeshIOSLib.h))
+sudo bash ./mediapipe/examples/ios/facemeshioslib/patch_ios_framework.sh ./bazel-bin/mediapipe/examples/ios/facemeshioslib/FaceMeshSDK.zip FaceMesh.h
+
+# There will be a resulting patched .framework folder at the same directory, this is our arm64 one, we copy it to our arm64 folder
+sudo cp -a ./bazel-bin/mediapipe/examples/ios/facemeshioslib/FaceMeshSDK.framework ./frameworkbuild/FaceMeshSDK/arm64
+
+# Create xcframework (because the classic lipo method with normal .framework no longer works (shows Building for iOS Simulator, but the linked and embedded framework was built for iOS + iOS Simulator))
+
+sudo xcodebuild -create-xcframework \
+    -framework ./frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework \
+    -output ./frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework

frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Headers/FaceMesh.h

@@ -0,0 +1,58 @@
+#import <CoreVideo/CoreVideo.h>
+#import <Foundation/Foundation.h>
+
+@interface IntPoint : NSObject
+@property(nonatomic) NSInteger x;
+@property(nonatomic) NSInteger y;
+
+- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y;
+@end
+
+@interface NSValue (IntPoint)
++ (instancetype)valuewithIntPoint:(IntPoint *)value;
+@property (readonly) IntPoint* intPointValue;
+@end
+
+@interface FaceMeshLandmarkPoint : NSObject
+@property(nonatomic) float x;
+@property(nonatomic) float y;
+@property(nonatomic) float z;
+@end
+
+@interface FaceMeshNormalizedRect : NSObject
+@property(nonatomic) float centerX;
+@property(nonatomic) float centerY;
+@property(nonatomic) float height;
+@property(nonatomic) float width;
+@property(nonatomic) float rotation;
+@end
+
+@protocol FaceMeshDelegate <NSObject>
+@optional
+/**
+ * Array of faces, with faces represented by arrays of face landmarks
+ */
+- (void)didReceiveFaces:(NSArray<NSArray<FaceMeshLandmarkPoint *> *> *)faces;
+
+- (void)didSavedRegions:(NSArray<NSURL *> *)foreheadURLs
+          leftcheekURLs:(NSArray<NSURL *> *)leftcheekURLs
+          rightcheekURLs:(NSArray<NSURL *> *)rightcheekURLs;
+@end
+
+@interface FaceMesh : NSObject
+- (instancetype)init;
+- (void)startGraph;
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer;
+- (CVPixelBufferRef)resize:(CVPixelBufferRef)pixelBuffer
+                     width:(int)width
+                    height:(int)height;
+- (uint8_t **)buffer2Array2D:(CVPixelBufferRef)pixelBuffer;
+- (void)extractRegions:(NSURL *)fileName
+            foreheadBoxes:(NSArray<NSArray<IntPoint *> *> *)foreheadBoxes
+            leftCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)leftCheekBoxes
+            rightCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)rightCheekBoxes
+            totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess
+            skipNFirstFrames:(NSInteger)skipNFirstFrames;
+@property(weak, nonatomic) id<FaceMeshDelegate> delegate;
+@property(nonatomic) size_t timestamp;
+@end

frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Headers/FaceMeshSDK.h

@@ -0,0 +1,16 @@
+//
+//  FaceMeshSDK.h
+//  FaceMeshSDK
+//
+
+#import <Foundation/Foundation.h>
+
+//! Project version number for FaceMeshSDK.
+FOUNDATION_EXPORT double FaceMeshSDKVersionNumber;
+
+//! Project version string for FaceMeshSDK.
+FOUNDATION_EXPORT const unsigned char FaceMeshSDKVersionString[];
+
+// In this header, you should import all the public headers of your framework using statements like #import <FaceMeshSDK/PublicHeader.h>
+
+#import "FaceMesh.h"

frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Modules/module.modulemap

@@ -0,0 +1,21 @@
+framework module FaceMeshSDK {
+  umbrella header "FaceMeshSDK.h"
+
+  export *
+  module * { export * }
+
+  link framework "AVFoundation"
+  link framework "Accelerate"
+  link framework "AssetsLibrary"
+  link framework "CoreFoundation"
+  link framework "CoreGraphics"
+  link framework "CoreImage"
+  link framework "CoreMedia"
+  link framework "CoreVideo"
+  link framework "GLKit"
+  link framework "Metal"
+  link framework "MetalKit"
+  link framework "OpenGLES"
+  link framework "QuartzCore"
+  link framework "UIKit"
+}

frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_mesh_ios_lib_gpu.binarypb

@@ -0,0 +1,5 @@
+
+€
ConstantSidePacketCalculator2PACKET:with_attentionBI
+Atype.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions
+

+¬
FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces*WITH_ATTENTION:with_attentionRinput_videozmulti_face_landmarks‚
	num_faces

frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/pure_face_mesh_mobile_gpu.binarypb

@@ -0,0 +1,3 @@
+
+<EFBFBD>
FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces
+SPacketPresenceCalculatorPACKET:multi_face_landmarks"PRESENCE:landmark_presenceRinput_videozmulti_face_landmarkszface_rects_from_landmarkszlandmark_presence‚
	num_faces

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/Info.plist

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>AvailableLibraries</key>
+	<array>
+		<dict>
+			<key>LibraryIdentifier</key>
+			<string>ios-arm64</string>
+			<key>LibraryPath</key>
+			<string>FaceMeshSDK.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>ios</string>
+		</dict>
+	</array>
+	<key>CFBundlePackageType</key>
+	<string>XFWK</string>
+	<key>XCFrameworkFormatVersion</key>
+	<string>1.0</string>
+</dict>
+</plist>

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Headers/FaceMesh.h

@@ -0,0 +1,58 @@
+#import <CoreVideo/CoreVideo.h>
+#import <Foundation/Foundation.h>
+
+@interface IntPoint : NSObject
+@property(nonatomic) NSInteger x;
+@property(nonatomic) NSInteger y;
+
+- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y;
+@end
+
+@interface NSValue (IntPoint)
++ (instancetype)valuewithIntPoint:(IntPoint *)value;
+@property (readonly) IntPoint* intPointValue;
+@end
+
+@interface FaceMeshLandmarkPoint : NSObject
+@property(nonatomic) float x;
+@property(nonatomic) float y;
+@property(nonatomic) float z;
+@end
+
+@interface FaceMeshNormalizedRect : NSObject
+@property(nonatomic) float centerX;
+@property(nonatomic) float centerY;
+@property(nonatomic) float height;
+@property(nonatomic) float width;
+@property(nonatomic) float rotation;
+@end
+
+@protocol FaceMeshDelegate <NSObject>
+@optional
+/**
+ * Array of faces, with faces represented by arrays of face landmarks
+ */
+- (void)didReceiveFaces:(NSArray<NSArray<FaceMeshLandmarkPoint *> *> *)faces;
+
+- (void)didSavedRegions:(NSArray<NSURL *> *)foreheadURLs
+          leftcheekURLs:(NSArray<NSURL *> *)leftcheekURLs
+          rightcheekURLs:(NSArray<NSURL *> *)rightcheekURLs;
+@end
+
+@interface FaceMesh : NSObject
+- (instancetype)init;
+- (void)startGraph;
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer;
+- (CVPixelBufferRef)resize:(CVPixelBufferRef)pixelBuffer
+                     width:(int)width
+                    height:(int)height;
+- (uint8_t **)buffer2Array2D:(CVPixelBufferRef)pixelBuffer;
+- (void)extractRegions:(NSURL *)fileName
+            foreheadBoxes:(NSArray<NSArray<IntPoint *> *> *)foreheadBoxes
+            leftCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)leftCheekBoxes
+            rightCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)rightCheekBoxes
+            totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess
+            skipNFirstFrames:(NSInteger)skipNFirstFrames;
+@property(weak, nonatomic) id<FaceMeshDelegate> delegate;
+@property(nonatomic) size_t timestamp;
+@end

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Headers/FaceMeshSDK.h

@@ -0,0 +1,16 @@
+//
+//  FaceMeshSDK.h
+//  FaceMeshSDK
+//
+
+#import <Foundation/Foundation.h>
+
+//! Project version number for FaceMeshSDK.
+FOUNDATION_EXPORT double FaceMeshSDKVersionNumber;
+
+//! Project version string for FaceMeshSDK.
+FOUNDATION_EXPORT const unsigned char FaceMeshSDKVersionString[];
+
+// In this header, you should import all the public headers of your framework using statements like #import <FaceMeshSDK/PublicHeader.h>
+
+#import "FaceMesh.h"

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Modules/module.modulemap

@@ -0,0 +1,21 @@
+framework module FaceMeshSDK {
+  umbrella header "FaceMeshSDK.h"
+
+  export *
+  module * { export * }
+
+  link framework "AVFoundation"
+  link framework "Accelerate"
+  link framework "AssetsLibrary"
+  link framework "CoreFoundation"
+  link framework "CoreGraphics"
+  link framework "CoreImage"
+  link framework "CoreMedia"
+  link framework "CoreVideo"
+  link framework "GLKit"
+  link framework "Metal"
+  link framework "MetalKit"
+  link framework "OpenGLES"
+  link framework "QuartzCore"
+  link framework "UIKit"
+}

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_mesh_ios_lib_gpu.binarypb

@@ -0,0 +1,5 @@
+
+€
ConstantSidePacketCalculator2PACKET:with_attentionBI
+Atype.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions
+

+¬
FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces*WITH_ATTENTION:with_attentionRinput_videozmulti_face_landmarks‚
	num_faces

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/pure_face_mesh_mobile_gpu.binarypb

@@ -0,0 +1,3 @@
+
+<EFBFBD>
FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces
+SPacketPresenceCalculatorPACKET:multi_face_landmarks"PRESENCE:landmark_presenceRinput_videozmulti_face_landmarkszface_rects_from_landmarkszlandmark_presence‚
	num_faces

mediapipe/examples/ios/facemeshioslib/BUILD

@@ -0,0 +1,68 @@
+load(
+    "@build_bazel_rules_apple//apple:ios.bzl",
+    "ios_framework"
+)
+load(
+    "//mediapipe/examples/ios:bundle_id.bzl",
+    "BUNDLE_ID_PREFIX",
+    "example_provisioning",
+)
+
+licenses(["notice"])  # Apache 2.0
+
+MIN_IOS_VERSION = "11.1"
+
+IOS_FAMILIES = [
+    "iphone",
+    "ipad",
+]
+
+FRAMEWORK_HEADERS = [
+    "FaceMesh.h",
+]
+
+ios_framework(
+    name = "FaceMeshSDK",
+    hdrs = FRAMEWORK_HEADERS,
+    bundle_id = BUNDLE_ID_PREFIX + ".FaceMeshSDK",
+    bundle_name = "FaceMeshSDK",
+    families = IOS_FAMILIES,
+    infoplists = [
+        "//mediapipe/examples/ios/common:Info.plist",
+#        "Info.plist",
+    ],
+    minimum_os_version = MIN_IOS_VERSION,
+    visibility = ["//visibility:public"],
+    deps = [
+        ":FaceMeshObj",
+        "@ios_opencv//:OpencvFramework",
+    ],
+)
+
+
+objc_library(
+    name = "FaceMeshObj",
+    srcs = [
+        "FaceMesh.mm",
+    ],
+    hdrs = FRAMEWORK_HEADERS,
+    copts = ["-std=c++17"],
+    data = [
+        "//mediapipe/graphs/face_mesh:face_mesh_ios_lib_gpu.binarypb",
+        "//mediapipe/modules/face_detection:face_detection_short_range.tflite",
+        "//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite",
+    ],
+    deps = [
+        "//mediapipe/objc:mediapipe_framework_ios",
+        "//mediapipe/objc:mediapipe_input_sources_ios",
+        "//mediapipe/calculators/core:packet_presence_calculator",
+        "//mediapipe/framework/port:opencv_video",
+        "//mediapipe/framework/port:opencv_imgcodecs",
+#        "//mediapipe/objc:mediapipe_layer_renderer", # no need for layer renderer since I don't render
+    ] + select({
+        "//conditions:default": [
+            "//mediapipe/graphs/face_mesh:mobile_calculators",
+            "//mediapipe/framework/formats:landmark_cc_proto",
+        ],
+    }),
+)

mediapipe/examples/ios/facemeshioslib/FaceMesh.h

@@ -0,0 +1,58 @@
+#import <CoreVideo/CoreVideo.h>
+#import <Foundation/Foundation.h>
+
+@interface IntPoint : NSObject
+@property(nonatomic) NSInteger x;
+@property(nonatomic) NSInteger y;
+
+- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y;
+@end
+
+@interface NSValue (IntPoint)
++ (instancetype)valuewithIntPoint:(IntPoint *)value;
+@property (readonly) IntPoint* intPointValue;
+@end
+
+@interface FaceMeshLandmarkPoint : NSObject
+@property(nonatomic) float x;
+@property(nonatomic) float y;
+@property(nonatomic) float z;
+@end
+
+@interface FaceMeshNormalizedRect : NSObject
+@property(nonatomic) float centerX;
+@property(nonatomic) float centerY;
+@property(nonatomic) float height;
+@property(nonatomic) float width;
+@property(nonatomic) float rotation;
+@end
+
+@protocol FaceMeshDelegate <NSObject>
+@optional
+/**
+ * Array of faces, with faces represented by arrays of face landmarks
+ */
+- (void)didReceiveFaces:(NSArray<NSArray<FaceMeshLandmarkPoint *> *> *)faces;
+
+- (void)didSavedRegions:(NSArray<NSURL *> *)foreheadURLs
+          leftcheekURLs:(NSArray<NSURL *> *)leftcheekURLs
+          rightcheekURLs:(NSArray<NSURL *> *)rightcheekURLs;
+@end
+
+@interface FaceMesh : NSObject
+- (instancetype)init;
+- (void)startGraph;
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer;
+- (CVPixelBufferRef)resize:(CVPixelBufferRef)pixelBuffer
+                     width:(int)width
+                    height:(int)height;
+- (uint8_t **)buffer2Array2D:(CVPixelBufferRef)pixelBuffer;
+- (void)extractRegions:(NSURL *)fileName
+            foreheadBoxes:(NSArray<NSArray<IntPoint *> *> *)foreheadBoxes
+            leftCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)leftCheekBoxes
+            rightCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)rightCheekBoxes
+            totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess
+            skipNFirstFrames:(NSInteger)skipNFirstFrames;
+@property(weak, nonatomic) id<FaceMeshDelegate> delegate;
+@property(nonatomic) size_t timestamp;
+@end

mediapipe/examples/ios/facemeshioslib/FaceMesh.mm

@@ -0,0 +1,397 @@
+#import "FaceMesh.h"
+#import "mediapipe/objc/MPPCameraInputSource.h"
+#import "mediapipe/objc/MPPGraph.h"
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_video_inc.h"
+#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
+#include "opencv2/imgcodecs/ios.h" 
+
+#import <UIKit/UIKit.h>
+
+//#import "mediapipe/objc/MPPLayerRenderer.h"
+
+// The graph name specified is supposed to be the same as in the pb file (binarypb?)
+static NSString* const kGraphName = @"face_mesh_ios_lib_gpu";
+// static NSString* const kGraphName = @"pure_face_mesh_mobile_gpu";
+
+static const char* kInputStream = "input_video";
+static const char* kNumFacesInputSidePacket = "num_faces";
+static const char* kLandmarksOutputStream = "multi_face_landmarks";
+
+// Max number of faces to detect/process.
+static const int kNumFaces = 1;
+
+
+@interface FaceMesh () <MPPGraphDelegate>
+@property(nonatomic) MPPGraph* mediapipeGraph;
+@end
+
+@implementation FaceMesh {}
+
+#pragma mark - Cleanup methods
+
+- (void)dealloc {
+  self.mediapipeGraph.delegate = nil;
+  [self.mediapipeGraph cancel];
+  // Ignore errors since we're cleaning up.
+  [self.mediapipeGraph closeAllInputStreamsWithError:nil];
+  [self.mediapipeGraph waitUntilDoneWithError:nil];
+}
+
+#pragma mark - MediaPipe graph methods
+
++ (MPPGraph*)loadGraphFromResource:(NSString*)resource {
+  // Load the graph config resource.
+  NSError* configLoadError = nil;
+  NSBundle* bundle = [NSBundle bundleForClass:[self class]];
+  if (!resource || resource.length == 0) {
+    return nil;
+  }
+  NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"];
+  NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError];
+  if (!data) {
+    NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError);
+    return nil;
+  }
+
+  // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object.
+  mediapipe::CalculatorGraphConfig config;
+  config.ParseFromArray(data.bytes, data.length);
+
+  // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
+  MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
+  
+  // Set graph configurations
+  [newGraph setSidePacket:(mediapipe::MakePacket<int>(kNumFaces))
+                              named:kNumFacesInputSidePacket];
+
+  [newGraph addFrameOutputStream:kLandmarksOutputStream
+                outputPacketType:MPPPacketTypeRaw];
+  return newGraph;
+}
+
+- (instancetype)init {
+  self = [super init];
+  if (self) {
+    self.mediapipeGraph = [[self class] loadGraphFromResource:kGraphName];
+    self.mediapipeGraph.delegate = self;
+    
+    // // Set maxFramesInFlight to a small value to avoid memory contention
+    // // for real-time processing.
+    // self.mediapipeGraph.maxFramesInFlight = 2;
+    NSLog(@"inited graph %@", kGraphName);
+  }
+  return self;
+}
+
+- (void)startGraph {
+  NSError* error;
+  if (![self.mediapipeGraph startWithError:&error]) {
+    NSLog(@"Failed to start graph: %@", error);
+  }
+  NSLog(@"Started graph %@", kGraphName);
+}
+
+#pragma mark - MPPGraphDelegate methods
+
+// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+    didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
+              fromStream:(const std::string&)streamName {
+  NSLog(@"recv pixelBuffer from %@", @(streamName.c_str()));
+}
+
+// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+     didOutputPacket:(const ::mediapipe::Packet&)packet
+          fromStream:(const std::string&)streamName {
+  if (streamName == kLandmarksOutputStream) {
+    if (packet.IsEmpty()) { // This condition never gets called because FaceLandmarkFrontGpu does not process when there are no detections
+      return;
+    }
+    const auto& multi_face_landmarks = packet.Get<std::vector<::mediapipe::NormalizedLandmarkList>>();
+    // NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(),
+    //       multi_face_landmarks.size());
+    NSMutableArray <NSArray <FaceMeshLandmarkPoint *>*>*faceLandmarks = [NSMutableArray new];
+    
+    for (int face_index = 0; face_index < multi_face_landmarks.size(); ++face_index) {
+      NSMutableArray *thisFaceLandmarks = [NSMutableArray new];
+      const auto& landmarks = multi_face_landmarks[face_index];
+//      NSLog(@"\tNumber of landmarks for face[%d]: %d", face_index, landmarks.landmark_size());
+      for (int i = 0; i < landmarks.landmark_size(); ++i) {
+//        NSLog(@"\t\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(),
+//              landmarks.landmark(i).y(), landmarks.landmark(i).z());
+        FaceMeshLandmarkPoint *obj_landmark = [FaceMeshLandmarkPoint new];
+        obj_landmark.x = landmarks.landmark(i).x();
+        obj_landmark.y = landmarks.landmark(i).y();
+        obj_landmark.z = landmarks.landmark(i).z();
+        [thisFaceLandmarks addObject:obj_landmark];
+      }
+      [faceLandmarks addObject:thisFaceLandmarks];
+    }
+    if([self.delegate respondsToSelector:@selector(didReceiveFaces:)]) {
+      [self.delegate didReceiveFaces:faceLandmarks];
+    }
+  } else {
+    NSLog(@"Unknown %@ packet with stream name %s", packet.IsEmpty() ? @"EMPTY" : @"NON-EMPTY",streamName.c_str());
+  }
+}
+
+
+#pragma mark - MPPInputSourceDelegate methods
+
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer {
+  const auto ts =
+      mediapipe::Timestamp(self.timestamp++ * mediapipe::Timestamp::kTimestampUnitsPerSecond);
+  NSError* err = nil;
+  // NSLog(@"sending imageBuffer @%@ to %s", @(ts.DebugString().c_str()), kInputStream);
+  auto sent = [self.mediapipeGraph sendPixelBuffer:imageBuffer
+                                        intoStream:kInputStream
+                                        packetType:MPPPacketTypePixelBuffer
+                                         timestamp:ts
+                                    allowOverwrite:NO
+                                             error:&err];
+  // NSLog(@"imageBuffer %s", sent ? "sent!" : "not sent.");
+  if (err) {
+    NSLog(@"sendPixelBuffer error: %@", err);
+  }
+}
+
+- (void)extractRegions:(NSURL *)fileName
+            foreheadBoxes:(NSArray<NSArray<IntPoint *> *> *)foreheadBoxes
+            leftCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)leftCheekBoxes
+            rightCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)rightCheekBoxes
+            totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess
+            skipNFirstFrames:(NSInteger)skipNFirstFrames {
+
+    NSString *filename = fileName.path;
+    std::string filePath = [filename UTF8String];
+
+    cv::VideoCapture vid(filePath);
+
+    if (!vid.isOpened()) {
+        printf("@Error Opening video file");
+    }
+    else {
+        printf("File Opened AAAA");
+
+        // NSMutableArray *foreheadURLs = [NSMutableArray new];
+        // NSMutableArray *leftcheekURLs = [NSMutableArray new];
+        // NSMutableArray *rightcheekURLs = [NSMutableArray new];
+
+        int startFrame = int(vid.get(cv::CAP_PROP_POS_FRAMES));
+        int totalFrame = int(vid.get(cv::CAP_PROP_FRAME_COUNT));
+        // int nframes = totalFrame - startFrame;
+
+        // if (totalFramesNeedProcess > nframes) {
+        //   NSLog(@"Video too short");
+        //   return;
+        // }
+
+        if (skipNFirstFrames < 0 || totalFramesNeedProcess < 0) {
+          vid.release();
+          return;
+        }
+
+        int frameIdx = skipNFirstFrames;
+        int maxFrameIndex = totalFramesNeedProcess;
+
+        if (skipNFirstFrames > 0) {
+          maxFrameIndex += skipNFirstFrames;
+        }
+
+        // Process forehead
+        std::vector<cv::Mat> foreheads;
+        std::vector<cv::Mat> leftcheeks;
+        std::vector<cv::Mat> rightcheeks;
+        
+        while (frameIdx < maxFrameIndex) {
+          cv::Mat frame;
+
+          if (!vid.read(frame)) {
+              NSLog(@"Failed to read frame.");
+              break;
+          }
+          cv::cvtColor(frame, frame, cv::COLOR_BGR2RGB);
+          // Process the frame (e.g., display or save it)
+
+          NSLog(@"frame index: %d", frameIdx);
+
+          NSMutableArray *rowForehead = [foreheadBoxes objectAtIndex:(frameIdx - skipNFirstFrames)];
+          NSMutableArray *rowLeftCheek = [leftCheekBoxes objectAtIndex:(frameIdx - skipNFirstFrames)];
+          NSMutableArray *rowRightCheek = [rightCheekBoxes objectAtIndex:(frameIdx - skipNFirstFrames)];
+
+          cv::Mat forehead = extractRegion(frame, rowForehead);
+          cv::Mat leftCheek = extractRegion(frame, rowLeftCheek);
+          cv::Mat rightCheek = extractRegion(frame, rowRightCheek);
+
+          foreheads.push_back(forehead);
+          leftcheeks.push_back(leftCheek);
+          rightcheeks.push_back(rightCheek);
+          
+          frameIdx++;
+        }
+
+        NSLog(@"total foreheads: %d", foreheads.size());
+        NSLog(@"total leftCheeks: %d", leftcheeks.size());
+        NSLog(@"total rightCheeks: %d", rightcheeks.size());
+
+        // for (int i = 0; i < foreheads.size(); i++) {
+        //   cv::Mat rgbaImage;
+        //   cv::cvtColor(foreheads[i], rgbaImage, cv::COLOR_RGB2RGBA);
+        //   saveCVImageAsPNG(MatToUIImage(rgbaImage), @"forehead");
+        // }
+
+        NSMutableArray *foreheadURLs = saveCVImagesAsPNGs(foreheads, @"forehead");
+        NSMutableArray *leftcheekURLs = saveCVImagesAsPNGs(leftcheeks, @"leftcheek");
+        NSMutableArray *rightcheekURLs = saveCVImagesAsPNGs(rightcheeks, @"rightcheek");
+
+        // cv::cvtColor(leftCheeks[0], rgbaImage, cv::COLOR_RGB2RGBA);
+        // NSData *firstData = [NSData dataWithBytes:rgbaImage.data length:rgbaImage.total() * rgbaImage.elemSize()];
+        // saveCVImageAsPNG([UIImage imageWithData:firstData], @"leftcheek");
+
+        // cv::cvtColor(rightCheeks[0], rgbaImage, cv::COLOR_RGB2RGBA);
+        // NSData *firstData = [NSData dataWithBytes:rgbaImage.data length:rgbaImage.total() * rgbaImage.elemSize()];
+        // saveCVImageAsPNG([UIImage imageWithData:firstData], @"rightcheek");
+
+        if([self.delegate respondsToSelector:@selector(didSavedRegions:leftcheekURLs:rightcheekURLs:)]) {
+            NSLog(@"nguyencse ==> has didSavedRegions");
+            [self.delegate didSavedRegions:foreheadURLs leftcheekURLs:leftcheekURLs rightcheekURLs:rightcheekURLs];
+        }
+    }
+
+    vid.release();
+}
+
+cv::Mat extractRegion(cv::Mat img, NSArray<IntPoint *> *box) {
+  IntPoint* point0 = [box objectAtIndex:0];
+  IntPoint* point1 = [box objectAtIndex:1];
+  IntPoint* point2 = [box objectAtIndex:2];
+  IntPoint* point3 = [box objectAtIndex:3];
+
+  // LEFT TOP --> RIGHT TOP --> RIGHT BOTTOM --> LEFT BOTTOM
+  int frameWidth = point1.x - point0.x;
+  int frameHeight = point3.y - point0.y;
+  cv::Mat region = cropROI(img, cv::Rect(point0.x, point0.y, frameWidth, frameHeight));
+
+  // square
+  region = square(region);
+
+  // resize to 32x32
+  region = resizeWithAreaInterpolation(region, cv::Size(32, 32));
+
+  return region;
+}
+
+cv::Mat cropROI(cv::Mat src, cv::Rect roi) {
+  // Crop the full image to that image contained by the rectangle myROI
+  // Note that this doesn't copy the data
+  cv::Mat croppedRef(src, roi);
+
+  cv::Mat cropped;
+  // Copy the data into new matrix
+  croppedRef.copyTo(cropped);
+
+  return cropped;
+}
+
+cv::Mat square(cv::Mat frame) {
+    if (frame.rows < frame.cols) {
+        int diff = frame.cols - frame.rows;
+        cv::Mat pad(frame.rows, diff, frame.type(), cv::Scalar(0));
+        cv::hconcat(frame, pad, frame);
+    } else if (frame.rows > frame.cols) {
+        int diff = frame.rows - frame.cols;
+        cv::Mat pad(diff, frame.cols, frame.type(), cv::Scalar(0));
+        cv::vconcat(frame, pad, frame);
+    }
+    
+    return frame;
+}
+
+cv::Mat resizeWithAreaInterpolation(cv::Mat image, cv::Size newShape) {
+    cv::Size originalShape = image.size();
+    cv::resize(image, image, newShape, 0, 0, cv::INTER_AREA);
+    return image;
+}
+
+// NSURL *saveCVImageAsPNG(UIImage *image, NSString *fileName) {
+//     // Create a unique file name for each image
+//     NSString *fileNameIndex = [fileName stringByAppendingFormat:@"_%d", 0];
+//     NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:[fileNameIndex stringByAppendingString:@".png"]];
+//     NSURL *fileURL = [NSURL fileURLWithPath:filePath];
+
+//     NSData *pngData = UIImagePNGRepresentation(image);
+
+//     if ([pngData writeToFile:filePath atomically:YES]) {
+//         NSLog(@"PNG file saved successfully at path: %@", filePath);
+//     } else {
+//         NSLog(@"Failed to save PNG file at path: %@", filePath);
+//     }
+
+//     return fileURL;
+// }
+
+NSArray<NSURL *> *saveCVImagesAsPNGs(std::vector<cv::Mat> frames, NSString *folderName) {
+    NSMutableArray<NSURL *> *fileURLs = [NSMutableArray arrayWithCapacity:frames.size()];
+
+    for (int i = 0; i < frames.size(); i++) {
+          // Create a unique file name for each image
+        NSString *fileNameIndex = [folderName stringByAppendingFormat:@"_%d", i];
+        NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:[fileNameIndex stringByAppendingString:@".png"]];
+        NSURL *fileURL = [NSURL fileURLWithPath:filePath];
+
+        // NSLog(@"File URL: %@", fileURL);
+
+        // Do NOT compress pixel
+        std::vector<int> compressionParams;
+        compressionParams.push_back(cv::IMWRITE_PNG_COMPRESSION);
+        compressionParams.push_back(0);
+
+        const char * cpath = [filePath cStringUsingEncoding:NSUTF8StringEncoding];
+        const cv::String newPath = (const cv::String)cpath;
+        cv::imwrite(newPath, frames[i], compressionParams);
+
+        // Add file URL to the array
+        [fileURLs addObject:fileURL];
+    }
+
+    return [fileURLs copy];
+}
+
+@end
+
+ 
+@implementation NSValue (IntPoint)
++ (instancetype)valuewithIntPoint:(IntPoint *)value {
+    return [self valueWithBytes:&value objCType:@encode(IntPoint)];
+}
+- (IntPoint *) intPointValue {
+    IntPoint* value;
+    [self getValue:&value];
+    return value;
+}
+@end
+
+@implementation FaceMeshLandmarkPoint
+@end
+
+@implementation FaceMeshNormalizedRect
+@end
+
+@implementation IntPoint
+- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y {
+    self = [super init];
+    if (self) {
+        _x = x;
+        _y = y;
+    }
+    return self;
+}
+@end

mediapipe/examples/ios/facemeshioslib/patch_ios_framework.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+set -eu
+set -o pipefail
+
+
+[[ $# -lt 2 ]] && echo "Usage: $0  <path/to/zipped .framework> <hdrs>..." && exit 1
+zipped=$(python3 -c "import os; print(os.path.realpath('$1'))"); shift
+name=$(basename "$zipped" .zip)
+parent=$(dirname "$zipped")
+named="$parent"/"$name".framework
+
+unzip "$zipped" -d "$parent"
+
+mkdir "$named"/Modules
+cat << EOF >"$named"/Modules/module.modulemap
+framework module $name {
+  umbrella header "$name.h"
+
+  export *
+  module * { export * }
+
+  link framework "AVFoundation"
+  link framework "Accelerate"
+  link framework "AssetsLibrary"
+  link framework "CoreFoundation"
+  link framework "CoreGraphics"
+  link framework "CoreImage"
+  link framework "CoreMedia"
+  link framework "CoreVideo"
+  link framework "GLKit"
+  link framework "Metal"
+  link framework "MetalKit"
+  link framework "OpenGLES"
+  link framework "QuartzCore"
+  link framework "UIKit"
+}
+EOF
+# NOTE: All these linked frameworks are required by mediapipe/objc.
+
+cat << EOF >"$named"/Headers/$name.h
+//
+//  $name.h
+//  $name
+//
+
+#import <Foundation/Foundation.h>
+
+//! Project version number for $name.
+FOUNDATION_EXPORT double ${name}VersionNumber;
+
+//! Project version string for $name.
+FOUNDATION_EXPORT const unsigned char ${name}VersionString[];
+
+// In this header, you should import all the public headers of your framework using statements like #import <$name/PublicHeader.h>
+
+EOF
+until [[ $# -eq 0 ]]; do
+  printf '#import "'"$1"'"\n' "$1" >>"$named"/Headers/$name.h
+  shift
+done

mediapipe/graphs/face_mesh/BUILD

@@ -67,3 +67,17 @@ mediapipe_binary_graph(
     output_name = "face_mesh_mobile_gpu.binarypb",
     deps = [":mobile_calculators"],
 )
+
+mediapipe_binary_graph(
+    name = "face_mesh_ios_lib_gpu_binary_graph",
+    graph = "face_mesh_ios_lib.pbtxt",
+    output_name = "face_mesh_ios_lib_gpu.binarypb",
+    deps = [":mobile_calculators"],
+)
+
+mediapipe_binary_graph(
+    name = "pure_face_mesh_mobile_gpu_binary_graph",
+    graph = "pure_face_mesh_mobile.pbtxt",
+    output_name = "pure_face_mesh_mobile_gpu.binarypb",
+    deps = [":mobile_calculators"],
+)

mediapipe/graphs/face_mesh/face_mesh_ios_lib.pbtxt

@@ -0,0 +1,70 @@
+# MediaPipe graph that performs face mesh with TensorFlow Lite on GPU.
+
+# GPU buffer. (GpuBuffer)
+input_stream: "input_video"
+
+# Max number of faces to detect/process. (int)
+input_side_packet: "num_faces"
+
+# Output image with rendered results. (GpuBuffer)
+# nope no rendering
+# output_stream: "output_video"
+# Collection of detected/processed faces, each represented as a list of
+# landmarks. (std::vector<NormalizedLandmarkList>)
+output_stream: "multi_face_landmarks"
+
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+# node {
+#  calculator: "FlowLimiterCalculator"
+# input_stream: "input_video"
+#  input_stream: "FINISHED:output_video"
+#  input_stream_info: {
+#    tag_index: "FINISHED"
+#    back_edge: true
+#  }
+#  output_stream: "throttled_input_video"
+#}
+
+# Defines side packets for further use in the graph.
+node {
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:with_attention"
+  node_options: {
+    [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
+      packet { bool_value: true }
+    }
+  }
+}
+
+# Subgraph that detects faces and corresponding landmarks.
+node {
+  calculator: "FaceLandmarkFrontGpu"
+  # input_stream: "IMAGE:throttled_input_video"
+  input_stream: "IMAGE:input_video"
+  input_side_packet: "NUM_FACES:num_faces"
+  input_side_packet: "WITH_ATTENTION:with_attention"
+  output_stream: "LANDMARKS:multi_face_landmarks"
+  output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
+  # output_stream: "DETECTIONS:face_detections"
+  # output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
+}
+
+# Subgraph that renders face-landmark annotation onto the input image.
+# node {
+#  calculator: "FaceRendererGpu"
+#  input_stream: "IMAGE:throttled_input_video"
+#  input_stream: "LANDMARKS:multi_face_landmarks"
+#  input_stream: "NORM_RECTS:face_rects_from_landmarks"
+#  input_stream: "DETECTIONS:face_detections"
+#  output_stream: "IMAGE:output_video"
+#}

mediapipe/graphs/face_mesh/pure_face_mesh_mobile.pbtxt

@@ -0,0 +1,74 @@
+# MediaPipe graph that performs face mesh with TensorFlow Lite on GPU.
+# Edited from face_mesh_mobile.pbtxt because I don't want babysitting of auto throttling and such. I'll do it myself
+
+# GPU buffer. (GpuBuffer)
+input_stream: "input_video"
+
+# Max number of faces to detect/process. (int)
+input_side_packet: "num_faces"
+
+# Output image with rendered results. (GpuBuffer)
+# nope no rendering
+# output_stream: "output_video"
+
+# Collection of detected/processed faces, each represented as a list of
+# landmarks. (std::vector<NormalizedLandmarkList>)
+output_stream: "multi_face_landmarks"
+
+# Regions of interest calculated based on landmarks.
+# (For more info see mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
+# (std::vector<NormalizedRect>)
+# For typings see "mediapipe/framework/formats/rect.pb.h"
+output_stream: "face_rects_from_landmarks"
+
+# The detections from the box model
+# see detection.proto
+# Regions of interest calculated based on face detections.
+# (std::vector<NormalizedRect>)
+# output_stream: "face_rects_from_detections"
+
+# Extra outputs (for debugging, for instance).
+# Detected faces. (std::vector<Detection>)
+# (std::vector<Detections>)
+# output_stream: "face_detections"
+
+# Landmark presence (needed because whole graph won't emit anything if no faces are detected)
+output_stream: "landmark_presence"
+
+# screw the throttling, we do that ourselves.
+# *throttling node code was deleted from here*
+
+# Subgraph that detects faces and corresponding landmarks.
+node {
+  calculator: "FaceLandmarkFrontGpu"
+# the IMAGE: part is saying, pipe this data into the input with the name `image`
+  input_stream: "IMAGE:input_video"
+  input_side_packet: "NUM_FACES:num_faces"
+  output_stream: "LANDMARKS:multi_face_landmarks"
+  output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
+  # face_detections is the stream that comes out from face_detection_short_range_common
+  # output_stream: "DETECTIONS:face_detections"
+  
+  # output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
+}
+
+# See this thread here https://github.com/google/mediapipe/issues/850#issuecomment-683268033
+# "if there are no packets in the corresponding output stream, it is designed to wait until the packet comes in"
+# That means that we'd get absolutely nothing to work with and won't know if our frame had anythin!
+# So we add PacketPresenceCalculator
+node {
+  calculator: "PacketPresenceCalculator"
+  input_stream: "PACKET:multi_face_landmarks"
+  output_stream: "PRESENCE:landmark_presence"
+}
+
+# nope not rendering.
+# Subgraph that renders face-landmark annotation onto the input image.
+# node {
+#  calculator: "FaceRendererGpu"
+#  input_stream: "IMAGE:throttled_input_video"
+#  input_stream: "LANDMARKS:multi_face_landmarks"
+#  input_stream: "NORM_RECTS:face_rects_from_landmarks"
+#  input_stream: "DETECTIONS:face_detections"
+#  output_stream: "IMAGE:output_video"
+#}