release facemesh sdk ios

WORKSPACE

@@ -375,6 +375,14 @@ http_archive(
     url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
 )
 
+# http_archive(
+#     name = "ios_opencv",
+#     sha256 = "c23e92c4a0cd343f73d4056e66c961cdf68e73ca699b1129638537a931c4ffc8",
+#     build_file = "@//third_party:opencv_ios.BUILD",
+#     type = "zip",
+#     url = "https://github.com/opencv/opencv/releases/download/4.7.0/opencv-4.7.0-ios-framework.zip",
+# )
+
 # Building an opencv.xcframework from the OpenCV 4.5.3 sources is necessary for
 # MediaPipe iOS Task Libraries to be supported on arm64(M1) Macs. An
 # `opencv.xcframework` archive has not been released and it is recommended to

build_face_mesh_ios_framework.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+sudo rm -rf frameworkbuild
+
 # Create output directories~
 mkdir -p ./frameworkbuild/FaceMeshSDK/arm64
 # XCFramework is how we're going to use it.

frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Headers/FaceMesh.h

@@ -0,0 +1,58 @@
+#import <CoreVideo/CoreVideo.h>
+#import <Foundation/Foundation.h>
+
+@interface IntPoint : NSObject
+@property(nonatomic) NSInteger x;
+@property(nonatomic) NSInteger y;
+
+- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y;
+@end
+
+@interface NSValue (IntPoint)
++ (instancetype)valuewithIntPoint:(IntPoint *)value;
+@property (readonly) IntPoint* intPointValue;
+@end
+
+@interface FaceMeshLandmarkPoint : NSObject
+@property(nonatomic) float x;
+@property(nonatomic) float y;
+@property(nonatomic) float z;
+@end
+
+@interface FaceMeshNormalizedRect : NSObject
+@property(nonatomic) float centerX;
+@property(nonatomic) float centerY;
+@property(nonatomic) float height;
+@property(nonatomic) float width;
+@property(nonatomic) float rotation;
+@end
+
+@protocol FaceMeshDelegate <NSObject>
+@optional
+/**
+ * Array of faces, with faces represented by arrays of face landmarks
+ */
+- (void)didReceiveFaces:(NSArray<NSArray<FaceMeshLandmarkPoint *> *> *)faces;
+
+- (void)didSavedRegions:(NSArray<NSURL *> *)foreheadURLs
+          leftcheekURLs:(NSArray<NSURL *> *)leftcheekURLs
+          rightcheekURLs:(NSArray<NSURL *> *)rightcheekURLs;
+@end
+
+@interface FaceMesh : NSObject
+- (instancetype)init;
+- (void)startGraph;
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer;
+- (CVPixelBufferRef)resize:(CVPixelBufferRef)pixelBuffer
+                     width:(int)width
+                    height:(int)height;
+- (uint8_t **)buffer2Array2D:(CVPixelBufferRef)pixelBuffer;
+- (void)extractRegions:(NSURL *)fileName
+            foreheadBoxes:(NSArray<NSArray<IntPoint *> *> *)foreheadBoxes
+            leftCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)leftCheekBoxes
+            rightCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)rightCheekBoxes
+            totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess
+            skipNFirstFrames:(NSInteger)skipNFirstFrames;
+@property(weak, nonatomic) id<FaceMeshDelegate> delegate;
+@property(nonatomic) size_t timestamp;
+@end

frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Headers/FaceMeshSDK.h

@@ -0,0 +1,16 @@
+//
+//  FaceMeshSDK.h
+//  FaceMeshSDK
+//
+
+#import <Foundation/Foundation.h>
+
+//! Project version number for FaceMeshSDK.
+FOUNDATION_EXPORT double FaceMeshSDKVersionNumber;
+
+//! Project version string for FaceMeshSDK.
+FOUNDATION_EXPORT const unsigned char FaceMeshSDKVersionString[];
+
+// In this header, you should import all the public headers of your framework using statements like #import <FaceMeshSDK/PublicHeader.h>
+
+#import "FaceMesh.h"

frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Modules/module.modulemap

@@ -0,0 +1,21 @@
+framework module FaceMeshSDK {
+  umbrella header "FaceMeshSDK.h"
+
+  export *
+  module * { export * }
+
+  link framework "AVFoundation"
+  link framework "Accelerate"
+  link framework "AssetsLibrary"
+  link framework "CoreFoundation"
+  link framework "CoreGraphics"
+  link framework "CoreImage"
+  link framework "CoreMedia"
+  link framework "CoreVideo"
+  link framework "GLKit"
+  link framework "Metal"
+  link framework "MetalKit"
+  link framework "OpenGLES"
+  link framework "QuartzCore"
+  link framework "UIKit"
+}

frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_mesh_ios_lib_gpu.binarypb

@@ -0,0 +1,5 @@
+
+€
ConstantSidePacketCalculator2PACKET:with_attentionBI
+Atype.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions
+

+¬
FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces*WITH_ATTENTION:with_attentionRinput_videozmulti_face_landmarks‚
	num_faces

frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/pure_face_mesh_mobile_gpu.binarypb

@@ -0,0 +1,3 @@
+
+<EFBFBD>
FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces
+SPacketPresenceCalculatorPACKET:multi_face_landmarks"PRESENCE:landmark_presenceRinput_videozmulti_face_landmarkszface_rects_from_landmarkszlandmark_presence‚
	num_faces

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/Info.plist

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>AvailableLibraries</key>
+	<array>
+		<dict>
+			<key>LibraryIdentifier</key>
+			<string>ios-arm64</string>
+			<key>LibraryPath</key>
+			<string>FaceMeshSDK.framework</string>
+			<key>SupportedArchitectures</key>
+			<array>
+				<string>arm64</string>
+			</array>
+			<key>SupportedPlatform</key>
+			<string>ios</string>
+		</dict>
+	</array>
+	<key>CFBundlePackageType</key>
+	<string>XFWK</string>
+	<key>XCFrameworkFormatVersion</key>
+	<string>1.0</string>
+</dict>
+</plist>

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Headers/FaceMesh.h

@@ -0,0 +1,58 @@
+#import <CoreVideo/CoreVideo.h>
+#import <Foundation/Foundation.h>
+
+@interface IntPoint : NSObject
+@property(nonatomic) NSInteger x;
+@property(nonatomic) NSInteger y;
+
+- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y;
+@end
+
+@interface NSValue (IntPoint)
++ (instancetype)valuewithIntPoint:(IntPoint *)value;
+@property (readonly) IntPoint* intPointValue;
+@end
+
+@interface FaceMeshLandmarkPoint : NSObject
+@property(nonatomic) float x;
+@property(nonatomic) float y;
+@property(nonatomic) float z;
+@end
+
+@interface FaceMeshNormalizedRect : NSObject
+@property(nonatomic) float centerX;
+@property(nonatomic) float centerY;
+@property(nonatomic) float height;
+@property(nonatomic) float width;
+@property(nonatomic) float rotation;
+@end
+
+@protocol FaceMeshDelegate <NSObject>
+@optional
+/**
+ * Array of faces, with faces represented by arrays of face landmarks
+ */
+- (void)didReceiveFaces:(NSArray<NSArray<FaceMeshLandmarkPoint *> *> *)faces;
+
+- (void)didSavedRegions:(NSArray<NSURL *> *)foreheadURLs
+          leftcheekURLs:(NSArray<NSURL *> *)leftcheekURLs
+          rightcheekURLs:(NSArray<NSURL *> *)rightcheekURLs;
+@end
+
+@interface FaceMesh : NSObject
+- (instancetype)init;
+- (void)startGraph;
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer;
+- (CVPixelBufferRef)resize:(CVPixelBufferRef)pixelBuffer
+                     width:(int)width
+                    height:(int)height;
+- (uint8_t **)buffer2Array2D:(CVPixelBufferRef)pixelBuffer;
+- (void)extractRegions:(NSURL *)fileName
+            foreheadBoxes:(NSArray<NSArray<IntPoint *> *> *)foreheadBoxes
+            leftCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)leftCheekBoxes
+            rightCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)rightCheekBoxes
+            totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess
+            skipNFirstFrames:(NSInteger)skipNFirstFrames;
+@property(weak, nonatomic) id<FaceMeshDelegate> delegate;
+@property(nonatomic) size_t timestamp;
+@end

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Headers/FaceMeshSDK.h

@@ -0,0 +1,16 @@
+//
+//  FaceMeshSDK.h
+//  FaceMeshSDK
+//
+
+#import <Foundation/Foundation.h>
+
+//! Project version number for FaceMeshSDK.
+FOUNDATION_EXPORT double FaceMeshSDKVersionNumber;
+
+//! Project version string for FaceMeshSDK.
+FOUNDATION_EXPORT const unsigned char FaceMeshSDKVersionString[];
+
+// In this header, you should import all the public headers of your framework using statements like #import <FaceMeshSDK/PublicHeader.h>
+
+#import "FaceMesh.h"

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Modules/module.modulemap

@@ -0,0 +1,21 @@
+framework module FaceMeshSDK {
+  umbrella header "FaceMeshSDK.h"
+
+  export *
+  module * { export * }
+
+  link framework "AVFoundation"
+  link framework "Accelerate"
+  link framework "AssetsLibrary"
+  link framework "CoreFoundation"
+  link framework "CoreGraphics"
+  link framework "CoreImage"
+  link framework "CoreMedia"
+  link framework "CoreVideo"
+  link framework "GLKit"
+  link framework "Metal"
+  link framework "MetalKit"
+  link framework "OpenGLES"
+  link framework "QuartzCore"
+  link framework "UIKit"
+}

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_mesh_ios_lib_gpu.binarypb

@@ -0,0 +1,5 @@
+
+€
ConstantSidePacketCalculator2PACKET:with_attentionBI
+Atype.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions
+

+¬
FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces*WITH_ATTENTION:with_attentionRinput_videozmulti_face_landmarks‚
	num_faces

frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/pure_face_mesh_mobile_gpu.binarypb

@@ -0,0 +1,3 @@
+
+<EFBFBD>
FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces
+SPacketPresenceCalculatorPACKET:multi_face_landmarks"PRESENCE:landmark_presenceRinput_videozmulti_face_landmarkszface_rects_from_landmarkszlandmark_presence‚
	num_faces

mediapipe/examples/ios/facemeshioslib/BUILD

@@ -56,6 +56,8 @@ objc_library(
         "//mediapipe/objc:mediapipe_framework_ios",
         "//mediapipe/objc:mediapipe_input_sources_ios",
         "//mediapipe/calculators/core:packet_presence_calculator",
+        "//mediapipe/framework/port:opencv_video",
+        "//mediapipe/framework/port:opencv_imgcodecs",
 #        "//mediapipe/objc:mediapipe_layer_renderer", # no need for layer renderer since I don't render
     ] + select({
         "//conditions:default": [

mediapipe/examples/ios/facemeshioslib/FaceMesh.h

@@ -1,6 +1,18 @@
 #import <CoreVideo/CoreVideo.h>
 #import <Foundation/Foundation.h>
 
+@interface IntPoint : NSObject
+@property(nonatomic) NSInteger x;
+@property(nonatomic) NSInteger y;
+
+- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y;
+@end
+
+@interface NSValue (IntPoint)
++ (instancetype)valuewithIntPoint:(IntPoint *)value;
+@property (readonly) IntPoint* intPointValue;
+@end
+
 @interface FaceMeshLandmarkPoint : NSObject
 @property(nonatomic) float x;
 @property(nonatomic) float y;
@@ -21,6 +33,10 @@
  * Array of faces, with faces represented by arrays of face landmarks
  */
 - (void)didReceiveFaces:(NSArray<NSArray<FaceMeshLandmarkPoint *> *> *)faces;
+
+- (void)didSavedRegions:(NSArray<NSURL *> *)foreheadURLs
+          leftcheekURLs:(NSArray<NSURL *> *)leftcheekURLs
+          rightcheekURLs:(NSArray<NSURL *> *)rightcheekURLs;
 @end
 
 @interface FaceMesh : NSObject
@@ -30,6 +46,13 @@
 - (CVPixelBufferRef)resize:(CVPixelBufferRef)pixelBuffer
                      width:(int)width
                     height:(int)height;
+- (uint8_t **)buffer2Array2D:(CVPixelBufferRef)pixelBuffer;
+- (void)extractRegions:(NSURL *)fileName
+            foreheadBoxes:(NSArray<NSArray<IntPoint *> *> *)foreheadBoxes
+            leftCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)leftCheekBoxes
+            rightCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)rightCheekBoxes
+            totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess
+            skipNFirstFrames:(NSInteger)skipNFirstFrames;
 @property(weak, nonatomic) id<FaceMeshDelegate> delegate;
 @property(nonatomic) size_t timestamp;
 @end

mediapipe/examples/ios/facemeshioslib/FaceMesh.mm

@@ -8,11 +8,17 @@
 
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_video_inc.h"
+#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
+#include "opencv2/imgcodecs/ios.h" 
+
+#import <UIKit/UIKit.h>
 
 //#import "mediapipe/objc/MPPLayerRenderer.h"
 
 // The graph name specified is supposed to be the same as in the pb file (binarypb?)
 static NSString* const kGraphName = @"face_mesh_ios_lib_gpu";
+// static NSString* const kGraphName = @"pure_face_mesh_mobile_gpu";
 
 static const char* kInputStream = "input_video";
 static const char* kNumFacesInputSidePacket = "num_faces";
@@ -111,7 +117,7 @@ static const int kNumFaces = 1;
     }
     const auto& multi_face_landmarks = packet.Get<std::vector<::mediapipe::NormalizedLandmarkList>>();
     // NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(),
-          // multi_face_landmarks.size());
+    //       multi_face_landmarks.size());
     NSMutableArray <NSArray <FaceMeshLandmarkPoint *>*>*faceLandmarks = [NSMutableArray new];
     
     for (int face_index = 0; face_index < multi_face_landmarks.size(); ++face_index) {
@@ -157,73 +163,235 @@ static const int kNumFaces = 1;
   }
 }
 
-// Resize the CVPixelBufferRef with INTER_AREA.
-- (CVPixelBufferRef)resize:(CVPixelBufferRef)pixelBuffer
-                    width:(int)width
-                    height:(int)height {
-  
-  OSType srcType = CVPixelBufferGetPixelFormatType(pixelBuffer);
-  size_t channels = 2;
-  if (srcType == kCVPixelFormatType_32ARGB || srcType == kCVPixelFormatType_32BGRA) {
-    channels = 4;
-  }
-  
-  // Lock the CVPixelBuffer
-  CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+- (void)extractRegions:(NSURL *)fileName
+            foreheadBoxes:(NSArray<NSArray<IntPoint *> *> *)foreheadBoxes
+            leftCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)leftCheekBoxes
+            rightCheekBoxes:(NSArray<NSArray<IntPoint *> *> *)rightCheekBoxes
+            totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess
+            skipNFirstFrames:(NSInteger)skipNFirstFrames {
 
-  // Get the pixel buffer attributes
-  size_t srcWidth = CVPixelBufferGetWidth(pixelBuffer);
-  size_t srcHeight = CVPixelBufferGetHeight(pixelBuffer);
-  size_t bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
+    NSString *filename = fileName.path;
+    std::string filePath = [filename UTF8String];
 
-  // Get the base address of the pixel buffer
-  unsigned char *baseAddress = (unsigned char *)CVPixelBufferGetBaseAddress(pixelBuffer);
+    cv::VideoCapture vid(filePath);
 
-  // Create a cv::Mat without copying the data
-  cv::Mat argbImage(srcHeight, srcWidth, CV_8UC(channels), baseAddress, bytesPerRow);
+    if (!vid.isOpened()) {
+        printf("@Error Opening video file");
+    }
+    else {
+        printf("File Opened AAAA");
 
-  // Create a cv::Mat to hold the resized image
-  cv::Mat resizedImage;
+        // NSMutableArray *foreheadURLs = [NSMutableArray new];
+        // NSMutableArray *leftcheekURLs = [NSMutableArray new];
+        // NSMutableArray *rightcheekURLs = [NSMutableArray new];
 
-  // Resize the image using cv::resize
-  cv::resize(argbImage, resizedImage, cv::Size(width, height), 0, 0, cv::INTER_AREA);
+        int startFrame = int(vid.get(cv::CAP_PROP_POS_FRAMES));
+        int totalFrame = int(vid.get(cv::CAP_PROP_FRAME_COUNT));
+        // int nframes = totalFrame - startFrame;
 
-  // Unlock the CVPixelBuffer
-  CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+        // if (totalFramesNeedProcess > nframes) {
+        //   NSLog(@"Video too short");
+        //   return;
+        // }
 
-  // Create a new CVPixelBuffer with the desired size and format
-  CVPixelBufferRef resizedPixelBuffer;
-  CVReturn result = CVPixelBufferCreate(NULL, width, height, srcType, NULL, &resizedPixelBuffer);
-  if (result != kCVReturnSuccess) {
-      NSLog(@"Failed to create CVPixelBuffer. Error: %d", result);
-      return nil;
-  }
+        if (skipNFirstFrames < 0 || totalFramesNeedProcess < 0) {
+          vid.release();
+          return;
+        }
 
-  // Lock the resized CVPixelBuffer
-  CVPixelBufferLockBaseAddress(resizedPixelBuffer, 0);
+        int frameIdx = skipNFirstFrames;
+        int maxFrameIndex = totalFramesNeedProcess;
 
-  // Get the base address and bytes per row of the resized pixel buffer
-  void *resizedBaseAddress = CVPixelBufferGetBaseAddress(resizedPixelBuffer);
-  size_t resizedBytesPerRow = CVPixelBufferGetBytesPerRow(resizedPixelBuffer);
+        if (skipNFirstFrames > 0) {
+          maxFrameIndex += skipNFirstFrames;
+        }
 
-  // Create a cv::Mat wrapper for the resized pixel buffer
-  cv::Mat resizedPixelBufferMat(height, width, CV_8UC(channels), resizedBaseAddress, resizedBytesPerRow);
+        // Process forehead
+        std::vector<cv::Mat> foreheads;
+        std::vector<cv::Mat> leftcheeks;
+        std::vector<cv::Mat> rightcheeks;
+        
+        while (frameIdx < maxFrameIndex) {
+          cv::Mat frame;
 
-  // Convert the resized image (cv::Mat) to the resized pixel buffer (CVPixelBuffer)
-  resizedImage.copyTo(resizedPixelBufferMat);
+          if (!vid.read(frame)) {
+              NSLog(@"Failed to read frame.");
+              break;
+          }
+          cv::cvtColor(frame, frame, cv::COLOR_BGR2RGB);
+          // Process the frame (e.g., display or save it)
 
-  // Unlock the resized CVPixelBuffer
-  CVPixelBufferUnlockBaseAddress(resizedPixelBuffer, 0);
+          NSLog(@"frame index: %d", frameIdx);
 
-  // Return the resized CVPixelBuffer
-  return resizedPixelBuffer;
+          NSMutableArray *rowForehead = [foreheadBoxes objectAtIndex:(frameIdx - skipNFirstFrames)];
+          NSMutableArray *rowLeftCheek = [leftCheekBoxes objectAtIndex:(frameIdx - skipNFirstFrames)];
+          NSMutableArray *rowRightCheek = [rightCheekBoxes objectAtIndex:(frameIdx - skipNFirstFrames)];
+
+          cv::Mat forehead = extractRegion(frame, rowForehead);
+          cv::Mat leftCheek = extractRegion(frame, rowLeftCheek);
+          cv::Mat rightCheek = extractRegion(frame, rowRightCheek);
+
+          foreheads.push_back(forehead);
+          leftcheeks.push_back(leftCheek);
+          rightcheeks.push_back(rightCheek);
+          
+          frameIdx++;
+        }
+
+        NSLog(@"total foreheads: %d", foreheads.size());
+        NSLog(@"total leftCheeks: %d", leftcheeks.size());
+        NSLog(@"total rightCheeks: %d", rightcheeks.size());
+
+        // for (int i = 0; i < foreheads.size(); i++) {
+        //   cv::Mat rgbaImage;
+        //   cv::cvtColor(foreheads[i], rgbaImage, cv::COLOR_RGB2RGBA);
+        //   saveCVImageAsPNG(MatToUIImage(rgbaImage), @"forehead");
+        // }
+
+        NSMutableArray *foreheadURLs = saveCVImagesAsPNGs(foreheads, @"forehead");
+        NSMutableArray *leftcheekURLs = saveCVImagesAsPNGs(leftcheeks, @"leftcheek");
+        NSMutableArray *rightcheekURLs = saveCVImagesAsPNGs(rightcheeks, @"rightcheek");
+
+        // cv::cvtColor(leftCheeks[0], rgbaImage, cv::COLOR_RGB2RGBA);
+        // NSData *firstData = [NSData dataWithBytes:rgbaImage.data length:rgbaImage.total() * rgbaImage.elemSize()];
+        // saveCVImageAsPNG([UIImage imageWithData:firstData], @"leftcheek");
+
+        // cv::cvtColor(rightCheeks[0], rgbaImage, cv::COLOR_RGB2RGBA);
+        // NSData *firstData = [NSData dataWithBytes:rgbaImage.data length:rgbaImage.total() * rgbaImage.elemSize()];
+        // saveCVImageAsPNG([UIImage imageWithData:firstData], @"rightcheek");
+
+        if([self.delegate respondsToSelector:@selector(didSavedRegions:leftcheekURLs:rightcheekURLs:)]) {
+            NSLog(@"nguyencse ==> has didSavedRegions");
+            [self.delegate didSavedRegions:foreheadURLs leftcheekURLs:leftcheekURLs rightcheekURLs:rightcheekURLs];
+        }
+    }
+
+    vid.release();
+}
+
+cv::Mat extractRegion(cv::Mat img, NSArray<IntPoint *> *box) {
+  IntPoint* point0 = [box objectAtIndex:0];
+  IntPoint* point1 = [box objectAtIndex:1];
+  IntPoint* point2 = [box objectAtIndex:2];
+  IntPoint* point3 = [box objectAtIndex:3];
+
+  // LEFT TOP --> RIGHT TOP --> RIGHT BOTTOM --> LEFT BOTTOM
+  int frameWidth = point1.x - point0.x;
+  int frameHeight = point3.y - point0.y;
+  cv::Mat region = cropROI(img, cv::Rect(point0.x, point0.y, frameWidth, frameHeight));
+
+  // square
+  region = square(region);
+
+  // resize to 32x32
+  region = resizeWithAreaInterpolation(region, cv::Size(32, 32));
+
+  return region;
+}
+
+cv::Mat cropROI(cv::Mat src, cv::Rect roi) {
+  // Crop the full image to that image contained by the rectangle myROI
+  // Note that this doesn't copy the data
+  cv::Mat croppedRef(src, roi);
+
+  cv::Mat cropped;
+  // Copy the data into new matrix
+  croppedRef.copyTo(cropped);
+
+  return cropped;
+}
+
+cv::Mat square(cv::Mat frame) {
+    if (frame.rows < frame.cols) {
+        int diff = frame.cols - frame.rows;
+        cv::Mat pad(frame.rows, diff, frame.type(), cv::Scalar(0));
+        cv::hconcat(frame, pad, frame);
+    } else if (frame.rows > frame.cols) {
+        int diff = frame.rows - frame.cols;
+        cv::Mat pad(diff, frame.cols, frame.type(), cv::Scalar(0));
+        cv::vconcat(frame, pad, frame);
+    }
+    
+    return frame;
+}
+
+cv::Mat resizeWithAreaInterpolation(cv::Mat image, cv::Size newShape) {
+    cv::Size originalShape = image.size();
+    cv::resize(image, image, newShape, 0, 0, cv::INTER_AREA);
+    return image;
+}
+
+// NSURL *saveCVImageAsPNG(UIImage *image, NSString *fileName) {
+//     // Create a unique file name for each image
+//     NSString *fileNameIndex = [fileName stringByAppendingFormat:@"_%d", 0];
+//     NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:[fileNameIndex stringByAppendingString:@".png"]];
+//     NSURL *fileURL = [NSURL fileURLWithPath:filePath];
+
+//     NSData *pngData = UIImagePNGRepresentation(image);
+
+//     if ([pngData writeToFile:filePath atomically:YES]) {
+//         NSLog(@"PNG file saved successfully at path: %@", filePath);
+//     } else {
+//         NSLog(@"Failed to save PNG file at path: %@", filePath);
+//     }
+
+//     return fileURL;
+// }
+
+NSArray<NSURL *> *saveCVImagesAsPNGs(std::vector<cv::Mat> frames, NSString *folderName) {
+    NSMutableArray<NSURL *> *fileURLs = [NSMutableArray arrayWithCapacity:frames.size()];
+
+    for (int i = 0; i < frames.size(); i++) {
+          // Create a unique file name for each image
+        NSString *fileNameIndex = [folderName stringByAppendingFormat:@"_%d", i];
+        NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:[fileNameIndex stringByAppendingString:@".png"]];
+        NSURL *fileURL = [NSURL fileURLWithPath:filePath];
+
+        // NSLog(@"File URL: %@", fileURL);
+
+        // Do NOT compress pixel
+        std::vector<int> compressionParams;
+        compressionParams.push_back(cv::IMWRITE_PNG_COMPRESSION);
+        compressionParams.push_back(0);
+
+        const char * cpath = [filePath cStringUsingEncoding:NSUTF8StringEncoding];
+        const cv::String newPath = (const cv::String)cpath;
+        cv::imwrite(newPath, frames[i], compressionParams);
+
+        // Add file URL to the array
+        [fileURLs addObject:fileURL];
+    }
+
+    return [fileURLs copy];
 }
 
 @end
 
+ 
+@implementation NSValue (IntPoint)
++ (instancetype)valuewithIntPoint:(IntPoint *)value {
+    return [self valueWithBytes:&value objCType:@encode(IntPoint)];
+}
+- (IntPoint *) intPointValue {
+    IntPoint* value;
+    [self getValue:&value];
+    return value;
+}
+@end
 
 @implementation FaceMeshLandmarkPoint
 @end
 
 @implementation FaceMeshNormalizedRect
 @end
+
+@implementation IntPoint
+- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y {
+    self = [super init];
+    if (self) {
+        _x = x;
+        _y = y;
+    }
+    return self;
+}
+@end

mediapipe/graphs/face_mesh/BUILD

@@ -74,3 +74,10 @@ mediapipe_binary_graph(
     output_name = "face_mesh_ios_lib_gpu.binarypb",
     deps = [":mobile_calculators"],
 )
+
+mediapipe_binary_graph(
+    name = "pure_face_mesh_mobile_gpu_binary_graph",
+    graph = "pure_face_mesh_mobile.pbtxt",
+    output_name = "pure_face_mesh_mobile_gpu.binarypb",
+    deps = [":mobile_calculators"],
+)

mediapipe/graphs/face_mesh/pure_face_mesh_mobile.pbtxt

@@ -0,0 +1,74 @@
+# MediaPipe graph that performs face mesh with TensorFlow Lite on GPU.
+# Edited from face_mesh_mobile.pbtxt because I don't want babysitting of auto throttling and such. I'll do it myself
+
+# GPU buffer. (GpuBuffer)
+input_stream: "input_video"
+
+# Max number of faces to detect/process. (int)
+input_side_packet: "num_faces"
+
+# Output image with rendered results. (GpuBuffer)
+# nope no rendering
+# output_stream: "output_video"
+
+# Collection of detected/processed faces, each represented as a list of
+# landmarks. (std::vector<NormalizedLandmarkList>)
+output_stream: "multi_face_landmarks"
+
+# Regions of interest calculated based on landmarks.
+# (For more info see mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)
+# (std::vector<NormalizedRect>)
+# For typings see "mediapipe/framework/formats/rect.pb.h"
+output_stream: "face_rects_from_landmarks"
+
+# The detections from the box model
+# see detection.proto
+# Regions of interest calculated based on face detections.
+# (std::vector<NormalizedRect>)
+# output_stream: "face_rects_from_detections"
+
+# Extra outputs (for debugging, for instance).
+# Detected faces. (std::vector<Detection>)
+# (std::vector<Detections>)
+# output_stream: "face_detections"
+
+# Landmark presence (needed because whole graph won't emit anything if no faces are detected)
+output_stream: "landmark_presence"
+
+# screw the throttling, we do that ourselves.
+# *throttling node code was deleted from here*
+
+# Subgraph that detects faces and corresponding landmarks.
+node {
+  calculator: "FaceLandmarkFrontGpu"
+# the IMAGE: part is saying, pipe this data into the input with the name `image`
+  input_stream: "IMAGE:input_video"
+  input_side_packet: "NUM_FACES:num_faces"
+  output_stream: "LANDMARKS:multi_face_landmarks"
+  output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
+  # face_detections is the stream that comes out from face_detection_short_range_common
+  # output_stream: "DETECTIONS:face_detections"
+  
+  # output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
+}
+
+# See this thread here https://github.com/google/mediapipe/issues/850#issuecomment-683268033
+# "if there are no packets in the corresponding output stream, it is designed to wait until the packet comes in"
+# That means that we'd get absolutely nothing to work with and won't know if our frame had anythin!
+# So we add PacketPresenceCalculator
+node {
+  calculator: "PacketPresenceCalculator"
+  input_stream: "PACKET:multi_face_landmarks"
+  output_stream: "PRESENCE:landmark_presence"
+}
+
+# nope not rendering.
+# Subgraph that renders face-landmark annotation onto the input image.
+# node {
+#  calculator: "FaceRendererGpu"
+#  input_stream: "IMAGE:throttled_input_video"
+#  input_stream: "LANDMARKS:multi_face_landmarks"
+#  input_stream: "NORM_RECTS:face_rects_from_landmarks"
+#  input_stream: "DETECTIONS:face_detections"
+#  output_stream: "IMAGE:output_video"
+#}