diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 000000000..d72ac5020 Binary files /dev/null and b/.DS_Store differ diff --git a/WORKSPACE b/WORKSPACE index 3a539569f..dec641046 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -390,6 +390,14 @@ http_archive( url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip", ) +# http_archive( +# name = "ios_opencv", +# sha256 = "c23e92c4a0cd343f73d4056e66c961cdf68e73ca699b1129638537a931c4ffc8", +# build_file = "@//third_party:opencv_ios.BUILD", +# type = "zip", +# url = "https://github.com/opencv/opencv/releases/download/4.7.0/opencv-4.7.0-ios-framework.zip", +# ) + # Building an opencv.xcframework from the OpenCV 4.5.3 sources is necessary for # MediaPipe iOS Task Libraries to be supported on arm64(M1) Macs. An # `opencv.xcframework` archive has not been released and it is recommended to diff --git a/build_face_mesh_ios_framework.sh b/build_face_mesh_ios_framework.sh new file mode 100644 index 000000000..616aa5074 --- /dev/null +++ b/build_face_mesh_ios_framework.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +sudo rm -rf frameworkbuild + +# Create output directories~ +mkdir -p ./frameworkbuild/FaceMeshSDK/arm64 +# XCFramework is how we're going to use it. +mkdir -p ./frameworkbuild/FaceMeshSDK/xcframework + +# Interesting fact. Bazel `build` command stores cached files in `/private/var/tmp/...` folders +# and when you run build, if it finds cached files, it kind of symlinks the files/folders +# into the `bazel-bin` folder found in the project root. So don't be afraid of re-running builds +# because the files are cached. + +# build the arm64 binary framework +# bazel build --copt=-fembed-bitcode --apple_bitcode=embedded --config=ios_arm64 mediapipe/examples/ios/facemeshioslib:FaceMeshIOSLibFramework +bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/facemeshioslib:FaceMeshSDK +# use --cxxopt=-O3 to reduce framework size +# bazel build --copt=-O3 --cxxopt=-O3 --config=ios_arm64 mediapipe/examples/ios/facemeshioslib:FaceMeshIOSLibFramework + +# The arm64 framework zip will be located at //bazel-bin/mediapipe/examples/ios/facemeshioslib/FaceMeshIOSLibFramework.zip + +# Call the framework patcher (First argument = compressed framework.zip, Second argument = header file's name(in this case FaceMeshIOSLib.h)) +sudo bash ./mediapipe/examples/ios/facemeshioslib/patch_ios_framework.sh ./bazel-bin/mediapipe/examples/ios/facemeshioslib/FaceMeshSDK.zip FaceMesh.h + +# There will be a resulting patched .framework folder at the same directory, this is our arm64 one, we copy it to our arm64 folder +sudo cp -a ./bazel-bin/mediapipe/examples/ios/facemeshioslib/FaceMeshSDK.framework ./frameworkbuild/FaceMeshSDK/arm64 + +# Create xcframework (because the classic lipo method with normal .framework no longer works (shows Building for iOS Simulator, but the linked and embedded framework was built for iOS + iOS Simulator)) + +sudo xcodebuild -create-xcframework \ + -framework ./frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework \ + -output ./frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework diff --git a/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/FaceMeshSDK b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/FaceMeshSDK new file mode 100755 index 000000000..f3c9925ab Binary files /dev/null and b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/FaceMeshSDK differ diff --git a/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Headers/FaceMesh.h b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Headers/FaceMesh.h new file mode 100644 index 000000000..097ac53de --- /dev/null +++ b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Headers/FaceMesh.h @@ -0,0 +1,58 @@ +#import +#import + +@interface IntPoint : NSObject +@property(nonatomic) NSInteger x; +@property(nonatomic) NSInteger y; + +- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y; +@end + +@interface NSValue (IntPoint) ++ (instancetype)valuewithIntPoint:(IntPoint *)value; +@property (readonly) IntPoint* intPointValue; +@end + +@interface FaceMeshLandmarkPoint : NSObject +@property(nonatomic) float x; +@property(nonatomic) float y; +@property(nonatomic) float z; +@end + +@interface FaceMeshNormalizedRect : NSObject +@property(nonatomic) float centerX; +@property(nonatomic) float centerY; +@property(nonatomic) float height; +@property(nonatomic) float width; +@property(nonatomic) float rotation; +@end + +@protocol FaceMeshDelegate +@optional +/** + * Array of faces, with faces represented by arrays of face landmarks + */ +- (void)didReceiveFaces:(NSArray *> *)faces; + +- (void)didSavedRegions:(NSArray *)foreheadURLs + leftcheekURLs:(NSArray *)leftcheekURLs + rightcheekURLs:(NSArray *)rightcheekURLs; +@end + +@interface FaceMesh : NSObject +- (instancetype)init; +- (void)startGraph; +- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer; +- (CVPixelBufferRef)resize:(CVPixelBufferRef)pixelBuffer + width:(int)width + height:(int)height; +- (uint8_t **)buffer2Array2D:(CVPixelBufferRef)pixelBuffer; +- (void)extractRegions:(NSURL *)fileName + foreheadBoxes:(NSArray *> *)foreheadBoxes + leftCheekBoxes:(NSArray *> *)leftCheekBoxes + rightCheekBoxes:(NSArray *> *)rightCheekBoxes + totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess + skipNFirstFrames:(NSInteger)skipNFirstFrames; +@property(weak, nonatomic) id delegate; +@property(nonatomic) size_t timestamp; +@end \ No newline at end of file diff --git a/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Headers/FaceMeshSDK.h b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Headers/FaceMeshSDK.h new file mode 100644 index 000000000..fb82b013d --- /dev/null +++ b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Headers/FaceMeshSDK.h @@ -0,0 +1,16 @@ +// +// FaceMeshSDK.h +// FaceMeshSDK +// + +#import + +//! Project version number for FaceMeshSDK. +FOUNDATION_EXPORT double FaceMeshSDKVersionNumber; + +//! Project version string for FaceMeshSDK. +FOUNDATION_EXPORT const unsigned char FaceMeshSDKVersionString[]; + +// In this header, you should import all the public headers of your framework using statements like #import + +#import "FaceMesh.h" diff --git a/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Info.plist b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Info.plist new file mode 100755 index 000000000..7a8f526d5 Binary files /dev/null and b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Info.plist differ diff --git a/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Modules/module.modulemap b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Modules/module.modulemap new file mode 100644 index 000000000..a9a2157ac --- /dev/null +++ b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/Modules/module.modulemap @@ -0,0 +1,21 @@ +framework module FaceMeshSDK { + umbrella header "FaceMeshSDK.h" + + export * + module * { export * } + + link framework "AVFoundation" + link framework "Accelerate" + link framework "AssetsLibrary" + link framework "CoreFoundation" + link framework "CoreGraphics" + link framework "CoreImage" + link framework "CoreMedia" + link framework "CoreVideo" + link framework "GLKit" + link framework "Metal" + link framework "MetalKit" + link framework "OpenGLES" + link framework "QuartzCore" + link framework "UIKit" +} diff --git a/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_detection_short_range.tflite b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_detection_short_range.tflite new file mode 100755 index 000000000..3b865a7dc Binary files /dev/null and b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_detection_short_range.tflite differ diff --git a/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_landmark.tflite b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_landmark.tflite new file mode 100755 index 000000000..277d26f2f Binary files /dev/null and b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_landmark.tflite differ diff --git a/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_landmark_with_attention.tflite b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_landmark_with_attention.tflite new file mode 100755 index 000000000..550ca8b06 Binary files /dev/null and b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_landmark_with_attention.tflite differ diff --git a/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_mesh_ios_lib_gpu.binarypb b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_mesh_ios_lib_gpu.binarypb new file mode 100755 index 000000000..e1f12ace9 --- /dev/null +++ b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/face_mesh_ios_lib_gpu.binarypb @@ -0,0 +1,5 @@ + +€ConstantSidePacketCalculator2PACKET:with_attentionBI +Atype.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions + +¬FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces*WITH_ATTENTION:with_attentionR input_videozmulti_face_landmarks‚ num_faces \ No newline at end of file diff --git a/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/pure_face_mesh_mobile_gpu.binarypb b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/pure_face_mesh_mobile_gpu.binarypb new file mode 100755 index 000000000..c59288f56 --- /dev/null +++ b/frameworkbuild/FaceMeshSDK/arm64/FaceMeshSDK.framework/pure_face_mesh_mobile_gpu.binarypb @@ -0,0 +1,3 @@ + +¨FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces +SPacketPresenceCalculatorPACKET:multi_face_landmarks"PRESENCE:landmark_presenceR input_videozmulti_face_landmarkszface_rects_from_landmarkszlandmark_presence‚ num_faces \ No newline at end of file diff --git a/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/Info.plist b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/Info.plist new file mode 100644 index 000000000..11e3da625 --- /dev/null +++ b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/Info.plist @@ -0,0 +1,25 @@ + + + + + AvailableLibraries + + + LibraryIdentifier + ios-arm64 + LibraryPath + FaceMeshSDK.framework + SupportedArchitectures + + arm64 + + SupportedPlatform + ios + + + CFBundlePackageType + XFWK + XCFrameworkFormatVersion + 1.0 + + diff --git a/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/FaceMeshSDK b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/FaceMeshSDK new file mode 100755 index 000000000..f3c9925ab Binary files /dev/null and b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/FaceMeshSDK differ diff --git a/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Headers/FaceMesh.h b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Headers/FaceMesh.h new file mode 100644 index 000000000..097ac53de --- /dev/null +++ b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Headers/FaceMesh.h @@ -0,0 +1,58 @@ +#import +#import + +@interface IntPoint : NSObject +@property(nonatomic) NSInteger x; +@property(nonatomic) NSInteger y; + +- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y; +@end + +@interface NSValue (IntPoint) ++ (instancetype)valuewithIntPoint:(IntPoint *)value; +@property (readonly) IntPoint* intPointValue; +@end + +@interface FaceMeshLandmarkPoint : NSObject +@property(nonatomic) float x; +@property(nonatomic) float y; +@property(nonatomic) float z; +@end + +@interface FaceMeshNormalizedRect : NSObject +@property(nonatomic) float centerX; +@property(nonatomic) float centerY; +@property(nonatomic) float height; +@property(nonatomic) float width; +@property(nonatomic) float rotation; +@end + +@protocol FaceMeshDelegate +@optional +/** + * Array of faces, with faces represented by arrays of face landmarks + */ +- (void)didReceiveFaces:(NSArray *> *)faces; + +- (void)didSavedRegions:(NSArray *)foreheadURLs + leftcheekURLs:(NSArray *)leftcheekURLs + rightcheekURLs:(NSArray *)rightcheekURLs; +@end + +@interface FaceMesh : NSObject +- (instancetype)init; +- (void)startGraph; +- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer; +- (CVPixelBufferRef)resize:(CVPixelBufferRef)pixelBuffer + width:(int)width + height:(int)height; +- (uint8_t **)buffer2Array2D:(CVPixelBufferRef)pixelBuffer; +- (void)extractRegions:(NSURL *)fileName + foreheadBoxes:(NSArray *> *)foreheadBoxes + leftCheekBoxes:(NSArray *> *)leftCheekBoxes + rightCheekBoxes:(NSArray *> *)rightCheekBoxes + totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess + skipNFirstFrames:(NSInteger)skipNFirstFrames; +@property(weak, nonatomic) id delegate; +@property(nonatomic) size_t timestamp; +@end \ No newline at end of file diff --git a/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Headers/FaceMeshSDK.h b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Headers/FaceMeshSDK.h new file mode 100644 index 000000000..fb82b013d --- /dev/null +++ b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Headers/FaceMeshSDK.h @@ -0,0 +1,16 @@ +// +// FaceMeshSDK.h +// FaceMeshSDK +// + +#import + +//! Project version number for FaceMeshSDK. +FOUNDATION_EXPORT double FaceMeshSDKVersionNumber; + +//! Project version string for FaceMeshSDK. +FOUNDATION_EXPORT const unsigned char FaceMeshSDKVersionString[]; + +// In this header, you should import all the public headers of your framework using statements like #import + +#import "FaceMesh.h" diff --git a/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Info.plist b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Info.plist new file mode 100755 index 000000000..7a8f526d5 Binary files /dev/null and b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Info.plist differ diff --git a/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Modules/module.modulemap b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Modules/module.modulemap new file mode 100644 index 000000000..a9a2157ac --- /dev/null +++ b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/Modules/module.modulemap @@ -0,0 +1,21 @@ +framework module FaceMeshSDK { + umbrella header "FaceMeshSDK.h" + + export * + module * { export * } + + link framework "AVFoundation" + link framework "Accelerate" + link framework "AssetsLibrary" + link framework "CoreFoundation" + link framework "CoreGraphics" + link framework "CoreImage" + link framework "CoreMedia" + link framework "CoreVideo" + link framework "GLKit" + link framework "Metal" + link framework "MetalKit" + link framework "OpenGLES" + link framework "QuartzCore" + link framework "UIKit" +} diff --git a/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_detection_short_range.tflite b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_detection_short_range.tflite new file mode 100755 index 000000000..3b865a7dc Binary files /dev/null and b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_detection_short_range.tflite differ diff --git a/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_landmark.tflite b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_landmark.tflite new file mode 100755 index 000000000..277d26f2f Binary files /dev/null and b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_landmark.tflite differ diff --git a/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_landmark_with_attention.tflite b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_landmark_with_attention.tflite new file mode 100755 index 000000000..550ca8b06 Binary files /dev/null and b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_landmark_with_attention.tflite differ diff --git a/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_mesh_ios_lib_gpu.binarypb b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_mesh_ios_lib_gpu.binarypb new file mode 100755 index 000000000..e1f12ace9 --- /dev/null +++ b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/face_mesh_ios_lib_gpu.binarypb @@ -0,0 +1,5 @@ + +€ConstantSidePacketCalculator2PACKET:with_attentionBI +Atype.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions + +¬FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces*WITH_ATTENTION:with_attentionR input_videozmulti_face_landmarks‚ num_faces \ No newline at end of file diff --git a/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/pure_face_mesh_mobile_gpu.binarypb b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/pure_face_mesh_mobile_gpu.binarypb new file mode 100755 index 000000000..c59288f56 --- /dev/null +++ b/frameworkbuild/FaceMeshSDK/xcframework/FaceMeshSDK.xcframework/ios-arm64/FaceMeshSDK.framework/pure_face_mesh_mobile_gpu.binarypb @@ -0,0 +1,3 @@ + +¨FaceLandmarkFrontGpuIMAGE:input_video"LANDMARKS:multi_face_landmarks"-ROIS_FROM_LANDMARKS:face_rects_from_landmarks*NUM_FACES:num_faces +SPacketPresenceCalculatorPACKET:multi_face_landmarks"PRESENCE:landmark_presenceR input_videozmulti_face_landmarkszface_rects_from_landmarkszlandmark_presence‚ num_faces \ No newline at end of file diff --git a/mediapipe/.DS_Store b/mediapipe/.DS_Store new file mode 100644 index 000000000..771ba7645 Binary files /dev/null and b/mediapipe/.DS_Store differ diff --git a/mediapipe/examples/.DS_Store b/mediapipe/examples/.DS_Store new file mode 100644 index 000000000..51e74737f Binary files /dev/null and b/mediapipe/examples/.DS_Store differ diff --git a/mediapipe/examples/ios/.DS_Store b/mediapipe/examples/ios/.DS_Store new file mode 100644 index 000000000..1f28d20c1 Binary files /dev/null and b/mediapipe/examples/ios/.DS_Store differ diff --git a/mediapipe/examples/ios/facemeshioslib/.DS_Store b/mediapipe/examples/ios/facemeshioslib/.DS_Store new file mode 100644 index 000000000..5008ddfcf Binary files /dev/null and b/mediapipe/examples/ios/facemeshioslib/.DS_Store differ diff --git a/mediapipe/examples/ios/facemeshioslib/BUILD b/mediapipe/examples/ios/facemeshioslib/BUILD new file mode 100644 index 000000000..500d97afb --- /dev/null +++ b/mediapipe/examples/ios/facemeshioslib/BUILD @@ -0,0 +1,68 @@ +load( + "@build_bazel_rules_apple//apple:ios.bzl", + "ios_framework" +) +load( + "//mediapipe/examples/ios:bundle_id.bzl", + "BUNDLE_ID_PREFIX", + "example_provisioning", +) + +licenses(["notice"]) # Apache 2.0 + +MIN_IOS_VERSION = "11.1" + +IOS_FAMILIES = [ + "iphone", + "ipad", +] + +FRAMEWORK_HEADERS = [ + "FaceMesh.h", +] + +ios_framework( + name = "FaceMeshSDK", + hdrs = FRAMEWORK_HEADERS, + bundle_id = BUNDLE_ID_PREFIX + ".FaceMeshSDK", + bundle_name = "FaceMeshSDK", + families = IOS_FAMILIES, + infoplists = [ + "//mediapipe/examples/ios/common:Info.plist", +# "Info.plist", + ], + minimum_os_version = MIN_IOS_VERSION, + visibility = ["//visibility:public"], + deps = [ + ":FaceMeshObj", + "@ios_opencv//:OpencvFramework", + ], +) + + +objc_library( + name = "FaceMeshObj", + srcs = [ + "FaceMesh.mm", + ], + hdrs = FRAMEWORK_HEADERS, + copts = ["-std=c++17"], + data = [ + "//mediapipe/graphs/face_mesh:face_mesh_ios_lib_gpu.binarypb", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", + "//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite", + ], + deps = [ + "//mediapipe/objc:mediapipe_framework_ios", + "//mediapipe/objc:mediapipe_input_sources_ios", + "//mediapipe/calculators/core:packet_presence_calculator", + "//mediapipe/framework/port:opencv_video", + "//mediapipe/framework/port:opencv_imgcodecs", +# "//mediapipe/objc:mediapipe_layer_renderer", # no need for layer renderer since I don't render + ] + select({ + "//conditions:default": [ + "//mediapipe/graphs/face_mesh:mobile_calculators", + "//mediapipe/framework/formats:landmark_cc_proto", + ], + }), +) \ No newline at end of file diff --git a/mediapipe/examples/ios/facemeshioslib/FaceMesh.h b/mediapipe/examples/ios/facemeshioslib/FaceMesh.h new file mode 100644 index 000000000..097ac53de --- /dev/null +++ b/mediapipe/examples/ios/facemeshioslib/FaceMesh.h @@ -0,0 +1,58 @@ +#import +#import + +@interface IntPoint : NSObject +@property(nonatomic) NSInteger x; +@property(nonatomic) NSInteger y; + +- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y; +@end + +@interface NSValue (IntPoint) ++ (instancetype)valuewithIntPoint:(IntPoint *)value; +@property (readonly) IntPoint* intPointValue; +@end + +@interface FaceMeshLandmarkPoint : NSObject +@property(nonatomic) float x; +@property(nonatomic) float y; +@property(nonatomic) float z; +@end + +@interface FaceMeshNormalizedRect : NSObject +@property(nonatomic) float centerX; +@property(nonatomic) float centerY; +@property(nonatomic) float height; +@property(nonatomic) float width; +@property(nonatomic) float rotation; +@end + +@protocol FaceMeshDelegate +@optional +/** + * Array of faces, with faces represented by arrays of face landmarks + */ +- (void)didReceiveFaces:(NSArray *> *)faces; + +- (void)didSavedRegions:(NSArray *)foreheadURLs + leftcheekURLs:(NSArray *)leftcheekURLs + rightcheekURLs:(NSArray *)rightcheekURLs; +@end + +@interface FaceMesh : NSObject +- (instancetype)init; +- (void)startGraph; +- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer; +- (CVPixelBufferRef)resize:(CVPixelBufferRef)pixelBuffer + width:(int)width + height:(int)height; +- (uint8_t **)buffer2Array2D:(CVPixelBufferRef)pixelBuffer; +- (void)extractRegions:(NSURL *)fileName + foreheadBoxes:(NSArray *> *)foreheadBoxes + leftCheekBoxes:(NSArray *> *)leftCheekBoxes + rightCheekBoxes:(NSArray *> *)rightCheekBoxes + totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess + skipNFirstFrames:(NSInteger)skipNFirstFrames; +@property(weak, nonatomic) id delegate; +@property(nonatomic) size_t timestamp; +@end \ No newline at end of file diff --git a/mediapipe/examples/ios/facemeshioslib/FaceMesh.mm b/mediapipe/examples/ios/facemeshioslib/FaceMesh.mm new file mode 100644 index 000000000..61fa133d9 --- /dev/null +++ b/mediapipe/examples/ios/facemeshioslib/FaceMesh.mm @@ -0,0 +1,397 @@ +#import "FaceMesh.h" +#import "mediapipe/objc/MPPCameraInputSource.h" +#import "mediapipe/objc/MPPGraph.h" + +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/formats/detection.pb.h" + +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" +#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" +#include "opencv2/imgcodecs/ios.h" + +#import + +//#import "mediapipe/objc/MPPLayerRenderer.h" + +// The graph name specified is supposed to be the same as in the pb file (binarypb?) +static NSString* const kGraphName = @"face_mesh_ios_lib_gpu"; +// static NSString* const kGraphName = @"pure_face_mesh_mobile_gpu"; + +static const char* kInputStream = "input_video"; +static const char* kNumFacesInputSidePacket = "num_faces"; +static const char* kLandmarksOutputStream = "multi_face_landmarks"; + +// Max number of faces to detect/process. +static const int kNumFaces = 1; + + +@interface FaceMesh () +@property(nonatomic) MPPGraph* mediapipeGraph; +@end + +@implementation FaceMesh {} + +#pragma mark - Cleanup methods + +- (void)dealloc { + self.mediapipeGraph.delegate = nil; + [self.mediapipeGraph cancel]; + // Ignore errors since we're cleaning up. + [self.mediapipeGraph closeAllInputStreamsWithError:nil]; + [self.mediapipeGraph waitUntilDoneWithError:nil]; +} + +#pragma mark - MediaPipe graph methods + ++ (MPPGraph*)loadGraphFromResource:(NSString*)resource { + // Load the graph config resource. + NSError* configLoadError = nil; + NSBundle* bundle = [NSBundle bundleForClass:[self class]]; + if (!resource || resource.length == 0) { + return nil; + } + NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"]; + NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError]; + if (!data) { + NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError); + return nil; + } + + // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object. + mediapipe::CalculatorGraphConfig config; + config.ParseFromArray(data.bytes, data.length); + + // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object. + MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config]; + + // Set graph configurations + [newGraph setSidePacket:(mediapipe::MakePacket(kNumFaces)) + named:kNumFacesInputSidePacket]; + + [newGraph addFrameOutputStream:kLandmarksOutputStream + outputPacketType:MPPPacketTypeRaw]; + return newGraph; +} + +- (instancetype)init { + self = [super init]; + if (self) { + self.mediapipeGraph = [[self class] loadGraphFromResource:kGraphName]; + self.mediapipeGraph.delegate = self; + + // // Set maxFramesInFlight to a small value to avoid memory contention + // // for real-time processing. + // self.mediapipeGraph.maxFramesInFlight = 2; + NSLog(@"inited graph %@", kGraphName); + } + return self; +} + +- (void)startGraph { + NSError* error; + if (![self.mediapipeGraph startWithError:&error]) { + NSLog(@"Failed to start graph: %@", error); + } + NSLog(@"Started graph %@", kGraphName); +} + +#pragma mark - MPPGraphDelegate methods + +// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread. +- (void)mediapipeGraph:(MPPGraph*)graph + didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer + fromStream:(const std::string&)streamName { + NSLog(@"recv pixelBuffer from %@", @(streamName.c_str())); +} + +// Receives a raw packet from the MediaPipe graph. Invoked on a MediaPipe worker thread. +- (void)mediapipeGraph:(MPPGraph*)graph + didOutputPacket:(const ::mediapipe::Packet&)packet + fromStream:(const std::string&)streamName { + if (streamName == kLandmarksOutputStream) { + if (packet.IsEmpty()) { // This condition never gets called because FaceLandmarkFrontGpu does not process when there are no detections + return; + } + const auto& multi_face_landmarks = packet.Get>(); + // NSLog(@"[TS:%lld] Number of face instances with landmarks: %lu", packet.Timestamp().Value(), + // multi_face_landmarks.size()); + NSMutableArray *>*faceLandmarks = [NSMutableArray new]; + + for (int face_index = 0; face_index < multi_face_landmarks.size(); ++face_index) { + NSMutableArray *thisFaceLandmarks = [NSMutableArray new]; + const auto& landmarks = multi_face_landmarks[face_index]; +// NSLog(@"\tNumber of landmarks for face[%d]: %d", face_index, landmarks.landmark_size()); + for (int i = 0; i < landmarks.landmark_size(); ++i) { +// NSLog(@"\t\tLandmark[%d]: (%f, %f, %f)", i, landmarks.landmark(i).x(), +// landmarks.landmark(i).y(), landmarks.landmark(i).z()); + FaceMeshLandmarkPoint *obj_landmark = [FaceMeshLandmarkPoint new]; + obj_landmark.x = landmarks.landmark(i).x(); + obj_landmark.y = landmarks.landmark(i).y(); + obj_landmark.z = landmarks.landmark(i).z(); + [thisFaceLandmarks addObject:obj_landmark]; + } + [faceLandmarks addObject:thisFaceLandmarks]; + } + if([self.delegate respondsToSelector:@selector(didReceiveFaces:)]) { + [self.delegate didReceiveFaces:faceLandmarks]; + } + } else { + NSLog(@"Unknown %@ packet with stream name %s", packet.IsEmpty() ? @"EMPTY" : @"NON-EMPTY",streamName.c_str()); + } +} + + +#pragma mark - MPPInputSourceDelegate methods + +- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer { + const auto ts = + mediapipe::Timestamp(self.timestamp++ * mediapipe::Timestamp::kTimestampUnitsPerSecond); + NSError* err = nil; + // NSLog(@"sending imageBuffer @%@ to %s", @(ts.DebugString().c_str()), kInputStream); + auto sent = [self.mediapipeGraph sendPixelBuffer:imageBuffer + intoStream:kInputStream + packetType:MPPPacketTypePixelBuffer + timestamp:ts + allowOverwrite:NO + error:&err]; + // NSLog(@"imageBuffer %s", sent ? "sent!" : "not sent."); + if (err) { + NSLog(@"sendPixelBuffer error: %@", err); + } +} + +- (void)extractRegions:(NSURL *)fileName + foreheadBoxes:(NSArray *> *)foreheadBoxes + leftCheekBoxes:(NSArray *> *)leftCheekBoxes + rightCheekBoxes:(NSArray *> *)rightCheekBoxes + totalFramesNeedProcess:(NSInteger)totalFramesNeedProcess + skipNFirstFrames:(NSInteger)skipNFirstFrames { + + NSString *filename = fileName.path; + std::string filePath = [filename UTF8String]; + + cv::VideoCapture vid(filePath); + + if (!vid.isOpened()) { + printf("@Error Opening video file"); + } + else { + printf("File Opened AAAA"); + + // NSMutableArray *foreheadURLs = [NSMutableArray new]; + // NSMutableArray *leftcheekURLs = [NSMutableArray new]; + // NSMutableArray *rightcheekURLs = [NSMutableArray new]; + + int startFrame = int(vid.get(cv::CAP_PROP_POS_FRAMES)); + int totalFrame = int(vid.get(cv::CAP_PROP_FRAME_COUNT)); + // int nframes = totalFrame - startFrame; + + // if (totalFramesNeedProcess > nframes) { + // NSLog(@"Video too short"); + // return; + // } + + if (skipNFirstFrames < 0 || totalFramesNeedProcess < 0) { + vid.release(); + return; + } + + int frameIdx = skipNFirstFrames; + int maxFrameIndex = totalFramesNeedProcess; + + if (skipNFirstFrames > 0) { + maxFrameIndex += skipNFirstFrames; + } + + // Process forehead + std::vector foreheads; + std::vector leftcheeks; + std::vector rightcheeks; + + while (frameIdx < maxFrameIndex) { + cv::Mat frame; + + if (!vid.read(frame)) { + NSLog(@"Failed to read frame."); + break; + } + cv::cvtColor(frame, frame, cv::COLOR_BGR2RGB); + // Process the frame (e.g., display or save it) + + NSLog(@"frame index: %d", frameIdx); + + NSMutableArray *rowForehead = [foreheadBoxes objectAtIndex:(frameIdx - skipNFirstFrames)]; + NSMutableArray *rowLeftCheek = [leftCheekBoxes objectAtIndex:(frameIdx - skipNFirstFrames)]; + NSMutableArray *rowRightCheek = [rightCheekBoxes objectAtIndex:(frameIdx - skipNFirstFrames)]; + + cv::Mat forehead = extractRegion(frame, rowForehead); + cv::Mat leftCheek = extractRegion(frame, rowLeftCheek); + cv::Mat rightCheek = extractRegion(frame, rowRightCheek); + + foreheads.push_back(forehead); + leftcheeks.push_back(leftCheek); + rightcheeks.push_back(rightCheek); + + frameIdx++; + } + + NSLog(@"total foreheads: %d", foreheads.size()); + NSLog(@"total leftCheeks: %d", leftcheeks.size()); + NSLog(@"total rightCheeks: %d", rightcheeks.size()); + + // for (int i = 0; i < foreheads.size(); i++) { + // cv::Mat rgbaImage; + // cv::cvtColor(foreheads[i], rgbaImage, cv::COLOR_RGB2RGBA); + // saveCVImageAsPNG(MatToUIImage(rgbaImage), @"forehead"); + // } + + NSMutableArray *foreheadURLs = saveCVImagesAsPNGs(foreheads, @"forehead"); + NSMutableArray *leftcheekURLs = saveCVImagesAsPNGs(leftcheeks, @"leftcheek"); + NSMutableArray *rightcheekURLs = saveCVImagesAsPNGs(rightcheeks, @"rightcheek"); + + // cv::cvtColor(leftCheeks[0], rgbaImage, cv::COLOR_RGB2RGBA); + // NSData *firstData = [NSData dataWithBytes:rgbaImage.data length:rgbaImage.total() * rgbaImage.elemSize()]; + // saveCVImageAsPNG([UIImage imageWithData:firstData], @"leftcheek"); + + // cv::cvtColor(rightCheeks[0], rgbaImage, cv::COLOR_RGB2RGBA); + // NSData *firstData = [NSData dataWithBytes:rgbaImage.data length:rgbaImage.total() * rgbaImage.elemSize()]; + // saveCVImageAsPNG([UIImage imageWithData:firstData], @"rightcheek"); + + if([self.delegate respondsToSelector:@selector(didSavedRegions:leftcheekURLs:rightcheekURLs:)]) { + NSLog(@"nguyencse ==> has didSavedRegions"); + [self.delegate didSavedRegions:foreheadURLs leftcheekURLs:leftcheekURLs rightcheekURLs:rightcheekURLs]; + } + } + + vid.release(); +} + +cv::Mat extractRegion(cv::Mat img, NSArray *box) { + IntPoint* point0 = [box objectAtIndex:0]; + IntPoint* point1 = [box objectAtIndex:1]; + IntPoint* point2 = [box objectAtIndex:2]; + IntPoint* point3 = [box objectAtIndex:3]; + + // LEFT TOP --> RIGHT TOP --> RIGHT BOTTOM --> LEFT BOTTOM + int frameWidth = point1.x - point0.x; + int frameHeight = point3.y - point0.y; + cv::Mat region = cropROI(img, cv::Rect(point0.x, point0.y, frameWidth, frameHeight)); + + // square + region = square(region); + + // resize to 32x32 + region = resizeWithAreaInterpolation(region, cv::Size(32, 32)); + + return region; +} + +cv::Mat cropROI(cv::Mat src, cv::Rect roi) { + // Crop the full image to that image contained by the rectangle myROI + // Note that this doesn't copy the data + cv::Mat croppedRef(src, roi); + + cv::Mat cropped; + // Copy the data into new matrix + croppedRef.copyTo(cropped); + + return cropped; +} + +cv::Mat square(cv::Mat frame) { + if (frame.rows < frame.cols) { + int diff = frame.cols - frame.rows; + cv::Mat pad(frame.rows, diff, frame.type(), cv::Scalar(0)); + cv::hconcat(frame, pad, frame); + } else if (frame.rows > frame.cols) { + int diff = frame.rows - frame.cols; + cv::Mat pad(diff, frame.cols, frame.type(), cv::Scalar(0)); + cv::vconcat(frame, pad, frame); + } + + return frame; +} + +cv::Mat resizeWithAreaInterpolation(cv::Mat image, cv::Size newShape) { + cv::Size originalShape = image.size(); + cv::resize(image, image, newShape, 0, 0, cv::INTER_AREA); + return image; +} + +// NSURL *saveCVImageAsPNG(UIImage *image, NSString *fileName) { +// // Create a unique file name for each image +// NSString *fileNameIndex = [fileName stringByAppendingFormat:@"_%d", 0]; +// NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:[fileNameIndex stringByAppendingString:@".png"]]; +// NSURL *fileURL = [NSURL fileURLWithPath:filePath]; + +// NSData *pngData = UIImagePNGRepresentation(image); + +// if ([pngData writeToFile:filePath atomically:YES]) { +// NSLog(@"PNG file saved successfully at path: %@", filePath); +// } else { +// NSLog(@"Failed to save PNG file at path: %@", filePath); +// } + +// return fileURL; +// } + +NSArray *saveCVImagesAsPNGs(std::vector frames, NSString *folderName) { + NSMutableArray *fileURLs = [NSMutableArray arrayWithCapacity:frames.size()]; + + for (int i = 0; i < frames.size(); i++) { + // Create a unique file name for each image + NSString *fileNameIndex = [folderName stringByAppendingFormat:@"_%d", i]; + NSString *filePath = [NSTemporaryDirectory() stringByAppendingPathComponent:[fileNameIndex stringByAppendingString:@".png"]]; + NSURL *fileURL = [NSURL fileURLWithPath:filePath]; + + // NSLog(@"File URL: %@", fileURL); + + // Do NOT compress pixel + std::vector compressionParams; + compressionParams.push_back(cv::IMWRITE_PNG_COMPRESSION); + compressionParams.push_back(0); + + const char * cpath = [filePath cStringUsingEncoding:NSUTF8StringEncoding]; + const cv::String newPath = (const cv::String)cpath; + cv::imwrite(newPath, frames[i], compressionParams); + + // Add file URL to the array + [fileURLs addObject:fileURL]; + } + + return [fileURLs copy]; +} + +@end + + +@implementation NSValue (IntPoint) ++ (instancetype)valuewithIntPoint:(IntPoint *)value { + return [self valueWithBytes:&value objCType:@encode(IntPoint)]; +} +- (IntPoint *) intPointValue { + IntPoint* value; + [self getValue:&value]; + return value; +} +@end + +@implementation FaceMeshLandmarkPoint +@end + +@implementation FaceMeshNormalizedRect +@end + +@implementation IntPoint +- (instancetype)initWithX:(NSInteger)x y:(NSInteger)y { + self = [super init]; + if (self) { + _x = x; + _y = y; + } + return self; +} +@end \ No newline at end of file diff --git a/mediapipe/examples/ios/facemeshioslib/patch_ios_framework.sh b/mediapipe/examples/ios/facemeshioslib/patch_ios_framework.sh new file mode 100644 index 000000000..931198533 --- /dev/null +++ b/mediapipe/examples/ios/facemeshioslib/patch_ios_framework.sh @@ -0,0 +1,60 @@ +#!/bin/bash +set -eu +set -o pipefail + + +[[ $# -lt 2 ]] && echo "Usage: $0 ..." && exit 1 +zipped=$(python3 -c "import os; print(os.path.realpath('$1'))"); shift +name=$(basename "$zipped" .zip) +parent=$(dirname "$zipped") +named="$parent"/"$name".framework + +unzip "$zipped" -d "$parent" + +mkdir "$named"/Modules +cat << EOF >"$named"/Modules/module.modulemap +framework module $name { + umbrella header "$name.h" + + export * + module * { export * } + + link framework "AVFoundation" + link framework "Accelerate" + link framework "AssetsLibrary" + link framework "CoreFoundation" + link framework "CoreGraphics" + link framework "CoreImage" + link framework "CoreMedia" + link framework "CoreVideo" + link framework "GLKit" + link framework "Metal" + link framework "MetalKit" + link framework "OpenGLES" + link framework "QuartzCore" + link framework "UIKit" +} +EOF +# NOTE: All these linked frameworks are required by mediapipe/objc. + +cat << EOF >"$named"/Headers/$name.h +// +// $name.h +// $name +// + +#import + +//! Project version number for $name. +FOUNDATION_EXPORT double ${name}VersionNumber; + +//! Project version string for $name. +FOUNDATION_EXPORT const unsigned char ${name}VersionString[]; + +// In this header, you should import all the public headers of your framework using statements like #import <$name/PublicHeader.h> + +EOF +until [[ $# -eq 0 ]]; do + printf '#import "'"$1"'"\n' "$1" >>"$named"/Headers/$name.h + shift +done \ No newline at end of file diff --git a/mediapipe/graphs/.DS_Store b/mediapipe/graphs/.DS_Store new file mode 100644 index 000000000..7df346bc0 Binary files /dev/null and b/mediapipe/graphs/.DS_Store differ diff --git a/mediapipe/graphs/face_mesh/.DS_Store b/mediapipe/graphs/face_mesh/.DS_Store new file mode 100644 index 000000000..5008ddfcf Binary files /dev/null and b/mediapipe/graphs/face_mesh/.DS_Store differ diff --git a/mediapipe/graphs/face_mesh/BUILD b/mediapipe/graphs/face_mesh/BUILD index 6926fda72..a786882cc 100644 --- a/mediapipe/graphs/face_mesh/BUILD +++ b/mediapipe/graphs/face_mesh/BUILD @@ -67,3 +67,17 @@ mediapipe_binary_graph( output_name = "face_mesh_mobile_gpu.binarypb", deps = [":mobile_calculators"], ) + +mediapipe_binary_graph( + name = "face_mesh_ios_lib_gpu_binary_graph", + graph = "face_mesh_ios_lib.pbtxt", + output_name = "face_mesh_ios_lib_gpu.binarypb", + deps = [":mobile_calculators"], +) + +mediapipe_binary_graph( + name = "pure_face_mesh_mobile_gpu_binary_graph", + graph = "pure_face_mesh_mobile.pbtxt", + output_name = "pure_face_mesh_mobile_gpu.binarypb", + deps = [":mobile_calculators"], +) diff --git a/mediapipe/graphs/face_mesh/face_mesh_ios_lib.pbtxt b/mediapipe/graphs/face_mesh/face_mesh_ios_lib.pbtxt new file mode 100644 index 000000000..f05ccc200 --- /dev/null +++ b/mediapipe/graphs/face_mesh/face_mesh_ios_lib.pbtxt @@ -0,0 +1,70 @@ +# MediaPipe graph that performs face mesh with TensorFlow Lite on GPU. + +# GPU buffer. (GpuBuffer) +input_stream: "input_video" + +# Max number of faces to detect/process. (int) +input_side_packet: "num_faces" + +# Output image with rendered results. (GpuBuffer) +# nope no rendering +# output_stream: "output_video" +# Collection of detected/processed faces, each represented as a list of +# landmarks. (std::vector) +output_stream: "multi_face_landmarks" + + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for downstream nodes +# (calculators and subgraphs) in the graph to finish their tasks before it +# passes through another image. All images that come in while waiting are +# dropped, limiting the number of in-flight images in most part of the graph to +# 1. This prevents the downstream nodes from queuing up incoming images and data +# excessively, which leads to increased latency and memory usage, unwanted in +# real-time mobile applications. It also eliminates unnecessarily computation, +# e.g., the output produced by a node may get dropped downstream if the +# subsequent nodes are still busy processing previous inputs. +# node { +# calculator: "FlowLimiterCalculator" +# input_stream: "input_video" +# input_stream: "FINISHED:output_video" +# input_stream_info: { +# tag_index: "FINISHED" +# back_edge: true +# } +# output_stream: "throttled_input_video" +#} + +# Defines side packets for further use in the graph. +node { + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:with_attention" + node_options: { + [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { + packet { bool_value: true } + } + } +} + +# Subgraph that detects faces and corresponding landmarks. +node { + calculator: "FaceLandmarkFrontGpu" + # input_stream: "IMAGE:throttled_input_video" + input_stream: "IMAGE:input_video" + input_side_packet: "NUM_FACES:num_faces" + input_side_packet: "WITH_ATTENTION:with_attention" + output_stream: "LANDMARKS:multi_face_landmarks" + output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" + # output_stream: "DETECTIONS:face_detections" + # output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" +} + +# Subgraph that renders face-landmark annotation onto the input image. +# node { +# calculator: "FaceRendererGpu" +# input_stream: "IMAGE:throttled_input_video" +# input_stream: "LANDMARKS:multi_face_landmarks" +# input_stream: "NORM_RECTS:face_rects_from_landmarks" +# input_stream: "DETECTIONS:face_detections" +# output_stream: "IMAGE:output_video" +#} diff --git a/mediapipe/graphs/face_mesh/pure_face_mesh_mobile.pbtxt b/mediapipe/graphs/face_mesh/pure_face_mesh_mobile.pbtxt new file mode 100644 index 000000000..8367ecbe5 --- /dev/null +++ b/mediapipe/graphs/face_mesh/pure_face_mesh_mobile.pbtxt @@ -0,0 +1,74 @@ +# MediaPipe graph that performs face mesh with TensorFlow Lite on GPU. +# Edited from face_mesh_mobile.pbtxt because I don't want babysitting of auto throttling and such. I'll do it myself + +# GPU buffer. (GpuBuffer) +input_stream: "input_video" + +# Max number of faces to detect/process. (int) +input_side_packet: "num_faces" + +# Output image with rendered results. (GpuBuffer) +# nope no rendering +# output_stream: "output_video" + +# Collection of detected/processed faces, each represented as a list of +# landmarks. (std::vector) +output_stream: "multi_face_landmarks" + +# Regions of interest calculated based on landmarks. +# (For more info see mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt) +# (std::vector) +# For typings see "mediapipe/framework/formats/rect.pb.h" +output_stream: "face_rects_from_landmarks" + +# The detections from the box model +# see detection.proto +# Regions of interest calculated based on face detections. +# (std::vector) +# output_stream: "face_rects_from_detections" + +# Extra outputs (for debugging, for instance). +# Detected faces. (std::vector) +# (std::vector) +# output_stream: "face_detections" + +# Landmark presence (needed because whole graph won't emit anything if no faces are detected) +output_stream: "landmark_presence" + +# screw the throttling, we do that ourselves. +# *throttling node code was deleted from here* + +# Subgraph that detects faces and corresponding landmarks. +node { + calculator: "FaceLandmarkFrontGpu" +# the IMAGE: part is saying, pipe this data into the input with the name `image` + input_stream: "IMAGE:input_video" + input_side_packet: "NUM_FACES:num_faces" + output_stream: "LANDMARKS:multi_face_landmarks" + output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" + # face_detections is the stream that comes out from face_detection_short_range_common + # output_stream: "DETECTIONS:face_detections" + + # output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" +} + +# See this thread here https://github.com/google/mediapipe/issues/850#issuecomment-683268033 +# "if there are no packets in the corresponding output stream, it is designed to wait until the packet comes in" +# That means that we'd get absolutely nothing to work with and won't know if our frame had anythin! +# So we add PacketPresenceCalculator +node { + calculator: "PacketPresenceCalculator" + input_stream: "PACKET:multi_face_landmarks" + output_stream: "PRESENCE:landmark_presence" +} + +# nope not rendering. +# Subgraph that renders face-landmark annotation onto the input image. +# node { +# calculator: "FaceRendererGpu" +# input_stream: "IMAGE:throttled_input_video" +# input_stream: "LANDMARKS:multi_face_landmarks" +# input_stream: "NORM_RECTS:face_rects_from_landmarks" +# input_stream: "DETECTIONS:face_detections" +# output_stream: "IMAGE:output_video" +#} diff --git a/third_party/.DS_Store b/third_party/.DS_Store new file mode 100644 index 000000000..3d06b225a Binary files /dev/null and b/third_party/.DS_Store differ