Merge pull request #3 from HFVladimir/AS-beautifier

As beautifier

.bazelversion

@@ -1 +1 @@
-5.0.0
+5.2.0

WORKSPACE

@@ -205,13 +205,13 @@ new_local_repository(
     # For local MacOS builds, the path should point to an opencv@3 installation.
     # If you edit the path here, you will also need to update the corresponding
     # prefix in "opencv_macos.BUILD".
-    path = "/usr/local",
+    path = "/opt/homebrew/Cellar/",
 )
 
 new_local_repository(
     name = "macos_ffmpeg",
     build_file = "@//third_party:ffmpeg_macos.BUILD",
-    path = "/usr/local/opt/ffmpeg",
+    path = "/opt/homebrew/Cellar/ffmpeg",
 )
 
 new_local_repository(

mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen

@@ -2,6 +2,7 @@
   "additionalFilePaths" : [
     "/BUILD",
     "mediapipe/BUILD",
+    "mediapipe/examples/ios/beauty/BUILD",
     "mediapipe/examples/ios/common/BUILD",
     "mediapipe/examples/ios/facedetectioncpu/BUILD",
     "mediapipe/examples/ios/facedetectiongpu/BUILD",
@@ -23,6 +24,7 @@
     "mediapipe/objc/testing/app/BUILD"
   ],
   "buildTargets" : [
+    "//mediapipe/examples/ios/beauty:BeautyApp",
     "//mediapipe/examples/ios/facedetectioncpu:FaceDetectionCpuApp",
     "//mediapipe/examples/ios/facedetectiongpu:FaceDetectionGpuApp",
     "//mediapipe/examples/ios/faceeffect:FaceEffectApp",
@@ -93,6 +95,7 @@
     "mediapipe/examples/ios",
     "mediapipe/examples/ios/common",
     "mediapipe/examples/ios/common/Base.lproj",
+    "mediapipe/examples/ios/beauty",
     "mediapipe/examples/ios/facedetectioncpu",
     "mediapipe/examples/ios/facedetectiongpu",
     "mediapipe/examples/ios/faceeffect",

mediapipe/MediaPipe.tulsiproj/project.tulsiconf

@@ -3,6 +3,9 @@
     "optionSet" : {
       "CLANG_CXX_LANGUAGE_STANDARD" : {
         "p" : "c++14"
+      },
+      "EnvironmentVariables" : {
+        "p" : "MEDIAPIPE_PROFILING=1"
       }
     }
   },
@@ -10,6 +13,7 @@
     "",
     "mediapipe",
     "mediapipe/examples/ios",
+    "mediapipe/examples/ios/beauty",
     "mediapipe/examples/ios/facedetectioncpu",
     "mediapipe/examples/ios/facedetectiongpu",
     "mediapipe/examples/ios/faceeffect",

mediapipe/calculators/beauty/smooth_face_calculator.cc

@@ -284,7 +284,6 @@ namespace mediapipe
 
   cv::Mat SmoothFaceCalculator::predict_forehead_mask(const std::unordered_map<std::string, cv::Mat> &mask_vec, double face_box_min_y)
   {
-
     cv::Mat part_forehead_mask = mask_vec.find("PART_FOREHEAD_B")->second.clone();
     part_forehead_mask.convertTo(part_forehead_mask, CV_32F, 1.0 / 255);
     part_forehead_mask.convertTo(part_forehead_mask, CV_8U);
@@ -360,8 +359,8 @@ namespace mediapipe
   {
     cv::Mat mouth_mask, mouth;
 
-    cv::Mat not_full_face = mask_vec.find("FACE_OVAL")->second.clone() +
-                            predict_forehead_mask(mask_vec, std::get<1>(face_box)) -
+    cv::Mat not_full_face = mask_vec.find("FACE_OVAL")->second.clone() -
+//                            predict_forehead_mask(mask_vec, std::get<1>(face_box)) -
                             mask_vec.find("LEFT_EYE")->second.clone() -
                             mask_vec.find("RIGHT_EYE")->second.clone() -
                             mask_vec.find("LEFT_BROW")->second.clone() -
@@ -394,6 +393,7 @@ namespace mediapipe
     cv::Mat patch_new, patch_wow;
     cv::cvtColor(patch_face, patch_wow, cv::COLOR_RGBA2RGB);
     cv::bilateralFilter(patch_wow, patch_new, 12, 50, 50);
+    //patch_wow.copyTo(patch_new);
 
     cv::Mat patch_new_nff, patch_new_mask, patch, patch_face_nff;
 

mediapipe/examples/desktop/beauty/BUILD

@@ -32,6 +32,14 @@ cc_binary(
     ],
 )
 
+cc_binary(
+    name = "beauty_mobile",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/beauty:mobile_calculators",
+    ],
+)
+
 cc_binary(
     name = "beauty_cpu_single",
     deps = [

mediapipe/graphs/beauty/BUILD

@@ -58,6 +58,7 @@ cc_library(
     	"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
         "//mediapipe/calculators/core:flow_limiter_calculator",
         "//mediapipe/graphs/beauty/subgraphs:face_renderer_cpu",
+        "//mediapipe/graphs/beauty/subgraphs:face_renderer_gpu",
         "//mediapipe/modules/face_landmark:face_landmark_front_gpu",
     ],
 )

mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt

@@ -10,6 +10,25 @@ output_stream: "output_video"
 output_stream: "multi_face_landmarks"
 
 
+profiler_config {
+  trace_enabled: true
+  enable_profiler: true
+  trace_log_interval_count: 200
+  trace_log_path: "/Users/alena/Workdir/mediapipe/logs/beauty/"
+}
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+
+
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"

mediapipe/graphs/beauty/beauty_desktop_cpu_single.pbtxt

@@ -10,6 +10,14 @@ output_stream: "output_video"
 output_stream: "multi_face_landmarks"
 
 
+profiler_config {
+  trace_enabled: true
+  enable_profiler: true
+  trace_log_interval_count: 200
+  trace_log_path: "/Users/alena/Workdir/mediapipe/logs/beauty/"
+}
+
+
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"

mediapipe/graphs/beauty/beauty_mobile_gpu.pbtxt

@@ -12,6 +12,12 @@ output_stream: "output_video"
 # landmarks. (std::vector<NormalizedLandmarkList>)
 output_stream: "multi_face_landmarks"
 
+profiler_config {
+  trace_enabled: true
+  enable_profiler: true
+  trace_log_interval_count: 200
+}
+
 # Throttles the images flowing downstream for flow control. It passes through
 # the very first incoming image unaltered, and waits for downstream nodes
 # (calculators and subgraphs) in the graph to finish their tasks before it
@@ -58,6 +64,9 @@ node {
   input_side_packet: "NUM_FACES:num_faces"
   input_side_packet: "WITH_ATTENTION:with_attention"
   output_stream: "LANDMARKS:multi_face_landmarks"
+  output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
+  output_stream: "DETECTIONS:face_detections"
+  output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
 }
 
 # Subgraph that renders face-landmark annotation onto the input image.
@@ -68,7 +77,6 @@ node {
   output_stream: "IMAGE:output_video_cpu"
 }
 
-# Defines side packets for further use in the graph.
 node {
   calculator: "ImageFrameToGpuBufferCalculator"
   input_stream: "output_video_cpu"

mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt

@@ -74,6 +74,3 @@ node {
   input_stream: "FACEBOX:0:multi_face_box"
   output_stream: "IMAGE:output_image"
 }
-
-
-

mediapipe/graphs/beauty/subgraphs/face_renderer_gpu.pbtxt

@@ -0,0 +1,86 @@
+# MediaPipe face mesh rendering subgraph.
+
+type: "FaceRendererGpu"
+
+# GPU image. (GpuBuffer)
+input_stream: "IMAGE:input_image"
+# Collection of detected/predicted faces, each represented as a list of
+# landmarks. (std::vector<NormalizedLandmarkList>)
+input_stream: "LANDMARKS:multi_face_landmarks"
+# Regions of interest calculated based on palm detections.
+# (std::vector<NormalizedRect>)
+input_stream: "NORM_RECTS:rects"
+# Detected palms. (std::vector<Detection>)
+input_stream: "DETECTIONS:detections"
+
+# GPU image with rendered data. (GpuBuffer)
+output_stream: "IMAGE:output_image"
+
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:input_image"
+  output_stream: "SIZE:image_size"
+}
+# Outputs each element of multi_face_landmarks at a fake timestamp for the rest
+# of the graph to process. At the end of the loop, outputs the BATCH_END
+# timestamp for downstream calculators to inform them that all elements in the
+# vector have been processed.
+node {
+  calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
+  input_stream: "ITERABLE:multi_face_landmarks"
+  input_stream: "IMAGE_GPU:input_image"
+  output_stream: "ITEM:face_landmarks"
+  output_stream: "IMAGE_GPU:loop_image"
+  output_stream: "BATCH_END:landmark_timestamp"
+}
+
+# Converts landmarks to face part masks.
+node {
+  calculator: "LandmarksToMaskCalculator"
+  input_stream: "IMAGE_GPU:loop_image"
+  input_stream: "NORM_LANDMARKS:face_landmarks"
+  output_stream: "FACEBOX:face_box"
+  output_stream: "MASK:mask"
+}
+
+# Collects a MapMask object for each hand into a vector. Upon receiving the
+# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END
+# timestamp.
+node {
+  calculator: "EndLoopMapMaskCalculator"
+  input_stream: "ITEM:mask"
+  input_stream: "BATCH_END:landmark_timestamp"
+  output_stream: "ITERABLE:multi_mask"
+}
+
+node {
+  calculator: "EndLoopFaceBoxCalculator"
+  input_stream: "ITEM:face_box"
+  input_stream: "BATCH_END:landmark_timestamp"
+  output_stream: "ITERABLE:multi_face_box"
+}
+
+#Applies lipstick to the face on the IMAGE using MASK.
+node {
+  calculator: "DrawLipstickCalculator"
+  input_stream: "IMAGE_GPU:input_image"
+  input_stream: "MASK:0:multi_mask"
+  output_stream: "IMAGE_GPU:input_image_1"
+}
+
+#Whitens teeth of the face on the IMAGE using MASK.
+node {
+  calculator: "WhitenTeethCalculator"
+  input_stream: "IMAGE_GPU:input_image_1"
+  input_stream: "MASK:0:multi_mask"
+  output_stream: "IMAGE_GPU:input_image_2"
+}
+
+#Smoothes face on the IMAGE using MASK.
+node {
+  calculator: "SmoothFaceCalculator"
+  input_stream: "IMAGE_GPU:input_image_2"
+  input_stream: "MASK:0:multi_mask"
+  input_stream: "FACEBOX:0:multi_face_box"
+  output_stream: "IMAGE_GPU:output_image"
+}

mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt

@@ -9,6 +9,13 @@ output_stream: "output_video"
 # landmarks. (std::vector<NormalizedLandmarkList>)
 output_stream: "multi_face_landmarks"
 
+profiler_config {
+  trace_enabled: true
+  enable_profiler: true
+  trace_log_interval_count: 200
+  trace_log_path: "/Users/alena/Workdir/mediapipe/logs"
+}
+
 # Throttles the images flowing downstream for flow control. It passes through
 # the very first incoming image unaltered, and waits for downstream nodes
 # (calculators and subgraphs) in the graph to finish their tasks before it

mediapipe/graphs/face_mesh/face_mesh_mobile.pbtxt

@@ -12,6 +12,12 @@ output_stream: "output_video"
 # landmarks. (std::vector<NormalizedLandmarkList>)
 output_stream: "multi_face_landmarks"
 
+profiler_config {
+  trace_enabled: true
+  enable_profiler: true
+  trace_log_interval_count: 200
+}
+
 # Throttles the images flowing downstream for flow control. It passes through
 # the very first incoming image unaltered, and waits for downstream nodes
 # (calculators and subgraphs) in the graph to finish their tasks before it

mediapipe/graphs/face_mesh/face_mesh_mobile_gpu.pbtxt

@@ -12,6 +12,12 @@ output_stream: "output_video"
 # landmarks. (std::vector<NormalizedLandmarkList>)
 output_stream: "multi_face_landmarks"
 
+profiler_config {
+  trace_enabled: true
+  enable_profiler: true
+  trace_log_interval_count: 200
+}
+
 # Throttles the images flowing downstream for flow control. It passes through
 # the very first incoming image unaltered, and waits for downstream nodes
 # (calculators and subgraphs) in the graph to finish their tasks before it

mediapipe/util/annotation_renderer.cc

@@ -108,7 +108,7 @@ void AnnotationRenderer::RenderDataOnImage(const RenderData &render_data)
   if (render_data.render_annotations().size()){
     DrawLipstick(render_data);
     WhitenTeeth(render_data);
-//    SmoothFace(render_data);
+    //SmoothFace(render_data);
   }
   else
   {
@@ -170,6 +170,7 @@ cv::Mat AnnotationRenderer::FormFacePartMask(std::vector<int> orderList, const R
   }
 
   if (points_array.size() != orderList.size()){
+    mask.convertTo(mask, CV_8U);
     return mask;
   }
 
@@ -290,7 +291,6 @@ cv::Mat AnnotationRenderer::predict_forehead_mask(const RenderData &render_data,
 
 void AnnotationRenderer::SmoothFace(const RenderData &render_data)
 {
-
   cv::Mat not_full_face = cv::Mat(FormFacePartMask(FACE_OVAL, render_data)) +
                           cv::Mat(predict_forehead_mask(render_data, std::get<1>(GetFaceBox(render_data)))) -
                           cv::Mat(FormFacePartMask(LEFT_EYE, render_data)) -
@@ -324,7 +324,9 @@ void AnnotationRenderer::SmoothFace(const RenderData &render_data)
   cv::Mat patch_nff = not_full_face(cv::Range(min_y, max_y), cv::Range(min_x, max_x));
   cv::Mat patch_new, patch_wow;
   cv::cvtColor(patch_face, patch_wow, cv::COLOR_RGBA2RGB);
-  cv::bilateralFilter(patch_wow, patch_new, 12, 50, 50);
+  if (patch_wow.data != patch_new.data) {
+    cv::bilateralFilter(patch_wow, patch_new, 12, 50, 50);
+  }
 
   cv::Mat patch_new_nff, patch_new_mask, patch, patch_face_nff;
 

third_party/opencv_linux.BUILD

@@ -1,41 +1,33 @@
 # Description:
-#   OpenCV libraries for video/image processing on Linux
+#   OpenCV libraries for video/image processing on MacOS
 
 licenses(["notice"])  # BSD license
 
 exports_files(["LICENSE"])
 
-# The following build rule assumes that OpenCV is installed by
-# 'apt-get install libopencv-core-dev libopencv-highgui-dev \'
-# '                libopencv-calib3d-dev libopencv-features2d-dev \'
-# '                libopencv-imgproc-dev libopencv-video-dev'
-# on Debian Buster/Ubuntu 18.04.
-# If you install OpenCV separately, please modify the build rule accordingly.
+load("@bazel_skylib//lib:paths.bzl", "paths")
+
+# The path to OpenCV is a combination of the path set for "macos_opencv"
+# in the WORKSPACE file and the prefix here.
+PREFIX = "opencv@3/3.4.16_3/"
+
 cc_library(
     name = "opencv",
-    hdrs = glob([
-        # For OpenCV 4.x
-        #"include/aarch64-linux-gnu/opencv4/opencv2/cvconfig.h",
-        #"include/arm-linux-gnueabihf/opencv4/opencv2/cvconfig.h",
-        #"include/x86_64-linux-gnu/opencv4/opencv2/cvconfig.h",
-        #"include/opencv4/opencv2/**/*.h*",
-    ]),
-    includes = [
-        # For OpenCV 4.x
-        #"include/aarch64-linux-gnu/opencv4/",
-        #"include/arm-linux-gnueabihf/opencv4/",
-        #"include/x86_64-linux-gnu/opencv4/",
-        #"include/opencv4/",
-    ],
-    linkopts = [
-        "-l:libopencv_core.so",
-        "-l:libopencv_calib3d.so",
-        "-l:libopencv_features2d.so",
-        "-l:libopencv_highgui.so",
-        "-l:libopencv_imgcodecs.so",
-        "-l:libopencv_imgproc.so",
-        "-l:libopencv_video.so",
-        "-l:libopencv_videoio.so",
-    ],
+    srcs = glob(
+        [
+            paths.join(PREFIX, "lib/libopencv_core.dylib"),
+            paths.join(PREFIX, "lib/libopencv_calib3d.dylib"),
+            paths.join(PREFIX, "lib/libopencv_features2d.dylib"),
+            paths.join(PREFIX, "lib/libopencv_highgui.dylib"),
+            paths.join(PREFIX, "lib/libopencv_imgcodecs.dylib"),
+            paths.join(PREFIX, "lib/libopencv_imgproc.dylib"),
+            paths.join(PREFIX, "lib/libopencv_video.dylib"),
+            paths.join(PREFIX, "lib/libopencv_videoio.dylib"),
+        ],
+    ),
+    hdrs = glob([paths.join(PREFIX, "include/opencv2/**/*.h*")]),
+    includes = [paths.join(PREFIX, "include/")],
+    linkstatic = 1,
     visibility = ["//visibility:public"],
 )
+

third_party/opencv_macos.BUILD

@@ -9,7 +9,7 @@ load("@bazel_skylib//lib:paths.bzl", "paths")
 
 # The path to OpenCV is a combination of the path set for "macos_opencv"
 # in the WORKSPACE file and the prefix here.
-PREFIX = "opt/opencv@3"
+PREFIX = "opencv@3/3.4.16_3/"
 
 cc_library(
     name = "opencv",