code optimizitation

mediapipe/calculators/beauty/BUILD

@@ -34,11 +34,8 @@ cc_library(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgproc",
-        "//mediapipe/framework/port:opencv_highgui",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
-        "//mediapipe/util:annotation_renderer",
-        "//mediapipe/util:render_data_cc_proto",
     ],
     alwayslink = 1,
 )
@@ -87,11 +84,8 @@ cc_library(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgproc",
-        "//mediapipe/framework/port:opencv_highgui",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
-        "//mediapipe/util:annotation_renderer",
-        "//mediapipe/util:render_data_cc_proto",
     ],
     alwayslink = 1,
 )
@@ -112,11 +106,8 @@ cc_library(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgproc",
-        "//mediapipe/framework/port:opencv_highgui",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
-        "//mediapipe/util:annotation_renderer",
-        "//mediapipe/util:render_data_cc_proto",
     ],
     alwayslink = 1,
 )

mediapipe/calculators/beauty/draw_lipstick_calculator.cc

@@ -18,7 +18,6 @@
 #include <cmath>
 #include <map>
 #include <string>
-//#include <android/log.h>
 
 #include <memory>
 
@@ -33,17 +32,13 @@
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
 #include "mediapipe/framework/port/status.h"
-#include "mediapipe/util/annotation_renderer.h"
-#include "mediapipe/util/render_data.pb.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/vector.h"
-#include "mediapipe/util/color.pb.h"
 
 namespace mediapipe
 {
   namespace
   {
-
     constexpr char kMaskTag[] = "MASK";
     constexpr char kImageFrameTag[] = "IMAGE";
 
@@ -54,12 +49,7 @@ namespace mediapipe
       NUM_ATTRIBUTES
     };
 
-    // Round up n to next multiple of m.
-    size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT
     inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
-
-    using Point = RenderAnnotation::Point;
-
   } // namespace
 
   class DrawLipstickCalculator : public CalculatorBase
@@ -91,10 +81,6 @@ namespace mediapipe
     // Indicates if image frame is available as input.
     bool image_frame_available_ = false;
     std::unordered_map<std::string, cv::Mat> all_masks;
-    int width_ = 0;
-    int height_ = 0;
-    int width_canvas_ = 0; // Size of overlay drawing texture canvas.
-    int height_canvas_ = 0;
   };
   REGISTER_CALCULATOR(DrawLipstickCalculator);
 
@@ -141,9 +127,6 @@ namespace mediapipe
     {
       image_frame_available_ = true;
     }
-    else
-    {
-    }
 
     // Set the output header based on the input header (if present).
     const char *tag = kImageFrameTag;
@@ -204,7 +187,6 @@ namespace mediapipe
       CalculatorContext *cc, const ImageFormat::Format &target_format,
       uchar *data_image, std::unique_ptr<cv::Mat> &image_mat)
   {
-
     cv::Mat mat_image_ = *image_mat.get();
 
     auto output_frame = absl::make_unique<ImageFrame>(

mediapipe/calculators/beauty/smooth_face_calculator.cc

@@ -17,7 +17,6 @@
 #include <algorithm>
 #include <cmath>
 #include <iostream>
-//#include <android/log.h>
 
 #include <memory>
 
@@ -32,11 +31,8 @@
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
 #include "mediapipe/framework/port/status.h"
-#include "mediapipe/util/annotation_renderer.h"
-#include "mediapipe/util/render_data.pb.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/vector.h"
-#include "mediapipe/util/color.pb.h"
 
 namespace mediapipe
 {
@@ -54,12 +50,7 @@ namespace mediapipe
       NUM_ATTRIBUTES
     };
 
-    // Round up n to next multiple of m.
-    size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT
     inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
-
-    using Point = RenderAnnotation::Point;
-
   } // namespace
 
   class SmoothFaceCalculator : public CalculatorBase
@@ -82,23 +73,22 @@ namespace mediapipe
 
     absl::Status RenderToCpu(
         CalculatorContext *cc, const ImageFormat::Format &target_format,
-        uchar *data_image, std::unique_ptr<cv::Mat> &image_mat);
+        uchar *data_image);
 
-    absl::Status SmoothFace(CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
+    absl::Status SmoothFace(CalculatorContext *cc,
                             ImageFormat::Format *target_format,
                             const std::unordered_map<std::string, cv::Mat> &mask_vec,
                             const std::tuple<double, double, double, double> &face_box);
 
-    cv::Mat predict_forehead_mask(std::unique_ptr<cv::Mat> &image_mat,
+    cv::Mat predict_forehead_mask(const std::unordered_map<std::string, cv::Mat> &mask_vec, double face_box_min_y);
-                                  const std::unordered_map<std::string, cv::Mat> &mask_vec, double face_box_min_y);
 
     // Indicates if image frame is available as input.
     bool image_frame_available_ = false;
-    std::unordered_map<std::string, cv::Mat> all_masks;
+
-    int width_ = 0;
+    int image_width_;
-    int height_ = 0;
+    int image_height_;
-    int width_canvas_ = 0; // Size of overlay drawing texture canvas.
+    cv::Mat mat_image_;
-    int height_canvas_ = 0;
+    std::unique_ptr<cv::Mat> image_mat;
   };
   REGISTER_CALCULATOR(SmoothFaceCalculator);
 
@@ -186,7 +176,6 @@ namespace mediapipe
     }
 
     // Initialize render target, drawn with OpenCV.
-    std::unique_ptr<cv::Mat> image_mat;
     ImageFormat::Format target_format;
 
     if (cc->Outputs().HasTag(kImageFrameTag))
@@ -194,6 +183,10 @@ namespace mediapipe
       MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
     }
 
+    mat_image_ = *image_mat.get();
+    image_width_ = image_mat->cols;
+    image_height_ = image_mat->rows;
+
     const std::vector<std::unordered_map<std::string, cv::Mat>> &mask_vec =
         cc->Inputs().Tag(kMaskTag).Get<std::vector<std::unordered_map<std::string, cv::Mat>>>();
 
@@ -203,12 +196,12 @@ namespace mediapipe
     if (mask_vec.size() > 0 && face_box.size() > 0)
     {
       for (int i = 0; i < mask_vec.size(); i++)
-        MP_RETURN_IF_ERROR(SmoothFace(cc, image_mat, &target_format, mask_vec[i], face_box[i]));
+        MP_RETURN_IF_ERROR(SmoothFace(cc, &target_format, mask_vec[i], face_box[i]));
     }
 
     // Copy the rendered image to output.
     uchar *image_mat_ptr = image_mat->data;
-    MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat));
+    MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr));
 
     return absl::OkStatus();
   }
@@ -220,15 +213,12 @@ namespace mediapipe
 
   absl::Status SmoothFaceCalculator::RenderToCpu(
       CalculatorContext *cc, const ImageFormat::Format &target_format,
-      uchar *data_image, std::unique_ptr<cv::Mat> &image_mat)
+      uchar *data_image)
   {
-
-    cv::Mat mat_image__ = *image_mat.get();
-
     auto output_frame = absl::make_unique<ImageFrame>(
-        target_format, mat_image__.cols, mat_image__.rows);
+        target_format, image_width_, image_height_);
 
-    output_frame->CopyPixelData(target_format, mat_image__.cols, mat_image__.rows, data_image,
+    output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image,
                                 ImageFrame::kDefaultAlignmentBoundary);
 
     if (cc->Outputs().HasTag(kImageFrameTag))
@@ -290,29 +280,23 @@ namespace mediapipe
     {
       image_mat = absl::make_unique<cv::Mat>(
           150, 150, CV_8UC4,
-          cv::Scalar(255, 255,
+          cv::Scalar::all(255));
-                     255));
       *target_format = ImageFormat::SRGBA;
     }
 
     return absl::OkStatus();
   }
 
-  cv::Mat SmoothFaceCalculator::predict_forehead_mask(std::unique_ptr<cv::Mat> &image_mat,
+  cv::Mat SmoothFaceCalculator::predict_forehead_mask(const std::unordered_map<std::string, cv::Mat> &mask_vec, double face_box_min_y)
-                                                      const std::unordered_map<std::string, cv::Mat> &mask_vec, double face_box_min_y)
   {
 
-    cv::Mat mat_image__ = *image_mat.get();
-    int image_width_ = image_mat->cols;
-    int image_height_ = image_mat->rows;
-
     cv::Mat part_forehead_mask = mask_vec.find("PART_FOREHEAD_B")->second.clone();
     part_forehead_mask.convertTo(part_forehead_mask, CV_32F, 1.0 / 255);
     part_forehead_mask.convertTo(part_forehead_mask, CV_8U);
 
     cv::Mat image_sm, image_sm_hsv, skinMask;
 
-    cv::resize(mat_image__, image_sm, cv::Size(mat_image__.size().width, mat_image__.size().height));
+    cv::resize(mat_image_, image_sm, cv::Size(image_width_, image_height_));
     cv::cvtColor(image_sm, image_sm_hsv, cv::COLOR_BGR2HSV);
 
     std::vector<int> x, y;
@@ -375,20 +359,14 @@ namespace mediapipe
   }
 
   absl::Status SmoothFaceCalculator::SmoothFace(CalculatorContext *cc,
-                                                std::unique_ptr<cv::Mat> &image_mat,
                                                 ImageFormat::Format *target_format,
                                                 const std::unordered_map<std::string, cv::Mat> &mask_vec,
                                                 const std::tuple<double, double, double, double> &face_box)
   {
-
-    cv::Mat mat_image__ = *image_mat.get();
-
     cv::Mat mouth_mask, mouth;
-    int image_width_ = image_mat->cols;
-    int image_height_ = image_mat->rows;
 
     cv::Mat not_full_face = mask_vec.find("FACE_OVAL")->second.clone() +
-                            predict_forehead_mask(image_mat, mask_vec, std::get<1>(face_box)) -
+                            predict_forehead_mask(mask_vec, std::get<1>(face_box)) -
                             mask_vec.find("LEFT_EYE")->second.clone() -
                             mask_vec.find("RIGHT_EYE")->second.clone() -
                             mask_vec.find("LEFT_BROW")->second.clone() -
@@ -397,7 +375,7 @@ namespace mediapipe
 
     cv::resize(not_full_face,
                not_full_face,
-               mat_image__.size(), 0, 0,
+               mat_image_.size(), 0, 0,
                cv::INTER_LINEAR);
 
     std::vector<int> x, y;
@@ -416,7 +394,7 @@ namespace mediapipe
     cv::minMaxLoc(x, &min_x, &max_x);
     cv::minMaxLoc(y, &min_y, &max_y);
 
-    cv::Mat patch_face = mat_image__(cv::Range(min_y, max_y), cv::Range(min_x, max_x));
+    cv::Mat patch_face = mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x));
     cv::Mat patch_nff = not_full_face(cv::Range(min_y, max_y), cv::Range(min_x, max_x));
     cv::Mat patch_new, patch_wow;
     cv::cvtColor(patch_face, patch_wow, cv::COLOR_RGBA2RGB);

mediapipe/calculators/beauty/whiten_teeth_calculator.cc

@@ -31,11 +31,8 @@
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
 #include "mediapipe/framework/port/status.h"
-#include "mediapipe/util/annotation_renderer.h"
-#include "mediapipe/util/render_data.pb.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/vector.h"
-#include "mediapipe/util/color.pb.h"
 
 namespace mediapipe
 {
@@ -52,12 +49,7 @@ namespace mediapipe
       NUM_ATTRIBUTES
     };
 
-    // Round up n to next multiple of m.
-    size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT
     inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
-
-    using Point = RenderAnnotation::Point;
-
   } // namespace
 
   class WhitenTeethCalculator : public CalculatorBase
@@ -82,17 +74,15 @@ namespace mediapipe
         CalculatorContext *cc, const ImageFormat::Format &target_format,
         uchar *data_image, std::unique_ptr<cv::Mat> &image_mat);
 
-    absl::Status WhitenTeeth(CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
+    absl::Status WhitenTeeth(CalculatorContext *cc, ImageFormat::Format *target_format,
-                             ImageFormat::Format *target_format,
                              const std::unordered_map<std::string, cv::Mat> &mask_vec);
 
     // Indicates if image frame is available as input.
     bool image_frame_available_ = false;
-    std::unordered_map<std::string, cv::Mat> all_masks;
+    std::unique_ptr<cv::Mat> image_mat;
-    int width_ = 0;
+    cv::Mat mat_image_;
-    int height_ = 0;
+    int image_width_;
-    int width_canvas_ = 0; // Size of overlay drawing texture canvas.
+    int image_height_;
-    int height_canvas_ = 0;
   };
   REGISTER_CALCULATOR(WhitenTeethCalculator);
 
@@ -170,20 +160,22 @@ namespace mediapipe
     }
 
     // Initialize render target, drawn with OpenCV.
-    std::unique_ptr<cv::Mat> image_mat;
     ImageFormat::Format target_format;
 
     if (cc->Outputs().HasTag(kImageFrameTag))
     {
       MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
     }
+    mat_image_ = *image_mat.get();
+    image_width_ = image_mat->cols;
+    image_height_ = image_mat->rows;
     
     const std::vector<std::unordered_map<std::string, cv::Mat>> &mask_vec =
         cc->Inputs().Tag(kMaskTag).Get<std::vector<std::unordered_map<std::string, cv::Mat>>>();
     if (mask_vec.size() > 0)
     {
       for (auto mask : mask_vec)
-        MP_RETURN_IF_ERROR(WhitenTeeth(cc, image_mat, &target_format, mask));
+        MP_RETURN_IF_ERROR(WhitenTeeth(cc, &target_format, mask));
     }
 
     // Copy the rendered image to output.
@@ -206,9 +198,9 @@ namespace mediapipe
     cv::Mat mat_image_ = *image_mat.get();
 
     auto output_frame = absl::make_unique<ImageFrame>(
-        target_format, mat_image_.cols, mat_image_.rows);
+        target_format, image_width_, image_height_);
 
-    output_frame->CopyPixelData(target_format, mat_image_.cols, mat_image_.rows, data_image,
+    output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image,
                                 ImageFrame::kDefaultAlignmentBoundary);
 
     if (cc->Outputs().HasTag(kImageFrameTag))
@@ -279,20 +271,15 @@ namespace mediapipe
   }
 
   absl::Status WhitenTeethCalculator::WhitenTeeth(CalculatorContext *cc,
-                                                  std::unique_ptr<cv::Mat> &image_mat,
                                                   ImageFormat::Format *target_format,
                                                   const std::unordered_map<std::string, cv::Mat> &mask_vec)
   {
-    cv::Mat mat_image__ = *image_mat.get();
-
     cv::Mat mouth_mask, mouth;
-    int image_width_ = image_mat->cols;
+    mouth_mask = cv::Mat::zeros(mat_image_.size(), CV_32F);
-    int image_height_ = image_mat->rows;
-    mouth_mask = cv::Mat::zeros(mat_image__.size(), CV_32F);
 
     mouth_mask = mask_vec.find("MOUTH_INSIDE")->second.clone();
 
-    cv::resize(mouth_mask, mouth, mat_image__.size(), cv::INTER_LINEAR);
+    cv::resize(mouth_mask, mouth, mat_image_.size(), cv::INTER_LINEAR);
 
     std::vector<int> x, y;
     std::vector<cv::Point> location;
@@ -323,7 +310,7 @@ namespace mediapipe
         mouth_max_x = static_cast<int>(std::min(mouth_max_x + mw * 0.1, (double)image_width_));
         mouth_crop_mask = mouth(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x));
         cv::Mat img_hsv, tmp_mask, img_hls;
-        cv::cvtColor(mat_image__(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)), img_hsv,
+        cv::cvtColor(mat_image_(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)), img_hsv,
                      cv::COLOR_RGBA2RGB);
         cv::cvtColor(img_hsv, img_hsv,
                      cv::COLOR_RGB2HSV);
@@ -357,7 +344,7 @@ namespace mediapipe
         cv::cvtColor(img_hls, img_hls, cv::COLOR_HLS2RGB);
         cv::cvtColor(img_hls, img_hls, cv::COLOR_RGB2RGBA);
 
-        cv::Mat slice = mat_image__(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x));
+        cv::Mat slice = mat_image_(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x));
         img_hls.copyTo(slice);
       }
     }

mediapipe/calculators/landmarks/BUILD

@@ -28,9 +28,6 @@ cc_library(
         "//mediapipe/framework:calculator_options_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:location_data_cc_proto",
-        "//mediapipe/framework/port:ret_check",
-        "//mediapipe/util:color_cc_proto",
-        "//mediapipe/util:render_data_cc_proto",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "//mediapipe/framework/formats:image_format_cc_proto",

mediapipe/calculators/landmarks/landmarks_to_mask_calculator.cc

@@ -17,28 +17,9 @@
 
 #include <algorithm>
 #include <cmath>
-#include <string>
-#include <map>
 #include <iostream>
 
-#include <memory>
-
 #include "absl/memory/memory.h"
-#include "absl/strings/str_cat.h"
-#include "absl/strings/str_join.h"
-#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/calculator_options.pb.h"
-#include "mediapipe/framework/formats/image_format.pb.h"
-#include "mediapipe/framework/formats/image_frame.h"
-#include "mediapipe/framework/formats/image_frame_opencv.h"
-#include "mediapipe/framework/formats/landmark.pb.h"
-#include "mediapipe/framework/port/opencv_core_inc.h"
-#include "mediapipe/framework/port/opencv_imgproc_inc.h"
-#include "mediapipe/framework/formats/location_data.pb.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/util/color.pb.h"
-#include "mediapipe/util/render_data.pb.h"
-#include "absl/strings/str_cat.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/vector.h"
@@ -49,7 +30,6 @@ namespace mediapipe
   {
     constexpr char kLandmarksTag[] = "LANDMARKS";
     constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS";
-    constexpr char kLandmarkLabel[] = "KEYPOINT";
     constexpr char kVectorTag[] = "VECTOR";
     constexpr char kMaskTag[] = "MASK";
     constexpr char kFaceBoxTag[] = "FACEBOX";
@@ -111,6 +91,11 @@ namespace mediapipe
 
     std::tuple<double, double, double, double> face_box;
 
+    std::unique_ptr<cv::Mat> image_mat;
+
+    int image_width_;
+    int image_height_;
+
     float scale_factor_ = 1.0;
 
     bool image_frame_available_ = false;
@@ -161,9 +146,6 @@ namespace mediapipe
     {
       image_frame_available_ = true;
     }
-    else
-    {
-    }
 
     return absl::OkStatus();
   }
@@ -190,15 +172,18 @@ namespace mediapipe
 
     // Initialize render target, drawn with OpenCV.
 
-    std::unique_ptr<cv::Mat> image_mat;
     ImageFormat::Format target_format;
     std::unordered_map<std::string, cv::Mat> all_masks;
 
     MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
 
-    MP_RETURN_IF_ERROR(GetMasks(cc, all_masks, image_mat));
+    cv::Mat mat_image_ = *image_mat.get();
+    image_width_ = image_mat->cols;
+    image_height_ = image_mat->rows;
 
-    MP_RETURN_IF_ERROR(GetFaceBox(cc, image_mat));
+    MP_RETURN_IF_ERROR(GetMasks(cc, all_masks));
+
+    MP_RETURN_IF_ERROR(GetFaceBox(cc));
 
     MP_RETURN_IF_ERROR(RenderToCpu(cc, all_masks));
 
@@ -289,12 +274,8 @@ namespace mediapipe
   }
 
   absl::Status LandmarksToMaskCalculator::GetMasks(CalculatorContext *cc,
-                                                   std::unordered_map<std::string, cv::Mat> &all_masks, std::unique_ptr<cv::Mat> &image_mat)
+                                                   std::unordered_map<std::string, cv::Mat> &all_masks)
   {
-
-    int image_width_ = image_mat->cols;
-    int image_height_ = image_mat->rows;
-
     if (cc->Inputs().HasTag(kLandmarksTag))
     {
        const LandmarkList &landmarks =
@@ -326,10 +307,11 @@ namespace mediapipe
 
         std::vector<std::vector<cv::Point>> point_vec;
         point_vec.push_back(point_array);
+       
         mask = cv::Mat::zeros(image_mat->size(), CV_32FC1);
         cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA);
         mask.convertTo(mask, CV_8U);
-        all_masks.insert(make_pair(key, mask));
+        all_masks.insert({key, mask});
         point_vec.clear();
         point_array.clear();
       }
@@ -377,13 +359,8 @@ namespace mediapipe
     return absl::OkStatus();
   }
 
-  absl::Status LandmarksToMaskCalculator::GetFaceBox(CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat)
+  absl::Status LandmarksToMaskCalculator::GetFaceBox(CalculatorContext *cc)
   {
-    cv::Mat mat_image_ = *image_mat.get();
-
-    int image_width_ = image_mat->cols;
-    int image_height_ = image_mat->rows;
-
     std::vector<int> x_s, y_s;
     double box_min_y, box_max_y, box_max_x, box_min_x;
     if (cc->Inputs().HasTag(kLandmarksTag))

mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h

@@ -14,46 +14,33 @@
 #ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_TO_MASK_CALCULATOR_H_
 #define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_TO_MASK_CALCULATOR_H_
 
-#include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_options.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/location_data.pb.h"
-#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/util/color.pb.h"
-#include "mediapipe/util/render_data.pb.h"
 #include "mediapipe/framework/formats/image_format.pb.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
-#include "mediapipe/framework/port/logging.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/framework/port/vector.h"
 
 namespace mediapipe
 {
 
-    // A calculator that converts Landmark proto to RenderData proto for
+    // A calculator that uses face landmarks to create face part masks and facebox for
-    // visualization. The input should be LandmarkList proto. It is also possible
+    // visualization. The input should be LandmarkList proto and ImageFrame. 
-    // to specify the connections between landmarks.
     //
     // Example config:
     // node {
     //   calculator: "LandmarksToMaskCalculator"
-    //   input_stream: "NORM_LANDMARKS:landmarks"
+    //   input_stream: "IMAGE:image"
-    //   output_stream: "RENDER_DATA:render_data"
+    //   input_stream: "NORM_LANDMARKS:face_landmarks"
-    //   options {
+    //   output_stream: "FACEBOX:face_box"
-    //     [LandmarksToRenderDataCalculatorOptions.ext] {
+    //   output_stream: "MASK:mask"
-    //       landmark_connections: [0, 1, 1, 2]
-    //       landmark_color { r: 0 g: 255 b: 0 }
-    //       connection_color { r: 0 g: 255 b: 0 }
-    //       thickness: 4.0
-    //     }
-    //   }
     // }
+
     class LandmarksToMaskCalculator : public CalculatorBase
     {
     public:
@@ -74,9 +61,9 @@ namespace mediapipe
         absl::Status RenderToCpu(CalculatorContext *cc,
                                  std::unordered_map<std::string, cv::Mat> &all_masks);
 
-        absl::Status GetFaceBox(CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat);
+        absl::Status GetFaceBox(CalculatorContext *cc);
 
-        absl::Status GetMasks(CalculatorContext *cc, std::unordered_map<std::string, cv::Mat> &all_masks, std::unique_ptr<cv::Mat> &image_mat);
+        absl::Status GetMasks(CalculatorContext *cc, std::unordered_map<std::string, cv::Mat> &all_masks);
 
         absl::Status CreateRenderTargetCpu(
             CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,

mediapipe/examples/desktop/beauty/BUILD

@@ -17,7 +17,7 @@ licenses(["notice"])
 package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 
 cc_binary(
-    name = "face_mesh_tflite",
+    name = "beauty_tflite",
     deps = [
         "//mediapipe/examples/desktop:simple_run_graph_main",
         "//mediapipe/graphs/beauty:desktop_calculators",
@@ -25,9 +25,25 @@ cc_binary(
 )
 
 cc_binary(
-    name = "face_mesh_cpu",
+    name = "beauty_cpu",
     deps = [
         "//mediapipe/examples/desktop:demo_run_graph_main",
         "//mediapipe/graphs/beauty:desktop_live_calculators",
     ],
 )
+
+cc_binary(
+    name = "beauty_cpu_over",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/beauty:desktop_live_over_calculators",
+    ],
+)
+
+cc_binary(
+    name = "beauty_gpu_over",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/beauty:desktop_gpu_over",
+    ],
+)

mediapipe/graphs/beauty/beauty_desktop.pbtxt

@@ -0,0 +1,50 @@
+# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU.
+
+# Input image. (ImageFrame)
+input_stream: "input_video"
+
+# Output image with rendered results. (ImageFrame)
+output_stream: "output_video"
+# Collection of detected/processed faces, each represented as a list of
+# landmarks. (std::vector<NormalizedLandmarkList>)
+output_stream: "multi_face_landmarks"
+
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:output_video"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+node {
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:0:num_faces"
+  output_side_packet: "PACKET:1:with_attention"
+  node_options: {
+    [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
+      packet { int_value: 1 }
+      packet { bool_value: true }
+    }
+  }
+}
+
+# Subgraph that detects faces and corresponding landmarks.
+node {
+  calculator: "FaceLandmarkFrontCpu"
+  input_stream: "IMAGE:throttled_input_video"
+  input_side_packet: "NUM_FACES:num_faces"
+  input_side_packet: "WITH_ATTENTION:with_attention"
+  output_stream: "LANDMARKS:multi_face_landmarks"
+}
+
+# Subgraph that renders onto the input image.
+node {
+  calculator: "FaceRendererCpu"
+  input_stream: "IMAGE:throttled_input_video"
+  input_stream: "LANDMARKS:multi_face_landmarks"
+  output_stream: "IMAGE:output_video"
+}

mediapipe/graphs/beauty/subgraphs/BUILD

@@ -27,15 +27,12 @@ cc_library(
         "//mediapipe/calculators/core:split_proto_list_calculator",
         "//mediapipe/util:annotation_renderer",
         "//mediapipe/calculators/util:annotation_overlay_calculator",
-        "//mediapipe/calculators/beauty:form_face_mask_calculator",
         "//mediapipe/calculators/beauty:smooth_face_calculator",
         "//mediapipe/calculators/beauty:draw_lipstick_calculator",
         "//mediapipe/calculators/beauty:whiten_teeth_calculator",
         "//mediapipe/calculators/util:detections_to_render_data_calculator",
         "//mediapipe/calculators/util:landmarks_to_render_data_calculator",
         "//mediapipe/calculators/landmarks:landmarks_to_mask_calculator",
-        "//mediapipe/calculators/util:rect_to_render_data_calculator",
-        "//mediapipe/graphs/beauty/calculators:face_landmarks_to_render_data_calculator",
     ],
 )
 

mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt

@@ -30,7 +30,7 @@ node {
   output_stream: "BATCH_END:landmark_timestamp"
 }
 
-# Converts landmarks to drawing primitives for annotation overlay.
+# Converts landmarks to face part masks.
 node {
   calculator: "LandmarksToMaskCalculator"
   input_stream: "IMAGE:loop_image"
@@ -39,7 +39,7 @@ node {
   output_stream: "MASK:mask"
 }
 
-# Collects a RenderData object for each hand into a vector. Upon receiving the
+# Collects a MapMask object for each hand into a vector. Upon receiving the
 # BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END
 # timestamp.
 node {
@@ -56,7 +56,7 @@ node {
   output_stream: "ITERABLE:multi_face_box"
 }
 
-
+#Applies lipstick to the face on the IMAGE using MASK.
 node {
   calculator: "DrawLipstickCalculator"
   input_stream: "IMAGE:input_image"
@@ -64,6 +64,7 @@ node {
   output_stream: "IMAGE:input_image_1"
 }
 
+#Whitens teeth of the face on the IMAGE using MASK.
 node {
   calculator: "WhitenTeethCalculator"
   input_stream: "IMAGE:input_image_1"
@@ -71,6 +72,7 @@ node {
   output_stream: "IMAGE:input_image_2"
 }
 
+#Smoothes face on the IMAGE using MASK.
 node {
   calculator: "SmoothFaceCalculator"
   input_stream: "IMAGE:input_image_2"

mediapipe/graphs/beauty/subgraphs/face_renderer_cpu_old.pbtxt

@@ -53,57 +53,13 @@ node {
   output_stream: "ITERABLE:multi_face_landmarks_render_data"
 }
 
-# Converts normalized rects to drawing primitives for annotation overlay.
+# Draws annotations and overlays them on top of the input images.
-#node {
-#  calculator: "RectToRenderDataCalculator"
-# input_stream: "NORM_RECTS:rects"
-#  output_stream: "RENDER_DATA:rects_render_data"
-#  node_options: {
-#    [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
-#      filled: false
-#      color { r: 255 g: 0 b: 0 }
-#      thickness: 4.0
-#    }
-#  }
-#}
-
 node {
-  calculator: "FormFaceMaskCalculator"
+  calculator: "AnnotationOverlayCalculator"
   input_stream: "IMAGE:input_image"
   input_stream: "VECTOR:0:multi_face_landmarks_render_data"
-  output_stream: "FACEBOX:face_box"
-  output_stream: "MASK:multi_mask"
-}
-
-node {
-  calculator: "DrawLipstickCalculator"
-  input_stream: "IMAGE:input_image"
-  input_stream: "MASK:0:multi_mask"
-  output_stream: "IMAGE:input_image_1"
-}
-
-node {
-  calculator: "WhitenTeethCalculator"
-  input_stream: "IMAGE:input_image_1"
-  input_stream: "MASK:0:multi_mask"
-  output_stream: "IMAGE:input_image_2"
-}
-
-node {
-  calculator: "SmoothFaceCalculator"
-  input_stream: "IMAGE:input_image_2"
-  input_stream: "MASK:0:multi_mask"
-  input_stream: "FACEBOX:face_box"
   output_stream: "IMAGE:output_image"
 }
 
-# Draws annotations and overlays them on top of the input images.
-#node {
-#  calculator: "AnnotationOverlayCalculator"
-#  input_stream: "IMAGE:input_image"
-#  input_stream: "VECTOR:0:multi_face_landmarks_render_data"
-#  output_stream: "IMAGE:output_image"
-#}
-