Makeup alpha

mediapipe/calculators/beauty/BUILD

@@ -22,29 +22,6 @@ cc_library(
     name = "draw_lipstick_calculator",
     srcs = ["draw_lipstick_calculator.cc"],
     visibility = ["//visibility:public"],
-    deps = [
-        "//mediapipe/framework:calculator_options_cc_proto",
-        "//mediapipe/framework/formats:image_format_cc_proto",
-        "//mediapipe/util:color_cc_proto",
-        "@com_google_absl//absl/strings",
-        "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/formats:image_frame",
-        "//mediapipe/framework/formats:image_frame_opencv",
-        "//mediapipe/framework/formats:video_stream_header",
-        "//mediapipe/framework/port:logging",
-        "//mediapipe/framework/port:opencv_core",
-        "//mediapipe/framework/port:opencv_imgproc",
-        "//mediapipe/framework/port:status",
-        "//mediapipe/framework/port:vector",
-    ],
-    alwayslink = 1,
-)
-
-
-cc_library(
-    name = "form_face_mask_calculator",
-    srcs = ["form_face_mask_calculator.cc"],
-    visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_options_cc_proto",
         "//mediapipe/framework/formats:image_format_cc_proto",
@@ -60,14 +37,10 @@ cc_library(
         "//mediapipe/framework/port:opencv_highgui",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
-        "//mediapipe/util:annotation_renderer",
-        "//mediapipe/util:render_data_cc_proto",
     ],
     alwayslink = 1,
 )
 
-
-
 cc_library(
     name = "smooth_face_calculator1",
     srcs = ["smooth_face1_calculator.cc"],
@@ -84,6 +57,7 @@ cc_library(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:opencv_highgui",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
     ],
@@ -106,6 +80,7 @@ cc_library(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgproc",
+                "//mediapipe/framework/port:opencv_highgui",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
     ],
@@ -128,6 +103,7 @@ cc_library(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgproc",
+          "//mediapipe/framework/port:opencv_highgui",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
     ],
@@ -150,14 +126,49 @@ cc_library(
         "//mediapipe/framework/port:logging",
         "//mediapipe/framework/port:opencv_core",
         "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:opencv_highgui",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:vector",
     ],
     alwayslink = 1,
 )
 
-
+cc_library(
-
+    name = "merging_images_calculator",
-
+    srcs = ["merging_images_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_options_cc_proto",
+        "//mediapipe/framework/formats:image_format_cc_proto",
+        "@com_google_absl//absl/strings",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:opencv_highgui",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:vector",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "imageframe_to_mat_calculator",
+    srcs = ["imageframe_to_mat_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+   "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:opencv_highgui",
+    ],
+    alwayslink = 1,
+)
 
 

mediapipe/calculators/beauty/bilateral_filter_calculator.cc

@@ -12,14 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <math.h>
-
-#include <algorithm>
-#include <cmath>
-#include <iostream>
-
-#include <memory>
-
 #include "absl/strings/str_cat.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_options.pb.h"
@@ -30,26 +22,18 @@
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/vector.h"
 
+using namespace std;
 namespace mediapipe
 {
   namespace
   {
-
-    constexpr char kMaskTag[] = "MASK";
-    constexpr char kFaceBoxTag[] = "FACEBOX";
     constexpr char kImageFrameTag[] = "IMAGE";
-    constexpr char kImageNewTag[] = "IMAGE2";
+    constexpr char kOutTag[] = "CVMAT";
-
-    enum
-    {
-      ATTRIB_VERTEX,
-      ATTRIB_TEXTURE_POSITION,
-      NUM_ATTRIBUTES
-    };
 
     inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
   } // namespace
@@ -68,24 +52,18 @@ namespace mediapipe
     absl::Status Close(CalculatorContext *cc) override;
 
   private:
-    absl::Status CreateRenderTargetCpu(CalculatorContext *cc,
-                                       std::unique_ptr<cv::Mat> &image_mat,
-                                       ImageFormat::Format *target_format);
-
-    absl::Status RenderToCpu(
-        CalculatorContext *cc, const ImageFormat::Format &target_format,
-        uchar *data_image);
-
     absl::Status BilateralFilter(CalculatorContext *cc,
                                  const std::vector<double> &face_box);
 
+    absl::Status RenderToCpu(CalculatorContext *cc);
+
     // Indicates if image frame is available as input.
     bool image_frame_available_ = false;
 
     int image_width_;
     int image_height_;
     cv::Mat mat_image_;
-    std::unique_ptr<cv::Mat> image_mat;
+    cv::Mat out_mat;
   };
   REGISTER_CALCULATOR(BilateralCalculator);
 
@@ -95,30 +73,13 @@ namespace mediapipe
 
     if (cc->Inputs().HasTag(kImageFrameTag))
     {
-      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+      cc->Inputs().Tag(kImageFrameTag).Set<std::pair<cv::Mat, std::vector<double>>>();
-      CHECK(cc->Outputs().HasTag(kImageNewTag));
+      CHECK(cc->Outputs().HasTag(kOutTag));
     }
 
-    // Data streams to render.
+    if (cc->Outputs().HasTag(kOutTag))
-    for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId();
-         ++id)
     {
-      auto tag_and_index = cc->Inputs().TagAndIndexFromId(id);
+      cc->Outputs().Tag(kOutTag).Set<cv::Mat>();
-      std::string tag = tag_and_index.first;
-      if (tag == kFaceBoxTag)
-      {
-        cc->Inputs().Get(id).Set<std::vector<double>>();
-      }
-      else if (tag.empty())
-      {
-        // Empty tag defaults to accepting a single object of Mat type.
-        cc->Inputs().Get(id).Set<cv::Mat>();
-      }
-    }
-
-    if (cc->Outputs().HasTag(kImageNewTag))
-    {
-      cc->Outputs().Tag(kImageNewTag).Set<ImageFrame>();
     }
 
     return absl::OkStatus();
@@ -135,7 +96,7 @@ namespace mediapipe
 
     // Set the output header based on the input header (if present).
     const char *tag = kImageFrameTag;
-    const char *out_tag = kImageNewTag;
+    const char *out_tag = kOutTag;
     if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty())
     {
       const auto &input_header =
@@ -154,32 +115,21 @@ namespace mediapipe
     {
       return absl::OkStatus();
     }
-    if (cc->Inputs().HasTag(kFaceBoxTag) &&
-        cc->Inputs().Tag(kFaceBoxTag).IsEmpty())
-    {
-      return absl::OkStatus();
-    }
 
+    const std::pair<cv::Mat, std::vector<double>> &face =
+        cc->Inputs().Tag(kImageFrameTag).Get<std::pair<cv::Mat, std::vector<double>>>();
     // Initialize render target, drawn with OpenCV.
     ImageFormat::Format target_format;
+    const std::vector<double> &face_box = face.second;
+    mat_image_ = face.first.clone();
 
-    if (cc->Outputs().HasTag(kImageNewTag))
+    image_width_ = mat_image_.cols;
-    {
+    image_height_ = mat_image_.rows;
-      MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
-    }
-
-    mat_image_ = *image_mat.get();
-    image_width_ = image_mat->cols;
-    image_height_ = image_mat->rows;
-
-    const std::vector<double> &face_box =
-        cc->Inputs().Tag(kFaceBoxTag).Get<std::vector<double>>();
 
+    if (!face_box.empty())
       MP_RETURN_IF_ERROR(BilateralFilter(cc, face_box));
 
-    // Copy the rendered image to output.
+    MP_RETURN_IF_ERROR(RenderToCpu(cc));
-    uchar *image_mat_ptr = image_mat->data;
-    MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr));
 
     return absl::OkStatus();
   }
@@ -189,92 +139,28 @@ namespace mediapipe
     return absl::OkStatus();
   }
 
-  absl::Status BilateralCalculator::RenderToCpu(
-      CalculatorContext *cc, const ImageFormat::Format &target_format,
-      uchar *data_image)
-  {
-    auto output_frame = absl::make_unique<ImageFrame>(
-        target_format, image_width_, image_height_);
-
-    output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image,
-                                ImageFrame::kDefaultAlignmentBoundary);
-
-    if (cc->Outputs().HasTag(kImageNewTag))
-    {
-      cc->Outputs()
-          .Tag(kImageNewTag)
-          .Add(output_frame.release(), cc->InputTimestamp());
-    }
-
-    return absl::OkStatus();
-  }
-
-  absl::Status BilateralCalculator::CreateRenderTargetCpu(
-      CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
-      ImageFormat::Format *target_format)
-  {
-    if (image_frame_available_)
-    {
-      const auto &input_frame =
-          cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-
-      int target_mat_type;
-      switch (input_frame.Format())
-      {
-      case ImageFormat::SRGBA:
-        *target_format = ImageFormat::SRGBA;
-        target_mat_type = CV_8UC4;
-        break;
-      case ImageFormat::SRGB:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      case ImageFormat::GRAY8:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      default:
-        return absl::UnknownError("Unexpected image frame format.");
-        break;
-      }
-
-      image_mat = absl::make_unique<cv::Mat>(
-          input_frame.Height(), input_frame.Width(), target_mat_type);
-
-      auto input_mat = formats::MatView(&input_frame);
-
-      if (input_frame.Format() == ImageFormat::GRAY8)
-      {
-        cv::Mat rgb_mat;
-        cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
-        rgb_mat.copyTo(*image_mat);
-      }
-      else
-      {
-        input_mat.copyTo(*image_mat);
-      }
-    }
-    else
-    {
-      image_mat = absl::make_unique<cv::Mat>(
-          150, 150, CV_8UC4,
-          cv::Scalar::all(255));
-      *target_format = ImageFormat::SRGBA;
-    }
-
-    return absl::OkStatus();
-  }
-
   absl::Status BilateralCalculator::BilateralFilter(CalculatorContext *cc,
                                                     const std::vector<double> &face_box)
   {
-    cv::Mat patch_face = mat_image_(cv::Range(face_box[1], face_box[3]), 
+    cv::Mat patch_wow = mat_image_(cv::Range(face_box[1], face_box[3]),
                                    cv::Range(face_box[0], face_box[2]));
-    cv::Mat patch_new, patch_wow;
-    cv::cvtColor(patch_face, patch_wow, cv::COLOR_RGBA2RGB);
-    cv::bilateralFilter(patch_wow, patch_new, 12, 50, 50);
-    cv::cvtColor(patch_new, patch_new, cv::COLOR_RGB2RGBA);
 
+    cv::cvtColor(patch_wow, patch_wow, CV_RGBA2RGB);
+    cv::bilateralFilter(patch_wow, out_mat, 12, 50, 50);
+    cv::cvtColor(out_mat, out_mat, CV_RGB2RGBA);
+    
+    return absl::OkStatus();
+  }
+  absl::Status BilateralCalculator::RenderToCpu(CalculatorContext *cc)
+  {
+    auto out_mat_ptr = absl::make_unique<cv::Mat>(out_mat);
+
+    if (cc->Outputs().HasTag(kOutTag))
+    {
+      cc->Outputs()
+          .Tag(kOutTag)
+          .Add(out_mat_ptr.release(), cc->InputTimestamp());
+    }
     return absl::OkStatus();
   }
 

mediapipe/calculators/beauty/draw_lipstick_calculator.cc

@@ -12,18 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <math.h>
-
-#include <algorithm>
-#include <cmath>
-#include <map>
-#include <string>
-
-#include <memory>
-
 #include "absl/strings/str_cat.h"
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/calculator_options.pb.h"
 #include "mediapipe/framework/formats/image_format.pb.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
@@ -31,16 +21,18 @@
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/vector.h"
 
+using namespace std;
 namespace mediapipe
 {
   namespace
   {
     constexpr char kMaskTag[] = "MASK";
-    constexpr char kImageFrameTag[] = "IMAGE";
+    constexpr char kMatTag[] = "MAT";
 
     inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
   } // namespace
@@ -59,21 +51,16 @@ namespace mediapipe
     absl::Status Close(CalculatorContext *cc) override;
 
   private:
-    absl::Status CreateRenderTargetCpu(CalculatorContext *cc,
-                                       std::unique_ptr<cv::Mat> &image_mat,
-                                       ImageFormat::Format *target_format);
-
     absl::Status RenderToCpu(
-        CalculatorContext *cc, const ImageFormat::Format &target_format,
+        CalculatorContext *cc);
-        uchar *data_image, std::unique_ptr<cv::Mat> &image_mat);
+    absl::Status DrawLipstick(CalculatorContext *cc,
-
+                              std::unordered_map<std::string, cv::Mat> &mask_vec);
-    absl::Status DrawLipstick(CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
-                              ImageFormat::Format *target_format,
-                              const std::unordered_map<std::string, cv::Mat> &mask_vec);
 
     // Indicates if image frame is available as input.
     bool image_frame_available_ = false;
     std::unordered_map<std::string, cv::Mat> all_masks;
+    cv::Mat spec_lips_mask;
+    cv::Mat mat_image_;
   };
   REGISTER_CALCULATOR(DrawLipstickCalculator);
 
@@ -81,10 +68,10 @@ namespace mediapipe
   {
     CHECK_GE(cc->Inputs().NumEntries(), 1);
 
-    if (cc->Inputs().HasTag(kImageFrameTag))
+    if (cc->Inputs().HasTag(kMatTag))
     {
-      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+      cc->Inputs().Tag(kMatTag).Set<cv::Mat>();
-      CHECK(cc->Outputs().HasTag(kImageFrameTag));
+      CHECK(cc->Outputs().HasTag(kMatTag));
     }
 
     // Data streams to render.
@@ -104,9 +91,13 @@ namespace mediapipe
       }
     }
 
-    if (cc->Outputs().HasTag(kImageFrameTag))
+    if (cc->Outputs().HasTag(kMatTag))
     {
-      cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
+      cc->Outputs().Tag(kMatTag).Set<cv::Mat>();
+    }
+    if (cc->Outputs().HasTag(kMaskTag))
+    {
+      cc->Outputs().Tag(kMaskTag).Set<cv::Mat>();
     }
 
     return absl::OkStatus();
@@ -116,13 +107,13 @@ namespace mediapipe
   {
     cc->SetOffset(TimestampDiff(0));
 
-    if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc))
+    if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc))
     {
       image_frame_available_ = true;
     }
 
     // Set the output header based on the input header (if present).
-    const char *tag = kImageFrameTag;
+    const char *tag = kMatTag;
     if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty())
     {
       const auto &input_header =
@@ -136,35 +127,34 @@ namespace mediapipe
 
   absl::Status DrawLipstickCalculator::Process(CalculatorContext *cc)
   {
-    if (cc->Inputs().HasTag(kImageFrameTag) &&
+    if (cc->Inputs().HasTag(kMatTag) &&
-        cc->Inputs().Tag(kImageFrameTag).IsEmpty())
+        cc->Inputs().Tag(kMatTag).IsEmpty())
     {
       return absl::OkStatus();
     }
 
-    // Initialize render target, drawn with OpenCV.
-    std::unique_ptr<cv::Mat> image_mat;
     ImageFormat::Format target_format;
 
-    if (cc->Outputs().HasTag(kImageFrameTag))
+    const cv::Mat &input_mat =
-    {
+        cc->Inputs().Tag(kMatTag).Get<cv::Mat>();
-      MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
+
-    }
+    mat_image_ = input_mat.clone();
 
     if (cc->Inputs().HasTag(kMaskTag) &&
         !cc->Inputs().Tag(kMaskTag).IsEmpty())
     {
       const std::vector<std::unordered_map<std::string, cv::Mat>> &mask_vec =
           cc->Inputs().Tag(kMaskTag).Get<std::vector<std::unordered_map<std::string, cv::Mat>>>();
+
       if (mask_vec.size() > 0)
       {
-        for (auto mask : mask_vec)
+        for (auto mask : mask_vec){
-          MP_RETURN_IF_ERROR(DrawLipstick(cc, image_mat, &target_format, mask));
+          MP_RETURN_IF_ERROR(DrawLipstick(cc, mask));
         }
       }
-    // Copy the rendered image to output.
+    }
-    uchar *image_mat_ptr = image_mat->data;
+
-    MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat));
+    MP_RETURN_IF_ERROR(RenderToCpu(cc));
 
     return absl::OkStatus();
   }
@@ -175,163 +165,71 @@ namespace mediapipe
   }
 
   absl::Status DrawLipstickCalculator::RenderToCpu(
-      CalculatorContext *cc, const ImageFormat::Format &target_format,
+      CalculatorContext *cc)
-      uchar *data_image, std::unique_ptr<cv::Mat> &image_mat)
   {
-    cv::Mat mat_image_ = *image_mat.get();
+    auto output_frame = absl::make_unique<cv::Mat>(mat_image_);
  
-    auto output_frame = absl::make_unique<ImageFrame>(
+    if (cc->Outputs().HasTag(kMatTag))
-        target_format, mat_image_.cols, mat_image_.rows);
-
-    output_frame->CopyPixelData(target_format, mat_image_.cols, mat_image_.rows, data_image,
-                                ImageFrame::kDefaultAlignmentBoundary);
-
-    if (cc->Outputs().HasTag(kImageFrameTag))
     {
       cc->Outputs()
-          .Tag(kImageFrameTag)
+          .Tag(kMatTag)
           .Add(output_frame.release(), cc->InputTimestamp());
     }
 
-    return absl::OkStatus();
+    spec_lips_mask.convertTo(spec_lips_mask, CV_32F, 1.0 / 255);
-  }
+    auto output_frame2 = absl::make_unique<cv::Mat>(spec_lips_mask);
 
-  absl::Status DrawLipstickCalculator::CreateRenderTargetCpu(
+    if (cc->Outputs().HasTag(kMaskTag))
-      CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
-      ImageFormat::Format *target_format)
     {
-    if (image_frame_available_)
+      cc->Outputs()
-    {
+          .Tag(kMaskTag)
-      const auto &input_frame =
+          .Add(output_frame2.release(), cc->InputTimestamp());
-          cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-
-      int target_mat_type;
-      switch (input_frame.Format())
-      {
-      case ImageFormat::SRGBA:
-        *target_format = ImageFormat::SRGBA;
-        target_mat_type = CV_8UC4;
-        break;
-      case ImageFormat::SRGB:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      case ImageFormat::GRAY8:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      default:
-        return absl::UnknownError("Unexpected image frame format.");
-        break;
-      }
-
-      image_mat = absl::make_unique<cv::Mat>(
-          input_frame.Height(), input_frame.Width(), target_mat_type);
-
-      auto input_mat = formats::MatView(&input_frame);
-
-      if (input_frame.Format() == ImageFormat::GRAY8)
-      {
-        cv::Mat rgb_mat;
-        cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
-        rgb_mat.copyTo(*image_mat);
-      }
-      else
-      {
-        input_mat.copyTo(*image_mat);
-      }
-    }
-    else
-    {
-      image_mat = absl::make_unique<cv::Mat>(
-          150, 150, CV_8UC4,
-          cv::Scalar(cv::Scalar::all(255)));
-      *target_format = ImageFormat::SRGBA;
     }
 
     return absl::OkStatus();
   }
 
   absl::Status DrawLipstickCalculator::DrawLipstick(CalculatorContext *cc,
-                                                    std::unique_ptr<cv::Mat> &image_mat,
+                                                     std::unordered_map<std::string, cv::Mat> &mask_vec)
-                                                    ImageFormat::Format *target_format,
-                                                    const std::unordered_map<std::string, cv::Mat> &mask_vec)
   {
-    cv::Mat mat_image__ = *image_mat.get();
+    cv::Mat upper_lips_mask = mask_vec.find("UPPER_LIP")->second;
-
+    cv::Mat lower_lips_mask = mask_vec.find("LOWER_LIP")->second;
-    cv::Mat spec_lips_mask, upper_lips_mask, lower_lips_mask;
-    spec_lips_mask = cv::Mat::zeros(mat_image__.size(), CV_32F);
-    upper_lips_mask = cv::Mat::zeros(mat_image__.size(), CV_32F);
-    lower_lips_mask = cv::Mat::zeros(mat_image__.size(), CV_32F);
-
-    upper_lips_mask = mask_vec.find("UPPER_LIP")->second;
-    lower_lips_mask = mask_vec.find("LOWER_LIP")->second;
 
     spec_lips_mask = upper_lips_mask + lower_lips_mask;
 
-    spec_lips_mask.convertTo(spec_lips_mask, CV_8U);
+    cv::resize(spec_lips_mask, spec_lips_mask, mat_image_.size(), cv::INTER_LINEAR);
 
-    cv::resize(spec_lips_mask, spec_lips_mask, mat_image__.size(), cv::INTER_LINEAR);
+    cv::Rect rect = cv::boundingRect(spec_lips_mask);
 
-    std::vector<int> x, y;
+    if (!rect.empty())
-    std::vector<cv::Point> location;
-
-    cv::findNonZero(spec_lips_mask, location);
-
-    for (auto &i : location)
     {
-      x.push_back(i.x);
+      double min_y = rect.y, max_y = rect.y + rect.height,
-      y.push_back(i.y);
+             max_x = rect.x + rect.width, min_x = rect.x;
-    }
-
-    if (!(x.empty()) && !(y.empty()))
-    {
-
-      double min_y, max_y, max_x, min_x;
-      cv::minMaxLoc(y, &min_y, &max_y);
-      cv::minMaxLoc(x, &min_x, &max_x);
 
       cv::Mat lips_crop_mask = spec_lips_mask(cv::Range(min_y, max_y), cv::Range(min_x, max_x));
       lips_crop_mask.convertTo(lips_crop_mask, CV_32F, 1.0 / 255);
 
-      cv::Mat lips_crop = cv::Mat(mat_image__(cv::Range(min_y, max_y), cv::Range(min_x, max_x)));
+      cv::Mat lips_crop = cv::Mat(mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x)));
-
       cv::Mat lips_blend = cv::Mat(lips_crop.size().height, lips_crop.size().width, CV_32FC4, cv::Scalar(255.0, 0, 0, 0));
-
       std::vector<cv::Mat> channels(4);
 
       cv::split(lips_blend, channels);
       channels[3] = lips_crop_mask * 20;
-
       cv::merge(channels, lips_blend);
 
       cv::Mat tmp_lip_mask;
+      channels[3].convertTo(tmp_lip_mask, CV_32F, 1.0 / 255);
 
-      channels[3].convertTo(tmp_lip_mask, CV_32FC1, 1.0 / 255);
+      cv::merge(std::vector{tmp_lip_mask, tmp_lip_mask, tmp_lip_mask, tmp_lip_mask}, tmp_lip_mask);
-
+      cv::multiply(lips_blend, tmp_lip_mask, lips_blend, 1.0, CV_32F);
-      cv::split(lips_blend, channels);
-      for (auto &ch : channels)
-      {
-        cv::multiply(ch, tmp_lip_mask, ch, 1.0, CV_32F);
-      }
-      cv::merge(channels, lips_blend);
-
       cv::subtract(1.0, tmp_lip_mask, tmp_lip_mask, cv::noArray(), CV_32F);
-
+      cv::multiply(lips_crop, tmp_lip_mask, lips_crop, 1.0, CV_8U);
-      cv::split(lips_crop, channels);
-      for (auto &ch : channels)
-      {
-        cv::multiply(ch, tmp_lip_mask, ch, 1.0, CV_8U);
-      }
-      cv::merge(channels, lips_crop);
-
       cv::add(lips_blend, lips_crop, lips_crop, cv::noArray(), CV_8U);
-
       lips_crop = cv::abs(lips_crop);
 
       cvtColor(lips_crop, lips_crop, cv::COLOR_RGBA2RGB);
 
-      cv::Mat slice = mat_image__(cv::Range(min_y, max_y), cv::Range(min_x, max_x));
+      cv::Mat slice = mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x));
       lips_crop_mask.convertTo(lips_crop_mask, slice.type());
       slice.copyTo(slice, lips_crop_mask);
 

mediapipe/calculators/beauty/imageframe_to_mat_calculator.cc

@@ -0,0 +1,131 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+using namespace std;
+namespace mediapipe
+{
+  namespace api2
+  {
+    class ImageFrameToMatCalculator : public Node
+    {
+    public:
+      static constexpr Input<ImageFrame> kImageFrame{"IMAGE"};
+      static constexpr Output<cv::Mat> kOut{"MAT"};
+
+      MEDIAPIPE_NODE_CONTRACT(kImageFrame, kOut);
+
+      static absl::Status UpdateContract(CalculatorContract *cc)
+      {
+        RET_CHECK(kOut(cc).IsConnected())
+            << "At least one output stream is expected.";
+        return absl::OkStatus();
+      }
+
+      absl::Status Process(CalculatorContext *cc)
+      {
+        // Initialize render target, drawn with OpenCV.
+        std::unique_ptr<cv::Mat> image_mat;
+        ImageFormat::Format target_format;
+
+        MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
+
+        MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat));
+
+        return absl::OkStatus();
+      }
+
+      absl::Status Close(CalculatorContext *cc)
+      {
+        return absl::OkStatus();
+      }
+
+      absl::Status RenderToCpu(
+          CalculatorContext *cc, const ImageFormat::Format &target_format, std::unique_ptr<cv::Mat> &image_mat)
+      {
+        auto output_frame = absl::make_unique<cv::Mat>(*image_mat);
+
+        kOut(cc).Send(std::move(output_frame));
+
+        return absl::OkStatus();
+      }
+
+      absl::Status CreateRenderTargetCpu(
+          CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
+          ImageFormat::Format *target_format)
+      {
+        if (!kImageFrame(cc).IsEmpty())
+        {
+          const auto &input_frame = *kImageFrame(cc);
+
+          int target_mat_type;
+          switch (input_frame.Format())
+          {
+          case ImageFormat::SRGBA:
+            *target_format = ImageFormat::SRGBA;
+            target_mat_type = CV_8UC4;
+            break;
+          case ImageFormat::SRGB:
+            *target_format = ImageFormat::SRGB;
+            target_mat_type = CV_8UC3;
+            break;
+          case ImageFormat::GRAY8:
+            *target_format = ImageFormat::SRGB;
+            target_mat_type = CV_8UC3;
+            break;
+          default:
+            return absl::UnknownError("Unexpected image frame format.");
+            break;
+          }
+
+          image_mat = absl::make_unique<cv::Mat>(
+              input_frame.Height(), input_frame.Width(), target_mat_type);
+
+          auto input_mat = formats::MatView(&input_frame);
+
+          if (input_frame.Format() == ImageFormat::GRAY8)
+          {
+            cv::Mat rgb_mat;
+            cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
+            rgb_mat.copyTo(*image_mat);
+          }
+          else
+          {
+            input_mat.copyTo(*image_mat);
+          }
+        }
+        else
+        {
+          image_mat = absl::make_unique<cv::Mat>(
+              1920, 1080, CV_8UC4,
+              cv::Scalar(cv::Scalar::all(255)));
+          *target_format = ImageFormat::SRGBA;
+        }
+
+        return absl::OkStatus();
+      }
+    };
+    MEDIAPIPE_REGISTER_NODE(ImageFrameToMatCalculator); // namespace mediapipe
+  }
+}

mediapipe/calculators/beauty/merging_images_calculator.cc

@@ -0,0 +1,179 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_options.pb.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/vector.h"
+
+using namespace std;
+
+namespace mediapipe
+{
+  namespace
+  {
+
+    constexpr char kMaskTag[] = "MASK";
+    constexpr char kMatTag[] = "MAT";
+    constexpr char kImageFrameTag[] = "IMAGE";
+  } // namespace
+
+  class MergeImagesCalculator : public CalculatorBase
+  {
+  public:
+    MergeImagesCalculator() = default;
+    ~MergeImagesCalculator() override = default;
+
+    static absl::Status GetContract(CalculatorContract *cc);
+
+    // From Calculator.
+    absl::Status Open(CalculatorContext *cc) override;
+    absl::Status Process(CalculatorContext *cc) override;
+    absl::Status Close(CalculatorContext *cc) override;
+
+  private:
+    absl::Status SmoothEnd(CalculatorContext *cc,
+                            const std::vector<double> &face_box);
+
+    cv::Mat mat_image_;
+  };
+  REGISTER_CALCULATOR(MergeImagesCalculator);
+
+  absl::Status MergeImagesCalculator::GetContract(CalculatorContract *cc)
+  {
+    CHECK_GE(cc->Inputs().NumEntries(), 1);
+    CHECK(cc->Outputs().HasTag(kImageFrameTag));
+
+    // Data streams to render.
+    for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId();
+         ++id)
+    {
+      auto tag_and_index = cc->Inputs().TagAndIndexFromId(id);
+      std::string tag = tag_and_index.first;
+      if (tag == kMatTag)
+      {
+        cc->Inputs().Get(id).Set<cv::Mat>();
+      }
+      else
+      {
+        // Every other tag defaults to accepting a single object of Mat type.
+        cc->Inputs().Get(id).Set<cv::Mat>();
+      }
+    }
+
+    if (cc->Outputs().HasTag(kImageFrameTag))
+    {
+      cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
+    }
+
+    return absl::OkStatus();
+  }
+
+  absl::Status MergeImagesCalculator::Open(CalculatorContext *cc)
+  {
+    cc->SetOffset(TimestampDiff(0));
+
+    // Set the output header based on the input header (if present).
+    const char *tag = kMatTag;
+    if (!cc->Inputs().Tag(tag).Header().IsEmpty())
+    {
+      const auto &input_header =
+          cc->Inputs().Tag(tag).Header().Get<VideoHeader>();
+      auto *output_video_header = new VideoHeader(input_header);
+      cc->Outputs().Tag(tag).SetHeader(Adopt(output_video_header));
+    }
+
+    return absl::OkStatus();
+  }
+
+  absl::Status MergeImagesCalculator::Process(CalculatorContext *cc)
+  {
+    if (cc->Inputs().HasTag(kMatTag) &&
+        cc->Inputs().Tag(kMatTag).IsEmpty())
+    {
+      return absl::OkStatus();
+    }
+
+    const auto &input_mat_c = cc->Inputs().Get(kMatTag, 0).Get<cv::Mat>();
+    cv::Mat input_mat = input_mat_c.clone();
+    cv::Mat input_mat2 = input_mat.clone();
+    cv::Mat mask = cc->Inputs().Get(kMaskTag, 0).Get<cv::Mat>();
+
+    cv::cvtColor(mask, mask, cv::COLOR_GRAY2RGBA);
+     
+    input_mat.convertTo(input_mat, CV_32F);
+    mat_image_ = mask.mul(input_mat);
+    int image_number = cc->Inputs().NumEntries() / 2;
+    cv::Mat all_masks = mask;
+
+    for (int i = 1; i < image_number; i++)
+    {
+        const auto &input_mat_c = cc->Inputs().Get(kMatTag, i).Get<cv::Mat>();
+        cv::Mat input_mat = input_mat_c.clone();
+        mask = cc->Inputs().Get(kMaskTag, i).Get<cv::Mat>();
+        cv::cvtColor(mask, mask, cv::COLOR_GRAY2RGBA);
+        input_mat.convertTo(input_mat, CV_32F);
+        mat_image_ += mask.mul(input_mat);
+        all_masks += mask;
+
+    }
+
+    input_mat2.convertTo(input_mat2, CV_32F);
+    cv::threshold(all_masks, all_masks, 1, 255, CV_THRESH_TRUNC);
+    input_mat2 = input_mat2.mul(1-(all_masks*255));
+
+    input_mat2.convertTo(input_mat2, CV_8U);
+    mat_image_.convertTo(mat_image_, CV_8U);
+
+    mat_image_ += input_mat2;
+
+    std::unique_ptr<cv::Mat> image_mat = absl::make_unique<cv::Mat>(
+          mat_image_.rows, mat_image_.cols, ImageFormat::SRGBA);
+    mat_image_.convertTo(mat_image_, CV_8U);
+    mat_image_.copyTo(*image_mat);
+
+    uchar *image_mat_ptr = image_mat->data;
+ 
+    // Copy the rendered image to output.
+    auto output_frame = absl::make_unique<ImageFrame>(
+        ImageFormat::SRGBA, mat_image_.cols, mat_image_.rows);
+
+    output_frame->CopyPixelData(ImageFormat::SRGBA, mat_image_.cols, mat_image_.rows, image_mat_ptr,
+                                ImageFrame::kDefaultAlignmentBoundary);
+
+    if (cc->Outputs().HasTag(kImageFrameTag))
+    {
+      cc->Outputs()
+          .Tag(kImageFrameTag)
+          .Add(output_frame.release(), cc->InputTimestamp());
+    }
+
+    return absl::OkStatus();
+  }
+
+  absl::Status MergeImagesCalculator::Close(CalculatorContext *cc)
+  {
+    return absl::OkStatus();
+  }
+} // namespace mediapipe

mediapipe/calculators/beauty/smooth_face1_calculator.cc

@@ -12,15 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <math.h>
-
-#include <algorithm>
-#include <cmath>
-#include <iostream>
-#include <tuple>
-
-#include <memory>
-
 #include "absl/strings/str_cat.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_options.pb.h"
@@ -31,25 +22,20 @@
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/vector.h"
 
+using namespace std;
 namespace mediapipe
 {
   namespace
   {
 
     constexpr char kMaskTag[] = "MASK";
-    constexpr char kFaceBoxTag[] = "FACEBOX";
+    constexpr char kMatTag[] = "MAT";
-    constexpr char kImageFrameTag[] = "IMAGE";
+    constexpr char kFaceTag[] = "FACE";
-
-    enum
-    {
-      ATTRIB_VERTEX,
-      ATTRIB_TEXTURE_POSITION,
-      NUM_ATTRIBUTES
-    };
 
     inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
   } // namespace
@@ -68,13 +54,8 @@ namespace mediapipe
     absl::Status Close(CalculatorContext *cc) override;
 
   private:
-    absl::Status CreateRenderTargetCpu(CalculatorContext *cc,
-                                       std::unique_ptr<cv::Mat> &image_mat,
-                                       ImageFormat::Format *target_format);
-
     absl::Status RenderToCpu(
-        CalculatorContext *cc, const ImageFormat::Format &target_format,
+        CalculatorContext *cc);
-        uchar *data_image);
 
     absl::Status SmoothFace(CalculatorContext *cc,
                             const std::unordered_map<std::string, cv::Mat> &mask_vec,
@@ -90,7 +71,7 @@ namespace mediapipe
     cv::Mat mat_image_;
     cv::Mat not_full_face;
     std::vector<double> face_box;
-    std::unique_ptr<cv::Mat> image_mat;
+    std::pair<cv::Mat, std::vector<double>> face;
   };
   REGISTER_CALCULATOR(SmoothFaceCalculator1);
 
@@ -98,10 +79,10 @@ namespace mediapipe
   {
     CHECK_GE(cc->Inputs().NumEntries(), 1);
 
-    if (cc->Inputs().HasTag(kImageFrameTag))
+    if (cc->Inputs().HasTag(kMatTag))
     {
-      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+      cc->Inputs().Tag(kMatTag).Set<cv::Mat>();
-      CHECK(cc->Outputs().HasTag(kImageFrameTag));
+      CHECK(cc->Outputs().HasTag(kMatTag));
       CHECK(cc->Outputs().HasTag(kMaskTag));
     }
 
@@ -121,23 +102,23 @@ namespace mediapipe
         cc->Inputs().Get(id).Set<cv::Mat>();
       }
 
-      if (tag == kFaceBoxTag)
+      if (tag == kFaceTag)
       {
         cc->Inputs().Get(id).Set<std::vector<std::tuple<double, double, double, double>>>();
       }
     }
 
-    if (cc->Outputs().HasTag(kImageFrameTag))
+    if (cc->Outputs().HasTag(kMatTag))
     {
-      cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
+      cc->Outputs().Tag(kMatTag).Set<cv::Mat>();
     }
     if (cc->Outputs().HasTag(kMaskTag))
     {
       cc->Outputs().Tag(kMaskTag).Set<cv::Mat>();
     }
-    if (cc->Outputs().HasTag(kFaceBoxTag))
+    if (cc->Outputs().HasTag(kFaceTag))
     {
-      cc->Outputs().Tag(kFaceBoxTag).Set<std::vector<double>>();
+      cc->Outputs().Tag(kFaceTag).Set<std::pair<cv::Mat, std::vector<double>>>();
     }
 
     return absl::OkStatus();
@@ -147,13 +128,13 @@ namespace mediapipe
   {
     cc->SetOffset(TimestampDiff(0));
 
-    if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc))
+    if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc))
     {
       image_frame_available_ = true;
     }
 
     // Set the output header based on the input header (if present).
-    const char *tag = kImageFrameTag;
+    const char *tag = kMatTag;
     if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty())
     {
       const auto &input_header =
@@ -167,8 +148,8 @@ namespace mediapipe
 
   absl::Status SmoothFaceCalculator1::Process(CalculatorContext *cc)
   {
-    if (cc->Inputs().HasTag(kImageFrameTag) &&
+    if (cc->Inputs().HasTag(kMatTag) &&
-        cc->Inputs().Tag(kImageFrameTag).IsEmpty())
+        cc->Inputs().Tag(kMatTag).IsEmpty())
     {
       return absl::OkStatus();
     }
@@ -176,25 +157,24 @@ namespace mediapipe
     // Initialize render target, drawn with OpenCV.
     ImageFormat::Format target_format;
 
-    if (cc->Outputs().HasTag(kImageFrameTag))
+    const cv::Mat &input_mat =
-    {
+        cc->Inputs().Tag(kMatTag).Get<cv::Mat>();
-      MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
-    }
 
-    mat_image_ = *image_mat.get();
+    mat_image_ = input_mat.clone();
-    image_width_ = image_mat->cols;
+
-    image_height_ = image_mat->rows;
+    image_width_ = input_mat.cols;
+    image_height_ = input_mat.rows;
 
     if (cc->Inputs().HasTag(kMaskTag) &&
         !cc->Inputs().Tag(kMaskTag).IsEmpty() &&
-        cc->Inputs().HasTag(kFaceBoxTag) &&
+        cc->Inputs().HasTag(kFaceTag) &&
-        !cc->Inputs().Tag(kFaceBoxTag).IsEmpty())
+        !cc->Inputs().Tag(kFaceTag).IsEmpty())
     {
       const std::vector<std::unordered_map<std::string, cv::Mat>> &mask_vec =
           cc->Inputs().Tag(kMaskTag).Get<std::vector<std::unordered_map<std::string, cv::Mat>>>();
 
       const std::vector<std::tuple<double, double, double, double>> &face_boxes =
-            cc->Inputs().Tag(kFaceBoxTag).Get<std::vector<std::tuple<double, double, double, double>>>();
+          cc->Inputs().Tag(kFaceTag).Get<std::vector<std::tuple<double, double, double, double>>>();
 
       if (mask_vec.size() > 0 && face_boxes.size() > 0)
       {
@@ -202,9 +182,8 @@ namespace mediapipe
           MP_RETURN_IF_ERROR(SmoothFace(cc, mask_vec[i], face_boxes[i]));
       }
     }
-    // Copy the rendered image to output.
+
-    uchar *image_mat_ptr = image_mat->data;
+    MP_RETURN_IF_ERROR(RenderToCpu(cc));
-    MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr));
 
     return absl::OkStatus();
   }
@@ -215,22 +194,18 @@ namespace mediapipe
   }
 
   absl::Status SmoothFaceCalculator1::RenderToCpu(
-      CalculatorContext *cc, const ImageFormat::Format &target_format,
+      CalculatorContext *cc)
-      uchar *data_image)
   {
-    auto output_frame1 = absl::make_unique<ImageFrame>(
+    auto output_frame1 = absl::make_unique<cv::Mat>(mat_image_);
-        target_format, image_width_, image_height_);
 
-    output_frame1->CopyPixelData(target_format, image_width_, image_height_, data_image,
+    if (cc->Outputs().HasTag(kMatTag))
-                                ImageFrame::kDefaultAlignmentBoundary);
-
-    if (cc->Outputs().HasTag(kImageFrameTag))
     {
       cc->Outputs()
-          .Tag(kImageFrameTag)
+          .Tag(kMatTag)
           .Add(output_frame1.release(), cc->InputTimestamp());
     }
 
+    not_full_face.convertTo(not_full_face, CV_32F, 1.0 / 255);
     auto output_frame2 = absl::make_unique<cv::Mat>(not_full_face);
 
     if (cc->Outputs().HasTag(kMaskTag))
@@ -240,71 +215,15 @@ namespace mediapipe
           .Add(output_frame2.release(), cc->InputTimestamp());
     }
 
-    auto output_frame3 = absl::make_unique<std::vector<double>>(face_box);
+    auto output_frame3 = absl::make_unique<std::pair<cv::Mat, std::vector<double>>>(
+        face);
 
-    if (cc->Outputs().HasTag(kFaceBoxTag))
+    if (cc->Outputs().HasTag(kFaceTag))
     {
       cc->Outputs()
-          .Tag(kFaceBoxTag)
+          .Tag(kFaceTag)
           .Add(output_frame3.release(), cc->InputTimestamp());
     }
-
-    return absl::OkStatus();
-  }
-
-  absl::Status SmoothFaceCalculator1::CreateRenderTargetCpu(
-      CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
-      ImageFormat::Format *target_format)
-  {
-    if (image_frame_available_)
-    {
-      const auto &input_frame =
-          cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-
-      int target_mat_type;
-      switch (input_frame.Format())
-      {
-      case ImageFormat::SRGBA:
-        *target_format = ImageFormat::SRGBA;
-        target_mat_type = CV_8UC4;
-        break;
-      case ImageFormat::SRGB:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      case ImageFormat::GRAY8:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      default:
-        return absl::UnknownError("Unexpected image frame format.");
-        break;
-      }
-
-      image_mat = absl::make_unique<cv::Mat>(
-          input_frame.Height(), input_frame.Width(), target_mat_type);
-
-      auto input_mat = formats::MatView(&input_frame);
-
-      if (input_frame.Format() == ImageFormat::GRAY8)
-      {
-        cv::Mat rgb_mat;
-        cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
-        rgb_mat.copyTo(*image_mat);
-      }
-      else
-      {
-        input_mat.copyTo(*image_mat);
-      }
-    }
-    else
-    {
-      image_mat = absl::make_unique<cv::Mat>(
-          150, 150, CV_8UC4,
-          cv::Scalar::all(255));
-      *target_format = ImageFormat::SRGBA;
-    }
-
     return absl::OkStatus();
   }
 
@@ -383,7 +302,7 @@ namespace mediapipe
                                                  const std::tuple<double, double, double, double> &face_boxx)
   {
     not_full_face = mask_vec.find("FACE_OVAL")->second.clone() -
-//                    predict_forehead_mask(mask_vec, std::get<1>(face_boxx)) -
+                    //                    predict_forehead_mask(mask_vec, std::get<1>(face_boxx)) -
                     mask_vec.find("LEFT_EYE")->second.clone() -
                     mask_vec.find("RIGHT_EYE")->second.clone() -
                     mask_vec.find("LEFT_BROW")->second.clone() -
@@ -395,26 +314,18 @@ namespace mediapipe
                mat_image_.size(), 0, 0,
                cv::INTER_LINEAR);
 
-    std::vector<int> x, y;
+    cv::Rect rect = cv::boundingRect(not_full_face);
-    std::vector<cv::Point> location;
 
-    cv::findNonZero(not_full_face, location);
+    if (!rect.empty())
-
-    double min_y, min_x, max_x, max_y;
-
-    for (auto &i : location)
     {
-      x.push_back(i.x);
+      double min_y = rect.y, max_y = rect.y + rect.height,
-      y.push_back(i.y);
+             max_x = rect.x + rect.width, min_x = rect.x;
+      face.second.push_back(min_x);
+      face.second.push_back(min_y);
+      face.second.push_back(max_x);
+      face.second.push_back(max_y);
+      face.first = mat_image_;
     }
-
-    cv::minMaxLoc(x, &min_x, &max_x);
-    cv::minMaxLoc(y, &min_y, &max_y);
-    face_box.push_back(min_x);
-    face_box.push_back(min_y);
-    face_box.push_back(max_x);
-    face_box.push_back(max_y);
-
     return absl::OkStatus();
   }
 

mediapipe/calculators/beauty/smooth_face2_calculator.cc

@@ -12,15 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <math.h>
-
-#include <algorithm>
-#include <cmath>
-#include <iostream>
-#include <tuple>
-
-#include <memory>
-
 #include "absl/strings/str_cat.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_options.pb.h"
@@ -31,10 +22,13 @@
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/vector.h"
 
+using namespace std;
+
 namespace mediapipe
 {
   namespace
@@ -42,16 +36,9 @@ namespace mediapipe
 
     constexpr char kMaskTag[] = "MASK";
     constexpr char kFaceBoxTag[] = "FACEBOX";
-    constexpr char kImageFrameTag[] = "IMAGE";
+    constexpr char kMatTag[] = "MAT";
     constexpr char kImageNewTag[] = "IMAGE2";
 
-    enum
-    {
-      ATTRIB_VERTEX,
-      ATTRIB_TEXTURE_POSITION,
-      NUM_ATTRIBUTES
-    };
-
     inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
   } // namespace
 
@@ -69,14 +56,8 @@ namespace mediapipe
     absl::Status Close(CalculatorContext *cc) override;
 
   private:
-    absl::Status CreateRenderTargetCpu(CalculatorContext *cc,
-                                       std::unique_ptr<cv::Mat> &image_mat,
-                                       std::unique_ptr<cv::Mat> &new_mat,
-                                       ImageFormat::Format *target_format);
-
     absl::Status RenderToCpu(
-        CalculatorContext *cc, const ImageFormat::Format &target_format,
+        CalculatorContext *cc);
-        uchar *data_image);
 
     absl::Status SmoothEnd(CalculatorContext *cc,
                            const std::vector<double> &face_box);
@@ -89,9 +70,6 @@ namespace mediapipe
     cv::Mat mat_image_;
     cv::Mat new_image_;
     cv::Mat not_full_face;
-    std::unique_ptr<cv::Mat> image_mat;
-    std::unique_ptr<cv::Mat> new_mat;
-    std::unique_ptr<cv::Mat> nff_mat;
   };
   REGISTER_CALCULATOR(SmoothFaceCalculator2);
 
@@ -99,40 +77,28 @@ namespace mediapipe
   {
     CHECK_GE(cc->Inputs().NumEntries(), 1);
 
-    if (cc->Inputs().HasTag(kImageFrameTag))
+    if (cc->Inputs().HasTag(kMatTag))
     {
-      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+      cc->Inputs().Tag(kMatTag).Set<cv::Mat>();
-      CHECK(cc->Outputs().HasTag(kImageFrameTag));
+      CHECK(cc->Outputs().HasTag(kMatTag));
     }
     if (cc->Inputs().HasTag(kImageNewTag))
     {
-      cc->Inputs().Tag(kImageNewTag).Set<ImageFrame>();
+      cc->Inputs().Tag(kImageNewTag).Set<cv::Mat>();
     }
     if (cc->Inputs().HasTag(kMaskTag))
     {
       cc->Inputs().Tag(kMaskTag).Set<cv::Mat>();
     }
 
-    // Data streams to render.
+    if (cc->Inputs().HasTag(kFaceBoxTag))
-    for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId();
-         ++id)
     {
-      auto tag_and_index = cc->Inputs().TagAndIndexFromId(id);
+      cc->Inputs().Tag(kFaceBoxTag).Set<std::pair<cv::Mat, std::vector<double>>>();
-      std::string tag = tag_and_index.first;
-      if (tag == kFaceBoxTag)
-      {
-        cc->Inputs().Get(id).Set<std::vector<double>>();
-      }
-      else if (tag.empty())
-      {
-        // Empty tag defaults to accepting a single object of Mat type.
-        cc->Inputs().Get(id).Set<cv::Mat>();
-      }
     }
 
-    if (cc->Outputs().HasTag(kImageFrameTag))
+    if (cc->Outputs().HasTag(kMatTag))
     {
-      cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
+      cc->Outputs().Tag(kMatTag).Set<cv::Mat>();
     }
 
     return absl::OkStatus();
@@ -142,13 +108,13 @@ namespace mediapipe
   {
     cc->SetOffset(TimestampDiff(0));
 
-    if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc))
+    if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc))
     {
       image_frame_available_ = true;
     }
 
     // Set the output header based on the input header (if present).
-    const char *tag = kImageFrameTag;
+    const char *tag = kMatTag;
     if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty())
     {
       const auto &input_header =
@@ -162,8 +128,9 @@ namespace mediapipe
 
   absl::Status SmoothFaceCalculator2::Process(CalculatorContext *cc)
   {
-    if (cc->Inputs().HasTag(kImageFrameTag) &&
+
-        cc->Inputs().Tag(kImageFrameTag).IsEmpty())
+    if (cc->Inputs().HasTag(kMatTag) &&
+        cc->Inputs().Tag(kMatTag).IsEmpty())
     {
       return absl::OkStatus();
     }
@@ -183,28 +150,33 @@ namespace mediapipe
       return absl::OkStatus();
     }
 
-    // Initialize render target, drawn with OpenCV.
+    not_full_face = cc->Inputs().Tag(kMaskTag).Get<cv::Mat>();
-    ImageFormat::Format target_format;
+    not_full_face.convertTo(not_full_face, CV_8U, 255);
 
-    if (cc->Outputs().HasTag(kImageFrameTag))
+    const cv::Mat &input_mat =
+        cc->Inputs().Tag(kMatTag).Get<cv::Mat>();
+
+    mat_image_ = input_mat.clone();
+
+    const cv::Mat &input_new =
+        cc->Inputs().Tag(kImageNewTag).Get<cv::Mat>();
+
+    new_image_ = input_new.clone();
+
+    image_width_ = input_mat.cols;
+    image_height_ = input_mat.rows;
+
+    const auto &face_box_pair =
+        cc->Inputs().Tag(kFaceBoxTag).Get<std::pair<cv::Mat, std::vector<double>>>();
+
+    const auto &face_box = face_box_pair.second;
+
+    if (!face_box.empty())
     {
-      MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, new_mat, &target_format));
+      MP_RETURN_IF_ERROR(SmoothEnd(cc, face_box));
     }
 
-    not_full_face = cc->Inputs().Tag(kMaskTag).Get<cv::Mat>();
+    MP_RETURN_IF_ERROR(RenderToCpu(cc));
-    new_image_ = *new_mat.get();
-    mat_image_ = *image_mat.get();
-    image_width_ = image_mat->cols;
-    image_height_ = image_mat->rows;
-
-    const std::vector<double> &face_box =
-        cc->Inputs().Tag(kFaceBoxTag).Get<std::vector<double>>();
-
-    MP_RETURN_IF_ERROR(SmoothEnd(cc, face_box));
-
-    // Copy the rendered image to output.
-    uchar *image_mat_ptr = image_mat->data;
-    MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr));
 
     return absl::OkStatus();
   }
@@ -215,114 +187,31 @@ namespace mediapipe
   }
 
   absl::Status SmoothFaceCalculator2::RenderToCpu(
-      CalculatorContext *cc, const ImageFormat::Format &target_format,
+      CalculatorContext *cc)
-      uchar *data_image)
   {
-    auto output_frame = absl::make_unique<ImageFrame>(
+    auto output_frame = absl::make_unique<cv::Mat>(mat_image_);
-        target_format, image_width_, image_height_);
 
-    output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image,
+    if (cc->Outputs().HasTag(kMatTag))
-                                ImageFrame::kDefaultAlignmentBoundary);
-
-    if (cc->Outputs().HasTag(kImageFrameTag))
     {
       cc->Outputs()
-          .Tag(kImageFrameTag)
+          .Tag(kMatTag)
           .Add(output_frame.release(), cc->InputTimestamp());
     }
 
     return absl::OkStatus();
   }
 
-  absl::Status SmoothFaceCalculator2::CreateRenderTargetCpu(
-      CalculatorContext *cc, 
-      std::unique_ptr<cv::Mat> &image_mat,
-      std::unique_ptr<cv::Mat> &new_mat,
-      ImageFormat::Format *target_format)
-  {
-    if (image_frame_available_)
-    {
-      const auto &input_frame =
-          cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-      const auto &new_frame =
-          cc->Inputs().Tag(kImageNewTag).Get<ImageFrame>();
-
-      int target_mat_type;
-      switch (input_frame.Format())
-      {
-      case ImageFormat::SRGBA:
-        *target_format = ImageFormat::SRGBA;
-        target_mat_type = CV_8UC4;
-        break;
-      case ImageFormat::SRGB:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      case ImageFormat::GRAY8:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      default:
-        return absl::UnknownError("Unexpected image frame format.");
-        break;
-      }
-
-      image_mat = absl::make_unique<cv::Mat>(
-          input_frame.Height(), input_frame.Width(), target_mat_type);
-
-      auto input_mat = formats::MatView(&input_frame);
-
-      new_mat = absl::make_unique<cv::Mat>(
-          new_frame.Height(), new_frame.Width(), target_mat_type);
-
-      auto new_input_mat = formats::MatView(&new_frame);
-
-      if (input_frame.Format() == ImageFormat::GRAY8)
-      {
-        cv::Mat rgb_mat;
-        cv::Mat rgb_mat2;
-        cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
-        rgb_mat.copyTo(*image_mat);
-        cv::cvtColor(new_input_mat, rgb_mat2, CV_GRAY2RGB);
-        rgb_mat2.copyTo(*new_mat);
-      }
-      else
-      {
-        input_mat.copyTo(*image_mat);
-        new_input_mat.copyTo(*new_mat);
-      }
-    }
-    else
-    {
-      image_mat = absl::make_unique<cv::Mat>(
-          150, 150, CV_8UC4,
-          cv::Scalar::all(255));
-      nff_mat = absl::make_unique<cv::Mat>(
-          150, 150, CV_8UC4,
-          cv::Scalar::all(255));
-      new_mat = absl::make_unique<cv::Mat>(
-          150, 150, CV_8UC4,
-          cv::Scalar::all(255));
-      *target_format = ImageFormat::SRGBA;
-    }
-
-    return absl::OkStatus();
-  }
-  
-
   absl::Status SmoothFaceCalculator2::SmoothEnd(CalculatorContext *cc,
                                                 const std::vector<double> &face_box)
   {
     cv::Mat patch_face = mat_image_(cv::Range(face_box[1], face_box[3]),
                                     cv::Range(face_box[0], face_box[2]));
-    cv::Mat patch_new = new_image_(cv::Range(face_box[1], face_box[3]), 
-                                   cv::Range(face_box[0], face_box[2]));
     cv::Mat patch_nff = not_full_face(cv::Range(face_box[1], face_box[3]),
                                       cv::Range(face_box[0], face_box[2]));
 
     cv::Mat patch_new_nff, patch_new_mask, patch, patch_face_nff;
 
-    patch_new.copyTo(patch_new_nff, patch_nff);
+    new_image_.copyTo(patch_new_nff, patch_nff);
 
     patch_face.copyTo(patch_face_nff, patch_nff);
     cv::cvtColor(patch_face_nff, patch_face_nff, cv::COLOR_RGBA2RGB);

mediapipe/calculators/beauty/whiten_teeth_calculator.cc

@@ -12,16 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <math.h>
-
-#include <algorithm>
-#include <cmath>
-
-#include <memory>
-
 #include "absl/strings/str_cat.h"
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/calculator_options.pb.h"
 #include "mediapipe/framework/formats/image_format.pb.h"
 #include "mediapipe/framework/formats/image_frame.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
@@ -29,17 +21,19 @@
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/vector.h"
 
+using namespace std;
 namespace mediapipe
 {
   namespace
   {
 
     constexpr char kMaskTag[] = "MASK";
-    constexpr char kImageFrameTag[] = "IMAGE";
+    constexpr char kMatTag[] = "MAT";
 
     inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
   } // namespace
@@ -58,20 +52,15 @@ namespace mediapipe
     absl::Status Close(CalculatorContext *cc) override;
 
   private:
-    absl::Status CreateRenderTargetCpu(CalculatorContext *cc,
-                                       std::unique_ptr<cv::Mat> &image_mat,
-                                       ImageFormat::Format *target_format);
-
     absl::Status RenderToCpu(
-        CalculatorContext *cc, const ImageFormat::Format &target_format,
+        CalculatorContext *cc);
-        uchar *data_image, std::unique_ptr<cv::Mat> &image_mat);
 
-    absl::Status WhitenTeeth(CalculatorContext *cc, ImageFormat::Format *target_format,
+    absl::Status WhitenTeeth(CalculatorContext *cc,
                              const std::unordered_map<std::string, cv::Mat> &mask_vec);
 
     // Indicates if image frame is available as input.
     bool image_frame_available_ = false;
-    std::unique_ptr<cv::Mat> image_mat;
+    cv::Mat mouth;
     cv::Mat mat_image_;
     int image_width_;
     int image_height_;
@@ -82,10 +71,10 @@ namespace mediapipe
   {
     CHECK_GE(cc->Inputs().NumEntries(), 1);
 
-    if (cc->Inputs().HasTag(kImageFrameTag))
+    if (cc->Inputs().HasTag(kMatTag))
     {
-      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+      cc->Inputs().Tag(kMatTag).Set<cv::Mat>();
-      CHECK(cc->Outputs().HasTag(kImageFrameTag));
+      CHECK(cc->Outputs().HasTag(kMatTag));
     }
 
     // Data streams to render.
@@ -105,9 +94,13 @@ namespace mediapipe
       }
     }
 
-    if (cc->Outputs().HasTag(kImageFrameTag))
+    if (cc->Outputs().HasTag(kMatTag))
     {
-      cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
+      cc->Outputs().Tag(kMatTag).Set<cv::Mat>();
+    }
+    if (cc->Outputs().HasTag(kMaskTag))
+    {
+      cc->Outputs().Tag(kMaskTag).Set<cv::Mat>();
     }
 
     return absl::OkStatus();
@@ -117,16 +110,13 @@ namespace mediapipe
   {
     cc->SetOffset(TimestampDiff(0));
 
-    if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc))
+    if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc))
     {
       image_frame_available_ = true;
     }
-    else
-    {
-    }
 
     // Set the output header based on the input header (if present).
-    const char *tag = kImageFrameTag;
+    const char *tag = kMatTag;
     if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty())
     {
       const auto &input_header =
@@ -140,22 +130,21 @@ namespace mediapipe
 
   absl::Status WhitenTeethCalculator::Process(CalculatorContext *cc)
   {
-    if (cc->Inputs().HasTag(kImageFrameTag) &&
+    if (cc->Inputs().HasTag(kMatTag) &&
-        cc->Inputs().Tag(kImageFrameTag).IsEmpty())
+        cc->Inputs().Tag(kMatTag).IsEmpty())
     {
       return absl::OkStatus();
     }
 
-    // Initialize render target, drawn with OpenCV.
     ImageFormat::Format target_format;
 
-    if (cc->Outputs().HasTag(kImageFrameTag))
+    const cv::Mat &input_mat =
-    {
+        cc->Inputs().Tag(kMatTag).Get<cv::Mat>();
-      MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
+
-    }
+    mat_image_ = input_mat.clone();
-    mat_image_ = *image_mat.get();
+
-    image_width_ = image_mat->cols;
+    image_width_ = input_mat.cols;
-    image_height_ = image_mat->rows;
+    image_height_ = input_mat.rows;
 
     if (cc->Inputs().HasTag(kMaskTag) &&
         !cc->Inputs().Tag(kMaskTag).IsEmpty())
@@ -165,12 +154,11 @@ namespace mediapipe
       if (mask_vec.size() > 0)
       {
         for (auto mask : mask_vec)
-          MP_RETURN_IF_ERROR(WhitenTeeth(cc, &target_format, mask));
+          MP_RETURN_IF_ERROR(WhitenTeeth(cc, mask));
       }
     }
-    // Copy the rendered image to output.
+
-    uchar *image_mat_ptr = image_mat->data;
+    MP_RETURN_IF_ERROR(RenderToCpu(cc));
-    MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat));
 
     return absl::OkStatus();
   }
@@ -181,124 +169,57 @@ namespace mediapipe
   }
 
   absl::Status WhitenTeethCalculator::RenderToCpu(
-      CalculatorContext *cc, const ImageFormat::Format &target_format,
+      CalculatorContext *cc)
-      uchar *data_image, std::unique_ptr<cv::Mat> &image_mat)
   {
+    auto output_frame = absl::make_unique<cv::Mat>(mat_image_);
 
-    cv::Mat mat_image_ = *image_mat.get();
+    if (cc->Outputs().HasTag(kMatTag))
-
-    auto output_frame = absl::make_unique<ImageFrame>(
-        target_format, image_width_, image_height_);
-
-    output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image,
-                                ImageFrame::kDefaultAlignmentBoundary);
-
-    if (cc->Outputs().HasTag(kImageFrameTag))
     {
       cc->Outputs()
-          .Tag(kImageFrameTag)
+          .Tag(kMatTag)
           .Add(output_frame.release(), cc->InputTimestamp());
     }
 
-    return absl::OkStatus();
+    mouth.convertTo(mouth, CV_32F, 255);
-  }
+    auto output_frame2 = absl::make_unique<cv::Mat>(mouth);
 
-  absl::Status WhitenTeethCalculator::CreateRenderTargetCpu(
+    if (cc->Outputs().HasTag(kMaskTag))
-      CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
-      ImageFormat::Format *target_format)
     {
-    if (image_frame_available_)
+      cc->Outputs()
-    {
+          .Tag(kMaskTag)
-      const auto &input_frame =
+          .Add(output_frame2.release(), cc->InputTimestamp());
-          cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-
-      int target_mat_type;
-      switch (input_frame.Format())
-      {
-      case ImageFormat::SRGBA:
-        *target_format = ImageFormat::SRGBA;
-        target_mat_type = CV_8UC4;
-        break;
-      case ImageFormat::SRGB:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      case ImageFormat::GRAY8:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      default:
-        return absl::UnknownError("Unexpected image frame format.");
-        break;
-      }
-
-      image_mat = absl::make_unique<cv::Mat>(
-          input_frame.Height(), input_frame.Width(), target_mat_type);
-
-      auto input_mat = formats::MatView(&input_frame);
-
-      if (input_frame.Format() == ImageFormat::GRAY8)
-      {
-        cv::Mat rgb_mat;
-        cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
-        rgb_mat.copyTo(*image_mat);
-      }
-      else
-      {
-        input_mat.copyTo(*image_mat);
-      }
-    }
-    else
-    {
-      image_mat = absl::make_unique<cv::Mat>(
-          150, 150, CV_8UC4,
-          cv::Scalar(255, 255,
-                     255));
-      *target_format = ImageFormat::SRGBA;
     }
 
     return absl::OkStatus();
   }
 
   absl::Status WhitenTeethCalculator::WhitenTeeth(CalculatorContext *cc,
-                                                  ImageFormat::Format *target_format,
                                                   const std::unordered_map<std::string, cv::Mat> &mask_vec)
   {
-    cv::Mat mouth_mask, mouth;
+    cv::Mat mouth_mask = mask_vec.find("MOUTH_INSIDE")->second.clone();
-    mouth_mask = cv::Mat::zeros(mat_image_.size(), CV_32F);
-
-    mouth_mask = mask_vec.find("MOUTH_INSIDE")->second.clone();
 
     cv::resize(mouth_mask, mouth, mat_image_.size(), cv::INTER_LINEAR);
 
-    std::vector<int> x, y;
+    cv::Rect rect = cv::boundingRect(mouth);
-    std::vector<cv::Point> location;
 
-    cv::findNonZero(mouth, location);
+    if (!rect.empty())
-
-    for (auto &i : location)
     {
-      x.push_back(i.x);
+      double mouth_min_y = rect.y, mouth_max_y = rect.y + rect.height,
-      y.push_back(i.y);
+             mouth_max_x = rect.x + rect.width, mouth_min_x = rect.x;
-    }
 
-    if (!(x.empty()) && !(y.empty()))
-    {
-      double mouth_min_y, mouth_max_y, mouth_max_x, mouth_min_x;
-      cv::minMaxLoc(y, &mouth_min_y, &mouth_max_y);
-      cv::minMaxLoc(x, &mouth_min_x, &mouth_max_x);
       double mh = mouth_max_y - mouth_min_y;
       double mw = mouth_max_x - mouth_min_x;
-      cv::Mat mouth_crop_mask;
+
       mouth.convertTo(mouth, CV_32F, 1.0 / 255);
       mouth.convertTo(mouth, CV_32F, 1.0 / 255);
+
       if (mh / mw > 0.17)
       {
         mouth_min_y = static_cast<int>(std::max(mouth_min_y - mh * 0.1, 0.0));
         mouth_max_y = static_cast<int>(std::min(mouth_max_y + mh * 0.1, (double)image_height_));
         mouth_min_x = static_cast<int>(std::max(mouth_min_x - mw * 0.1, 0.0));
         mouth_max_x = static_cast<int>(std::min(mouth_max_x + mw * 0.1, (double)image_width_));
-        mouth_crop_mask = mouth(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x));
+        cv::Mat mouth_crop_mask = mouth(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x));
         cv::Mat img_hsv, tmp_mask, img_hls;
         cv::cvtColor(mat_image_(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)), img_hsv,
                      cv::COLOR_RGBA2RGB);

mediapipe/calculators/core/BUILD

@@ -194,6 +194,7 @@ cc_library(
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:opencv_core",		
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -16,6 +16,7 @@
 
 #include <vector>
 
+#include "mediapipe/framework/port/opencv_core_inc.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/matrix.h"
@@ -46,6 +47,9 @@ REGISTER_CALCULATOR(BeginLoopDetectionCalculator);
 typedef BeginLoopCalculator<std::vector<Matrix>> BeginLoopMatrixCalculator;
 REGISTER_CALCULATOR(BeginLoopMatrixCalculator);
 
+typedef BeginLoopCalculator<std::vector<std::vector<cv::Point3d>>> BeginLoopPointsCalculator;
+REGISTER_CALCULATOR(BeginLoopPointsCalculator);
+
 // A calculator to process std::vector<std::vector<Matrix>>.
 typedef BeginLoopCalculator<std::vector<std::vector<Matrix>>>
     BeginLoopMatrixVectorCalculator;

mediapipe/calculators/core/end_loop_calculator.cc

@@ -57,6 +57,10 @@ typedef EndLoopCalculator<std::vector<std::tuple<double, double, double, double>
     EndLoopFaceBoxCalculator;
 REGISTER_CALCULATOR(EndLoopFaceBoxCalculator);
 
+typedef EndLoopCalculator<std::vector<std::vector<cv::Point3d>>>
+    EndLoopPointsCalculator;
+REGISTER_CALCULATOR(EndLoopPointsCalculator);
+
 typedef EndLoopCalculator<std::vector<TfLiteTensor>> EndLoopTensorCalculator;
 REGISTER_CALCULATOR(EndLoopTensorCalculator);
 

mediapipe/calculators/image_style/fast_utils_calculator.cc

@@ -329,7 +329,7 @@ namespace mediapipe
     orig_width = size.first;
     orig_height = size.second;
     CHECK_GT(size.first, 0);
-    CHECK_GT(orig_height, 0);
+    CHECK_GT(size.second, 0);
 
     MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
     mat_image_ = *image_mat.get();

mediapipe/calculators/landmarks/BUILD

@@ -40,3 +40,54 @@ cc_library(
     ],
     alwayslink = 1,
 )
+
+cc_library(
+    name = "points_to_face_box_calculator",
+    srcs = ["points_to_face_box_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+   "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:opencv_highgui",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "landmarks_to_point_array_calculator",
+    srcs = ["landmarks_to_point_array_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+   "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:opencv_highgui",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "point_vector_to_mask_calculator",
+    srcs = ["point_vector_to_mask_calculator.cc"],
+    hdrs = ["point_vector_to_mask_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+   "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/api2:node",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:opencv_highgui",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+    ],
+    alwayslink = 1,
+)

mediapipe/calculators/landmarks/forehead_mask_calculator.cc

@@ -0,0 +1,149 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/landmarks/point_vector_to_mask_calculator.h"
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+using namespace std;
+using namespace cv;
+
+namespace mediapipe
+{
+    namespace api2
+    {
+        class PointVectorToMaskCalculator : public Node
+        {
+        public:
+            static constexpr Input<Mat> kImage{"IMAGE"};
+            static constexpr Input<unordered_map<string, Mat>> kMasks{"MASKS"};
+            static constexpr Input<tuple<double, double, double, double>> kFaceBox{"FACE_BOX"};
+            static constexpr Output<Mat> kOut{"FOREHEAD_MASK"};
+
+            MEDIAPIPE_NODE_CONTRACT(kImage, kMasks, kFaceBox, kOut);
+
+            static absl::Status UpdateContract(CalculatorContract *cc)
+            {
+                RET_CHECK(kOut(cc).IsConnected())
+                    << "At least one output stream is expected.";
+                return absl::OkStatus();
+            }
+
+            absl::Status Process(CalculatorContext *cc) override
+            {
+                if (kImage(cc).IsEmpty() || kMasks(cc).IsEmpty() || kFaceBox(cc).IsEmpty())
+                {
+                    return absl::OkStatus();
+                }
+
+                const auto &mat_image_ = *kImage(cc);
+                const auto &masks = *kMasks(cc);
+                const auto &face_box = *kFaceBox(cc);
+
+                MP_RETURN_IF_ERROR(PredictForeheadMask(cc, mat_image_, masks, face_box));
+
+                auto output_frame = absl::make_unique<Mat>(new_skin_mask);
+
+                kOut(cc).Send(std::move(output_frame));
+
+                return absl::OkStatus();
+            }
+
+            absl::Status PredictForeheadMask(CalculatorContext *cc, Mat mat_image_, 
+            unordered_map<string, Mat> mask_vec, tuple<double, double, double, double> face_box)
+            {
+
+                cv::Mat part_forehead_mask = mask_vec.find("PART_FOREHEAD_B")->second.clone();
+                part_forehead_mask.convertTo(part_forehead_mask, CV_32F, 1.0 / 255);
+                part_forehead_mask.convertTo(part_forehead_mask, CV_8U);
+
+                cv::Mat image_sm, image_sm_hsv, skinMask;
+
+                cv::resize(mat_image_, image_sm, cv::Size(mat_image_.cols, mat_image_.rows));
+                cv::cvtColor(image_sm, image_sm_hsv, cv::COLOR_BGR2HSV);
+
+                std::vector<int> x, y;
+                std::vector<cv::Point> location;
+
+                cv::Vec3d hsv_min, hsv_max;
+
+                std::vector<cv::Mat> channels(3);
+                cv::split(image_sm_hsv, channels);
+                std::vector<std::vector<double>> minx(3), maxx(3);
+                int c = 0;
+                for (auto ch : channels)
+                {
+                    cv::Mat row, mask_row;
+                    double min, max;
+                    for (int i = 0; i < ch.rows; i++)
+                    {
+                        row = ch.row(i);
+                        mask_row = part_forehead_mask.row(i);
+                        cv::minMaxLoc(row, &min, &max, 0, 0, mask_row);
+                        minx[c].push_back(min);
+                        maxx[c].push_back(max);
+                    }
+                    c++;
+                }
+                for (int i = 0; i < 3; i++)
+                {
+                    hsv_min[i] = *std::min_element(minx[i].begin(), minx[i].end());
+                }
+                for (int i = 0; i < 3; i++)
+                {
+                    hsv_max[i] = *std::max_element(maxx[i].begin(), maxx[i].end());
+                }
+
+                cv::Mat _forehead_kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(1, 1));
+                cv::inRange(image_sm_hsv, hsv_min, hsv_max, skinMask);
+                cv::erode(skinMask, skinMask, _forehead_kernel, cv::Point(-1, -1), 2);
+                cv::dilate(skinMask, skinMask, _forehead_kernel, cv::Point(-1, -1), 2);
+                skinMask.convertTo(skinMask, CV_8U, 1.0 / 255);
+
+                cv::findNonZero(skinMask, location);
+
+                double max_part_f, x_min_part, x_max_part;
+
+                for (auto &i : location)
+                {
+                    x.push_back(i.x);
+                    y.push_back(i.y);
+                }
+
+                cv::minMaxLoc(y, NULL, &max_part_f);
+                cv::minMaxLoc(x, &x_min_part, &x_max_part);
+
+                cv::Mat new_skin_mask = cv::Mat::zeros(skinMask.size(), CV_8U);
+
+                new_skin_mask(cv::Range(get<1>(face_box), max_part_f), cv::Range(x_min_part, x_max_part)) =
+                    skinMask(cv::Range(get<1>(face_box), max_part_f), cv::Range(x_min_part, x_max_part));
+
+                return absl::OkStatus();
+            }
+
+        private:
+            Mat new_skin_mask;
+        };
+        MEDIAPIPE_REGISTER_NODE(PointVectorToMaskCalculator);
+    }
+} // namespace mediapipe

mediapipe/calculators/landmarks/landmarks_to_mask_calculator.cc

@@ -13,17 +13,12 @@
 // limitations under the License.
 #include "mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h"
 
-#include <math.h>
-
-#include <algorithm>
-#include <cmath>
-#include <iostream>
-
 #include "absl/memory/memory.h"
 #include "mediapipe/framework/port/logging.h"
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/vector.h"
 
+using namespace std;
 namespace mediapipe
 {
   namespace
@@ -33,20 +28,7 @@ namespace mediapipe
     constexpr char kVectorTag[] = "VECTOR";
     constexpr char kMaskTag[] = "MASK";
     constexpr char kFaceBoxTag[] = "FACEBOX";
-    constexpr char kImageFrameTag[] = "IMAGE";
+    constexpr char kImageSizeTag[] = "SIZE";
-
-    std::unordered_map<std::string, const std::vector<int>> orderList = {
-        {"UPPER_LIP", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78}},
-        {"LOWER_LIP", {61, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
-        {"FACE_OVAL", {10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356, 454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378, 378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150, 136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162, 21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10}},
-        {"MOUTH_INSIDE", {78, 191, 80, 81, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95}},
-        {"LEFT_EYE", {130, 33, 246, 161, 160, 159, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7}},
-        {"RIGHT_EYE", {362, 398, 384, 385, 386, 387, 388, 466, 263, 249, 390, 373, 374, 380, 381, 382}},
-        {"LEFT_BROW", {70, 63, 105, 66, 107, 55, 65, 52, 53, 46}},
-        {"RIGHT_BROW", {336, 296, 334, 293, 301, 300, 283, 282, 295, 285}},
-        {"LIPS", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
-        {"PART_FOREHEAD_B", {21, 54, 103, 67, 109, 10, 338, 297, 332, 284, 251, 301, 293, 334, 296, 336, 9, 107, 66, 105, 63, 71}},
-    };
 
     template <class LandmarkType>
     bool IsLandmarkVisibleAndPresent(const LandmarkType &landmark,
@@ -91,15 +73,10 @@ namespace mediapipe
 
     std::tuple<double, double, double, double> face_box;
 
-    std::unique_ptr<cv::Mat> image_mat;
-
     int image_width_;
     int image_height_;
 
     float scale_factor_ = 1.0;
-
-    bool image_frame_available_ = false;
-
   } // namespace
 
   absl::Status LandmarksToMaskCalculator::GetContract(
@@ -113,9 +90,9 @@ namespace mediapipe
         << "Can only one type of landmark can be taken. Either absolute or "
            "normalized landmarks.";
 
-    if (cc->Inputs().HasTag(kImageFrameTag))
+    if (cc->Inputs().HasTag(kImageSizeTag))
     {
-      cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
+      cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
     }
 
     if (cc->Inputs().HasTag(kLandmarksTag))
@@ -142,18 +119,11 @@ namespace mediapipe
   {
     cc->SetOffset(TimestampDiff(0));
 
-    if (cc->Inputs().HasTag(kImageFrameTag))
-    {
-      image_frame_available_ = true;
-    }
-
     return absl::OkStatus();
   }
 
   absl::Status LandmarksToMaskCalculator::Process(CalculatorContext *cc)
   {
-    // Check that landmarks are not empty and skip rendering if so.
-    // Don't emit an empty packet for this timestamp.
     if (cc->Inputs().HasTag(kLandmarksTag) &&
         cc->Inputs().Tag(kLandmarksTag).IsEmpty())
     {
@@ -164,27 +134,17 @@ namespace mediapipe
     {
       return absl::OkStatus();
     }
-    if (cc->Inputs().HasTag(kImageFrameTag) &&
-        cc->Inputs().Tag(kImageFrameTag).IsEmpty())
-    {
-      return absl::OkStatus();
-    }
 
-    // Initialize render target, drawn with OpenCV.
-
-    ImageFormat::Format target_format;
     std::unordered_map<std::string, cv::Mat> all_masks;
 
-    MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
+    const auto size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
-
+    image_width_ = size.first;
-    cv::Mat mat_image_ = *image_mat.get();
+    image_height_ = size.second;
-    image_width_ = image_mat->cols;
+    CHECK_GT(image_width_, 0);
-    image_height_ = image_mat->rows;
+    CHECK_GT(image_height_, 0);
 
     MP_RETURN_IF_ERROR(GetMasks(cc, all_masks));
 
-    MP_RETURN_IF_ERROR(GetFaceBox(cc));
-
     MP_RETURN_IF_ERROR(RenderToCpu(cc, all_masks));
 
     return absl::OkStatus();
@@ -216,116 +176,23 @@ namespace mediapipe
     return absl::OkStatus();
   }
 
-  absl::Status LandmarksToMaskCalculator::CreateRenderTargetCpu(
-      CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
-      ImageFormat::Format *target_format)
-  {
-    if (image_frame_available_)
-    {
-      const auto &input_frame =
-          cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
-
-      int target_mat_type;
-      switch (input_frame.Format())
-      {
-      case ImageFormat::SRGBA:
-        *target_format = ImageFormat::SRGBA;
-        target_mat_type = CV_8UC4;
-        break;
-      case ImageFormat::SRGB:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      case ImageFormat::GRAY8:
-        *target_format = ImageFormat::SRGB;
-        target_mat_type = CV_8UC3;
-        break;
-      default:
-        return absl::UnknownError("Unexpected image frame format.");
-        break;
-      }
-
-      image_mat = absl::make_unique<cv::Mat>(
-          input_frame.Height(), input_frame.Width(), target_mat_type);
-
-      auto input_mat = formats::MatView(&input_frame);
-
-      if (input_frame.Format() == ImageFormat::GRAY8)
-      {
-        cv::Mat rgb_mat;
-        cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
-        rgb_mat.copyTo(*image_mat);
-      }
-      else
-      {
-        input_mat.copyTo(*image_mat);
-      }
-    }
-    else
-    {
-      image_mat = absl::make_unique<cv::Mat>(
-          150, 150, CV_8UC4,
-          cv::Scalar(255, 255,
-                     255));
-      *target_format = ImageFormat::SRGBA;
-    }
-
-    return absl::OkStatus();
-  }
-
   absl::Status LandmarksToMaskCalculator::GetMasks(CalculatorContext *cc,
                                                    std::unordered_map<std::string, cv::Mat> &all_masks)
   {
-    if (cc->Inputs().HasTag(kLandmarksTag))
-    {
-       const LandmarkList &landmarks =
-          cc->Inputs().Tag(kNormLandmarksTag).Get<LandmarkList>();
-
-      cv::Mat mask;
-      std::vector<cv::Point> point_array;
-      for (const auto &[key, value] : orderList)
-      {
-        for (auto order : value)
-        {
-          const Landmark &landmark = landmarks.landmark(order);
-
-          if (!IsLandmarkVisibleAndPresent<Landmark>(
-                  landmark, false,
-                  0.0, false,
-                  0.0))
-          {
-            continue;
-          }
-
-          const auto &point = landmark;
-          int x = -1;
-          int y = -1;
-          CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_,
-                                             image_height_, &x, &y));
-          point_array.push_back(cv::Point(x, y));
-        }
-
-        std::vector<std::vector<cv::Point>> point_vec;
-        point_vec.push_back(point_array);
-       
-        mask = cv::Mat::zeros(image_mat->size(), CV_32FC1);
-        cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA);
-        mask.convertTo(mask, CV_8U);
-        all_masks.insert({key, mask});
-        point_vec.clear();
-        point_array.clear();
-      }
-    }
-
     if (cc->Inputs().HasTag(kNormLandmarksTag))
     {
+      std::vector<int> x_s, y_s;
+      double box_min_y, box_max_y, box_max_x, box_min_x;
       const NormalizedLandmarkList &landmarks =
           cc->Inputs().Tag(kNormLandmarksTag).Get<NormalizedLandmarkList>();
 
       cv::Mat mask;
       std::vector<cv::Point> point_array;
+      std::vector<std::vector<cv::Point>> point_vec;
       for (const auto &[key, value] : orderList)
       {
+        point_vec = {};
+        point_array = {};
         for (auto order : value)
         {
           const NormalizedLandmark &landmark = landmarks.landmark(order);
@@ -345,88 +212,25 @@ namespace mediapipe
                                              image_height_, &x, &y));
           point_array.push_back(cv::Point(x, y));
         }
+        cv::Mat po(point_array);
+        po.convertTo(po, CV_32F);
+        cv::Mat min, max;
+
+        cv::reduce(po, min, 0, CV_REDUCE_MIN, CV_32F);
+        cv::reduce(po, max, 0, CV_REDUCE_MAX, CV_32F);
+
+        min.at<float>(0,1)*=0.9;
+        face_box = {min.at<float>(0,0), min.at<float>(0,1), max.at<float>(0,0), max.at<float>(0,1)};
 
-        std::vector<std::vector<cv::Point>> point_vec;
         point_vec.push_back(point_array);
-        mask = cv::Mat::zeros(image_mat->size(), CV_32FC1);
+        mask = cv::Mat::zeros({image_width_, image_height_}, CV_32FC1);
         cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA);
         mask.convertTo(mask, CV_8U);
-        all_masks.insert(make_pair(key, mask));
+        all_masks.insert({key, mask});
-        point_vec.clear();
-        point_array.clear();
       }
     }
     return absl::OkStatus();
   }
 
-  absl::Status LandmarksToMaskCalculator::GetFaceBox(CalculatorContext *cc)
-  {
-    std::vector<int> x_s, y_s;
-    double box_min_y, box_max_y, box_max_x, box_min_x;
-    if (cc->Inputs().HasTag(kLandmarksTag))
-    {
-      const LandmarkList &landmarks =
-          cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
-
-      for (int i = 0; i < landmarks.landmark_size(); ++i)
-      {
-        const Landmark &landmark = landmarks.landmark(i);
-
-        if (!IsLandmarkVisibleAndPresent<Landmark>(
-                landmark, false,
-                0.0, false,
-                0.0))
-        {
-          continue;
-        }
-
-        const auto &point = landmark;
-        int x = -1;
-        int y = -1;
-        CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_,
-                                           image_height_, &x, &y));
-        x_s.push_back(point.x());
-        x_s.push_back(point.y());
-      }
-      cv::minMaxLoc(y_s, &box_min_y, &box_max_y);
-      cv::minMaxLoc(x_s, &box_min_x, &box_max_x);
-      box_min_y = box_min_y * 0.9;
-      face_box = std::make_tuple(box_min_x, box_min_y, box_max_x, box_max_y);
-    }
-
-    if (cc->Inputs().HasTag(kNormLandmarksTag))
-    {
-      const NormalizedLandmarkList &landmarks =
-          cc->Inputs().Tag(kNormLandmarksTag).Get<NormalizedLandmarkList>();
-
-      for (int i = 0; i < landmarks.landmark_size(); ++i)
-      {
-        const NormalizedLandmark &landmark = landmarks.landmark(i);
-
-        if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
-                landmark, false,
-                0.0, false,
-                0.0))
-        {
-          continue;
-        }
-
-        const auto &point = landmark;
-        int x = -1;
-        int y = -1;
-        CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_,
-                                           image_height_, &x, &y));
-        x_s.push_back(point.x());
-        x_s.push_back(point.y());
-      }
-      cv::minMaxLoc(y_s, &box_min_y, &box_max_y);
-      cv::minMaxLoc(x_s, &box_min_x, &box_max_x);
-      box_min_y = box_min_y * 0.9;
-      face_box = std::make_tuple(box_min_x, box_min_y, box_max_x, box_max_y);
-    }
-
-    return absl::OkStatus();
-  }
-
   REGISTER_CALCULATOR(LandmarksToMaskCalculator);
 } // namespace mediapipe

mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h

@@ -41,6 +41,19 @@ namespace mediapipe
     //   output_stream: "MASK:mask"
     // }
 
+    std::unordered_map<std::string, const std::vector<int>> orderList = {
+        {"UPPER_LIP", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78}},
+        {"LOWER_LIP", {61, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
+        {"FACE_OVAL", {10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356, 454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378, 378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150, 136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162, 21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10}},
+        {"MOUTH_INSIDE", {78, 191, 80, 81, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95}},
+        {"LEFT_EYE", {130, 33, 246, 161, 160, 159, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7}},
+        {"RIGHT_EYE", {362, 398, 384, 385, 386, 387, 388, 466, 263, 249, 390, 373, 374, 380, 381, 382}},
+        {"LEFT_BROW", {70, 63, 105, 66, 107, 55, 65, 52, 53, 46}},
+        {"RIGHT_BROW", {336, 296, 334, 293, 301, 300, 283, 282, 295, 285}},
+        {"LIPS", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
+        {"PART_FOREHEAD_B", {21, 54, 103, 67, 109, 10, 338, 297, 332, 284, 251, 301, 293, 334, 296, 336, 9, 107, 66, 105, 63, 71}},
+    };
+
     class LandmarksToMaskCalculator : public CalculatorBase
     {
     public:

mediapipe/calculators/landmarks/landmarks_to_point_array_calculator.cc

@@ -0,0 +1,152 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+using namespace std;
+using namespace cv;
+
+namespace mediapipe
+{
+    namespace
+    {
+        template <class LandmarkType>
+        bool IsLandmarkVisibleAndPresent(const LandmarkType &landmark,
+                                         bool utilize_visibility,
+                                         float visibility_threshold,
+                                         bool utilize_presence,
+                                         float presence_threshold)
+        {
+            if (utilize_visibility && landmark.has_visibility() &&
+                landmark.visibility() < visibility_threshold)
+            {
+                return false;
+            }
+            if (utilize_presence && landmark.has_presence() &&
+                landmark.presence() < presence_threshold)
+            {
+                return false;
+            }
+            return true;
+        }
+
+        bool NormalizedtoPixelCoordinates(double normalized_x, double normalized_y, double normalized_z,
+                                          int image_width, int image_height, double *x_px,
+                                          double *y_px, double *z_px)
+        {
+            CHECK(x_px != nullptr);
+            CHECK(y_px != nullptr);
+            CHECK_GT(image_width, 0);
+            CHECK_GT(image_height, 0);
+
+            if (normalized_x < 0 || normalized_x > 1.0 || normalized_y < 0 ||
+                normalized_y > 1.0 || normalized_z < 0 ||
+                normalized_z > 1.0)
+            {
+                VLOG(1) << "Normalized coordinates must be between 0.0 and 1.0";
+            }
+
+            *x_px = static_cast<double>(normalized_x) * image_width;
+            *y_px = static_cast<double>(normalized_y) * image_height;
+            *z_px = static_cast<double>(normalized_z) * image_width;
+
+            return true;
+        }
+    }
+    namespace api2
+    {
+        class LandmarksToPointArrayCalculator : public Node
+        {
+        public:
+            static constexpr Input<NormalizedLandmarkList> kNormLandmarks{"NORM_LANDMARKS"};
+            static constexpr Input<pair<int, int>> kImageSize{"IMAGE_SIZE"};
+            static constexpr Output<vector<Point3d>> kOut{"POINTS"};
+
+            MEDIAPIPE_NODE_CONTRACT(kNormLandmarks, kImageSize, kOut);
+
+            static absl::Status UpdateContract(CalculatorContract *cc)
+            {
+                RET_CHECK(kOut(cc).IsConnected())
+                    << "At least one output stream is expected.";
+                return absl::OkStatus();
+            }
+
+            absl::Status Process(CalculatorContext *cc) override
+            {
+                if (kNormLandmarks(cc).IsEmpty())
+                {
+                    return absl::OkStatus();
+                }
+                if (kImageSize(cc).IsEmpty())
+                {
+                    return absl::OkStatus();
+                }
+
+                const auto &size = *kImageSize(cc);
+
+                MP_RETURN_IF_ERROR(GetPoints(cc, size));
+
+                auto output_frame = absl::make_unique<vector<Point3d>>(point_array);
+
+                kOut(cc).Send(std::move(output_frame));
+                return absl::OkStatus();
+            }
+
+            absl::Status GetPoints(CalculatorContext *cc, pair<int, int> size)
+            {
+
+                const auto &landmarks = *kNormLandmarks(cc);
+                point_array = {};
+                for (int i = 0; i < landmarks.landmark_size(); i++)
+                {
+                    const NormalizedLandmark &landmark = landmarks.landmark(i);
+
+                    if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
+                            landmark, false,
+                            0.0, false,
+                            0.0))
+                    {
+                        continue;
+                    }
+
+                    const auto &point = landmark;
+
+                    double x = -1;
+                    double y = -1;
+                    double z = -1;
+
+                    CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), point.z(), size.first,
+                                                       size.second, &x, &y, &z));
+
+                    point_array.push_back({x, y, z});
+                }
+
+                return absl::OkStatus();
+            }
+
+        private:
+            std::vector<cv::Point3d> point_array;
+        };
+        MEDIAPIPE_REGISTER_NODE(LandmarksToPointArrayCalculator);
+    }
+} // namespace mediapipe

mediapipe/calculators/landmarks/point_vector_to_mask_calculator.cc

@@ -0,0 +1,91 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/landmarks/point_vector_to_mask_calculator.h"
+
+using namespace std;
+using namespace cv;
+
+namespace mediapipe
+{
+    namespace api2
+    {
+        class PointVectorToMaskCalculator : public Node
+        {
+        public:
+            static constexpr Input<vector<Point3d>> kPoints{"POINTS"};
+            static constexpr Input<pair<int, int>> kImageSize{"IMAGE_SIZE"};
+            static constexpr Output<std::unordered_map<std::string, cv::Mat>> kOut{"MASKS"};
+
+            MEDIAPIPE_NODE_CONTRACT(kPoints, kImageSize, kOut);
+
+            static absl::Status UpdateContract(CalculatorContract *cc)
+            {
+                RET_CHECK(kOut(cc).IsConnected())
+                    << "At least one output stream is expected.";
+                return absl::OkStatus();
+            }
+
+            absl::Status Process(CalculatorContext *cc) override
+            {
+                if (kPoints(cc).IsEmpty())
+                {
+                    return absl::OkStatus();
+                }
+                if (kImageSize(cc).IsEmpty())
+                {
+                    return absl::OkStatus();
+                }
+
+                const auto &size = *kImageSize(cc);
+                const auto &points = *kPoints(cc);
+
+                MP_RETURN_IF_ERROR(GetMasks(cc, size, points));
+
+                auto output_frame = absl::make_unique<std::unordered_map<std::string, cv::Mat>>(all_masks);
+
+                kOut(cc).Send(std::move(output_frame), cc->InputTimestamp());
+
+                return absl::OkStatus();
+            }
+
+            absl::Status GetMasks(CalculatorContext *cc, pair<int, int> size, vector<Point3d> points)
+            {
+                all_masks= {};
+
+                for (const auto &[key, value] : orderList)
+                {
+                    vector<vector<Point>> point_vec = {};
+                    vector<Point> point_array = {};
+                    for (auto order : value)
+                    {
+                        point_array.push_back({points[order].x, points[order].y});
+                    }
+                    point_vec.push_back(point_array);
+                    
+                    cv::Mat mask = cv::Mat::zeros({size.first, size.second}, CV_32FC1);
+                    cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA);
+                    mask.convertTo(mask, CV_8U);
+
+                    all_masks.insert({key, mask});
+                }
+                return absl::OkStatus();
+            }
+
+        private:
+            std::unordered_map<std::string, cv::Mat> all_masks;
+        };
+        MEDIAPIPE_REGISTER_NODE(PointVectorToMaskCalculator);
+    }
+} // namespace mediapipe

mediapipe/calculators/landmarks/point_vector_to_mask_calculator.h

@@ -0,0 +1,81 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <unordered_map>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe
+{
+
+    // A calculator that uses face landmarks to create face part masks and facebox for
+    // visualization. The input should be LandmarkList proto and ImageFrame.
+    //
+    // Example config:
+    // node {
+    //   calculator: "LandmarksToMaskCalculator"
+    //   input_stream: "IMAGE:image"
+    //   input_stream: "NORM_LANDMARKS:face_landmarks"
+    //   output_stream: "FACEBOX:face_box"
+    //   output_stream: "MASK:mask"
+    // }
+
+    std::unordered_map<std::string, const std::vector<int>> orderList = {
+        {"UPPER_LIP", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78}},
+        {"LOWER_LIP", {61, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
+        {"FACE_OVAL", {10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356, 454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378, 378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150, 136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162, 21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10}},
+        {"MOUTH_INSIDE", {78, 191, 80, 81, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95}},
+        {"LEFT_EYE", {33, 246, 161, 160, 159, 158, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7}},
+        {"RIGHT_EYE", {362, 398, 384, 385, 386, 387, 388, 466, 263, 249, 390, 373, 374, 380, 381, 382}},
+        {"LEFT_IRIS", {468, 469, 470, 471, 472}},
+        {"RIGHT_IRIS", {473, 474, 475, 476, 477}},
+        {"LEFT_BROW", {468, 469, 470, 471, 472}},
+        {"RIGHT_BROW", {336, 296, 334, 293, 301, 300, 283, 282, 295, 285}},
+        {"LIPS", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
+        {"PART_FOREHEAD_B", {
+                                21,
+                                54,
+                                103,
+                                67,
+                                109,
+                                10,
+                                338,
+                                297,
+                                332,
+                                284,
+                                251,
+                                301,
+                                293,
+                                334,
+                                296,
+                                336,
+                                9,
+                                107,
+                                66,
+                                105,
+                                63,
+                                71,
+                            }},
+    };
+
+} // namespace mediapipe

mediapipe/calculators/landmarks/points_to_face_box_calculator.cc

@@ -0,0 +1,93 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
+#include "mediapipe/framework/port/status.h"
+
+using namespace std;
+using namespace cv;
+
+namespace mediapipe
+{
+    namespace api2
+    {
+        class PointsToFaceBoxCalculator : public Node
+        {
+        public:
+            static constexpr Input<vector<Point3d>> kPoints{"POINTS"};
+            static constexpr Input<pair<int, int>> kImageSize{"IMAGE_SIZE"};
+            static constexpr Output<tuple<double, double, double, double>> kOut{"FACE_BOX"};
+
+            MEDIAPIPE_NODE_CONTRACT(kPoints, kImageSize, kOut);
+
+            static absl::Status UpdateContract(CalculatorContract *cc)
+            {
+                RET_CHECK(kOut(cc).IsConnected())
+                    << "At least one output stream is expected.";
+                return absl::OkStatus();
+            }
+
+            absl::Status Process(CalculatorContext *cc) override
+            {
+                if (kPoints(cc).IsEmpty())
+                {
+                    return absl::OkStatus();
+                }
+                if (kImageSize(cc).IsEmpty())
+                {
+                    return absl::OkStatus();
+                }
+
+                const auto &size = *kImageSize(cc);
+                const auto &points = *kPoints(cc);
+
+                MP_RETURN_IF_ERROR(GetFaceBox(cc, size, points));
+
+                auto output_frame = absl::make_unique<tuple<double, double, double, double>>(face_box);
+
+                kOut(cc).Send(std::move(output_frame));
+
+                return absl::OkStatus();
+            }
+
+            absl::Status GetFaceBox(CalculatorContext *cc, pair<int, int> size, vector<Point3d> points)
+            {
+                cv::Mat points_mat(points);
+
+                points_mat.convertTo(points_mat, CV_32F);
+                cv::Mat min, max;
+
+                cv::reduce(points_mat, min, 0, CV_REDUCE_MIN, CV_32F);
+                cv::reduce(points_mat, max, 0, CV_REDUCE_MAX, CV_32F);
+
+                min.at<float>(0, 1) *= 0.9;
+                face_box = {min.at<float>(0, 0), min.at<float>(0, 1), max.at<float>(0, 0), max.at<float>(0, 1)};
+
+                return absl::OkStatus();
+            }
+
+        private:
+            tuple<double, double, double, double> face_box;
+        };
+        MEDIAPIPE_REGISTER_NODE(PointsToFaceBoxCalculator);
+    }
+} // namespace mediapipe

mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/AndroidManifest.xml

@@ -12,6 +12,7 @@
 
   <!-- For profiling -->
   <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+  <uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" />
   
   <application
       android:allowBackup="true"

mediapipe/examples/android/src/java/com/google/mediapipe/apps/deformation/BUILD

@@ -55,6 +55,7 @@ android_binary(
         ASSETS_DIR+":upperLipCnt.txt",
         ASSETS_DIR+":widerLowerLipPts1.txt",
         ASSETS_DIR+":widerUpperLipPts1.txt",      
+        "//mediapipe/graphs/deformation/calculators:Tensor.h"  
     ],
     assets_dir = "",
     manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml",

mediapipe/examples/desktop/beauty/BUILD

@@ -48,6 +48,12 @@ cc_binary(
     ],
 )
 
+
+
+
+
+
+
 cc_binary(
     name = "beauty_gpu_over",
     deps = [

mediapipe/examples/desktop/makeup/BUILD

@@ -0,0 +1,34 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_binary(
+    name = "makeup_cpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/makeup:desktop_live_calculators",
+    ],
+)
+
+cc_binary(
+    name = "makeup_mobile",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/makeup:mobile_calculators",
+    ],
+)
+

mediapipe/graphs/beauty/README.md

@@ -19,10 +19,7 @@ bazel-bin/mediapipe/examples/desktop/beauty/beauty_cpu
 ```
 Run with (using video):
 ```
-bazel-bin/mediapipe/examples/desktop/beauty/beauty_cpu 
+bazel-bin/mediapipe/examples/desktop/beauty/beauty_cpu --calculator_graph_config_file=mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt --input_video_path=/path/video.mp4 --output_video_path=/path/outvideo.mp4
---calculator_graph_config_file=mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt 
---input_video_path=/path/video.mp4 
---output_video_path=/path/outvideo.mp4
 ```
 
 2. Desktop-CPU-Single (Not divided, using render data)

mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt

@@ -37,6 +37,12 @@ node {
     tag_index: "FINISHED"
     back_edge: true
   }
+  node_options: {
+    [type.googleapis.com/mediapipe.FlowLimiterCalculatorOptions]: {
+	max_in_flight: 3
+	max_in_queue: 3
+    }
+  }
   output_stream: "throttled_input_video"
 }
 

mediapipe/graphs/beauty/beauty_desktop_cpu_single.pbtxt

@@ -14,7 +14,7 @@ profiler_config {
   trace_enabled: true
   enable_profiler: true
   trace_log_interval_count: 200
-  trace_log_path: "/Users/alena/Workdir/mediapipe/logs/beauty/"
+  trace_log_path: "/home/mslight/Work/clone/mediapipe/mediapipe/logs/beauty_single/"
 }
 
 node {

mediapipe/graphs/beauty/beauty_mobile_gpu.pbtxt

@@ -15,7 +15,7 @@ output_stream: "multi_face_landmarks"
 profiler_config {
   trace_enabled: true
   enable_profiler: true
-  trace_log_interval_count: 200
+  trace_log_path: "/storage/emulated/0/Download/profiles/"
 }
 
 # Throttles the images flowing downstream for flow control. It passes through

mediapipe/graphs/beauty/subgraphs/BUILD

@@ -24,16 +24,22 @@ package(default_visibility = ["//visibility:public"])
 cc_library(
     name = "renderer_calculators",
     deps = [
-        "//mediapipe/calculators/core:split_proto_list_calculator",
-        "//mediapipe/util:annotation_renderer",
-        "//mediapipe/calculators/util:annotation_overlay_calculator",
         "//mediapipe/calculators/beauty:smooth_face_calculator1",
         "//mediapipe/calculators/beauty:bilateral_filter_calculator",
+        "//mediapipe/calculators/beauty:merging_images_calculator",
         "//mediapipe/calculators/beauty:smooth_face_calculator2",
         "//mediapipe/calculators/beauty:draw_lipstick_calculator",
         "//mediapipe/calculators/beauty:whiten_teeth_calculator",
-        "//mediapipe/calculators/util:landmarks_to_render_data_calculator",
+        "//mediapipe/calculators/beauty:imageframe_to_mat_calculator",
         "//mediapipe/calculators/landmarks:landmarks_to_mask_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/image:color_convert_calculator",
+        "//mediapipe/calculators/core:round_robin_demux_calculator",
+        "//mediapipe/calculators/core:mux_calculator",        
+        "//mediapipe/calculators/util:landmarks_to_render_data_calculator",
+         "//mediapipe/util:annotation_renderer",
+         "//mediapipe/calculators/util:annotation_overlay_calculator",
+         "//mediapipe/calculators/core:split_proto_list_calculator",
            "//mediapipe/graphs/face_mesh/calculators:face_landmarks_to_render_data_calculator",      
     ],
 )
@@ -66,3 +72,4 @@ mediapipe_simple_subgraph(
         ":renderer_calculators",
     ],
 )
+

mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt

@@ -11,6 +11,12 @@ input_stream: "LANDMARKS:multi_face_landmarks"
 # CPU image with rendered data. (ImageFrame)
 output_stream: "IMAGE:output_image"
 
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:input_image"
+  output_stream: "SIZE:size"
+}
+
 # Outputs each element of multi_face_landmarks at a fake timestamp for the rest
 # of the graph to process. At the end of the loop, outputs the BATCH_END
 # timestamp for downstream calculators to inform them that all elements in the
@@ -18,16 +24,16 @@ output_stream: "IMAGE:output_image"
 node {
   calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
   input_stream: "ITERABLE:multi_face_landmarks"
-  input_stream: "CLONE:input_image"
+  input_stream: "CLONE:size"
   output_stream: "ITEM:face_landmarks"
-  output_stream: "CLONE:loop_image"
+  output_stream: "CLONE:cloned_size"
   output_stream: "BATCH_END:landmark_timestamp"
 }
 
 # Converts landmarks to face part masks.
 node {
   calculator: "LandmarksToMaskCalculator"
-  input_stream: "IMAGE:loop_image"
+  input_stream: "SIZE:cloned_size"
   input_stream: "NORM_LANDMARKS:face_landmarks"
   output_stream: "FACEBOX:face_box"
   output_stream: "MASK:mask"
@@ -50,47 +56,88 @@ node {
   output_stream: "ITERABLE:multi_face_box"
 }
 
-#Applies lipstick to the face on the IMAGE using MASK.
+node {
+  calculator: "ImageFrameToMatCalculator"
+  input_stream: "IMAGE:input_image"
+  output_stream: "MAT:input_mat"
+}
+
 node {
   calculator: "DrawLipstickCalculator"
-  input_stream: "IMAGE:input_image"
+  input_stream: "MAT:input_mat"
   input_stream: "MASK:0:multi_mask"
-  output_stream: "IMAGE:input_image_1"
+  output_stream: "MAT:lipstick_image"
+  output_stream: "MASK:lipstick_mask"
 }
 
 #Whitens teeth of the face on the IMAGE using MASK.
 node {
   calculator: "WhitenTeethCalculator"
-  input_stream: "IMAGE:input_image_1"
+  input_stream: "MAT:input_mat"
   input_stream: "MASK:0:multi_mask"
-  output_stream: "IMAGE:input_image_2"
+  output_stream: "MAT:teeth_image"
+  output_stream: "MASK:teeth_mask"
 }
 
 #Smoothes face on the IMAGE using MASK.
 node {
   calculator: "SmoothFaceCalculator1"
-  input_stream: "IMAGE:input_image_2"
+  input_stream: "MAT:input_mat"
   input_stream: "MASK:0:multi_mask"
-  input_stream: "FACEBOX:0:multi_face_box"
+  input_stream: "FACE:0:multi_face_box"
-  output_stream: "IMAGE:input_image_3"
+  output_stream: "MAT:tmp_image_1"
-  output_stream: "MASK:not_full_face"
+  output_stream: "MASK:face_mask"
-  output_stream: "FACEBOX:box1"
+  output_stream: "FACE:box_1"
 }
 
-#Smoothes face on the IMAGE using MASK.
 node {
+   calculator: "RoundRobinDemuxCalculator"
+   input_stream: "box_1"
+   output_stream: "OUTPUT:0:box1_0"
+   output_stream: "OUTPUT:1:box1_1"
+   output_stream: "SELECT:select"
+ }
+
+ node {
   calculator: "BilateralCalculator"
-  input_stream: "IMAGE:input_image_3"
+  input_stream: "IMAGE:box1_0"
-  input_stream: "FACEBOX:box1"
+  output_stream: "CVMAT:box14_0"
-  output_stream: "IMAGE2:input_image_4"
 }
+ node {
+   calculator: "BilateralCalculator"
+   input_stream: "IMAGE:box1_1"
+   output_stream: "CVMAT:box14_1"
+ }
+
+ node {
+   calculator: "MuxCalculator"
+   input_stream: "INPUT:0:box14_0"
+   input_stream: "INPUT:1:box14_1"
+   input_stream: "SELECT:select"
+  output_stream: "OUTPUT:tmp_image_2"
+   input_stream_handler {
+     input_stream_handler: "MuxInputStreamHandler"
+   }
+ }
 
 #Smoothes face on the IMAGE using MASK.
 node {
   calculator: "SmoothFaceCalculator2"
-  input_stream: "IMAGE:input_image_2"
+  input_stream: "MAT:input_mat"
-  input_stream: "IMAGE2:input_image_4"
+  input_stream: "IMAGE2:tmp_image_2"
-  input_stream: "MASK:not_full_face"
+  input_stream: "MASK:face_mask"
-  input_stream: "FACEBOX:box1"
+  input_stream: "FACEBOX:box_1"
+  output_stream: "MAT:face_image"
+}
+
+#Merges output images of DrawLipstick, WhitenTeeth and SmoothFace calculators
+node {
+  calculator: "MergeImagesCalculator"
+  input_stream: "MAT:0:lipstick_image"
+  input_stream: "MASK:0:lipstick_mask"
+  input_stream: "MAT:1:teeth_image"
+  input_stream: "MASK:1:teeth_mask"
+  input_stream: "MAT:2:face_image"
+  input_stream: "MASK:2:face_mask"
   output_stream: "IMAGE:output_image"
 }

mediapipe/graphs/beauty/subgraphs/face_renderer_gpu.pbtxt

@@ -18,7 +18,7 @@ output_stream: "IMAGE:output_image"
 
 node {
   calculator: "ImagePropertiesCalculator"
-  input_stream: "IMAGE_GPU:input_image"
+  input_stream: "IMAGE:input_image"
   output_stream: "SIZE:image_size"
 }
 # Outputs each element of multi_face_landmarks at a fake timestamp for the rest
@@ -28,16 +28,16 @@ node {
 node {
   calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
   input_stream: "ITERABLE:multi_face_landmarks"
-  input_stream: "IMAGE_GPU:input_image"
+  input_stream: "IMAGE:input_image"
   output_stream: "ITEM:face_landmarks"
-  output_stream: "IMAGE_GPU:loop_image"
+  output_stream: "IMAGE:loop_image"
   output_stream: "BATCH_END:landmark_timestamp"
 }
 
 # Converts landmarks to face part masks.
 node {
   calculator: "LandmarksToMaskCalculator"
-  input_stream: "IMAGE_GPU:loop_image"
+  input_stream: "IMAGE:loop_image"
   input_stream: "NORM_LANDMARKS:face_landmarks"
   output_stream: "FACEBOX:face_box"
   output_stream: "MASK:mask"
@@ -63,17 +63,17 @@ node {
 #Applies lipstick to the face on the IMAGE using MASK.
 node {
   calculator: "DrawLipstickCalculator"
-  input_stream: "IMAGE_GPU:input_image"
+  input_stream: "IMAGE:input_image"
   input_stream: "MASK:0:multi_mask"
-  output_stream: "IMAGE_GPU:input_image_1"
+  output_stream: "IMAGE:input_image_1"
 }
 
 #Whitens teeth of the face on the IMAGE using MASK.
 node {
   calculator: "WhitenTeethCalculator"
-  input_stream: "IMAGE_GPU:input_image_1"
+  input_stream: "IMAGE:input_image_1"
   input_stream: "MASK:0:multi_mask"
-  output_stream: "IMAGE_GPU:input_image_2"
+  output_stream: "IMAGE:input_image_2"
 }
 
 #Smoothes face on the IMAGE using MASK.
@@ -81,19 +81,41 @@ node {
   calculator: "SmoothFaceCalculator1"
   input_stream: "IMAGE:input_image_2"
   input_stream: "MASK:0:multi_mask"
-  input_stream: "FACEBOX:0:multi_face_box"
+  input_stream: "FACE:0:multi_face_box"
   output_stream: "IMAGE:input_image_3"
   output_stream: "MASK:not_full_face"
-  output_stream: "FACEBOX:box1"
+  output_stream: "FACE:box1"
 }
 
-#Smoothes face on the IMAGE using MASK.
 node {
+   calculator: "RoundRobinDemuxCalculator"
+   input_stream: "box1"
+   output_stream: "OUTPUT:0:box1_0"
+   output_stream: "OUTPUT:1:box1_1"
+   output_stream: "SELECT:select"
+ }
+
+ node {
   calculator: "BilateralCalculator"
-  input_stream: "IMAGE:input_image_3"
+  input_stream: "IMAGE:box1_0"
-  input_stream: "FACEBOX:box1"
+  output_stream: "CVMAT:box14_0"
-  output_stream: "IMAGE2:input_image_4"
 }
+ node {
+   calculator: "BilateralCalculator"
+   input_stream: "IMAGE:box1_1"
+   output_stream: "CVMAT:box14_1"
+ }
+
+ node {
+   calculator: "MuxCalculator"
+   input_stream: "INPUT:0:box14_0"
+   input_stream: "INPUT:1:box14_1"
+   input_stream: "SELECT:select"
+  output_stream: "OUTPUT:input_image_4"
+   input_stream_handler {
+     input_stream_handler: "MuxInputStreamHandler"
+   }
+ }
 
 #Smoothes face on the IMAGE using MASK.
 node {
@@ -104,3 +126,4 @@ node {
   input_stream: "FACEBOX:box1"
   output_stream: "IMAGE:output_image"
 }
+

mediapipe/graphs/deformation/BUILD

@@ -13,8 +13,8 @@
 # limitations under the License.
 
 load(
-    "//mediapipe/framework/tool:mediapipe_graph.bzl",
+    "//mediapipe/framework/tool:mediapipe_graph.bzl" ,
-    "mediapipe_binary_graph",
+    "mediapipe_binary_graph" ,
 )
 
 licenses(["notice"])
@@ -32,8 +32,8 @@ cc_library(
     "//mediapipe/calculators/core:constant_side_packet_calculator" ,
     "//mediapipe/gpu:gpu_buffer_to_image_frame_calculator" ,
     "//mediapipe/gpu:image_frame_to_gpu_buffer_calculator" ,
-        "//mediapipe/calculators/image:color_convert_calculator",
+    "//mediapipe/calculators/image:color_convert_calculator" ,
-        "//mediapipe/calculators/image:image_properties_calculator",
+    "//mediapipe/calculators/image:image_properties_calculator" ,
     ],
 )
 
@@ -46,16 +46,14 @@ cc_library(
     "//mediapipe/calculators/image:image_transformation_calculator" ,
     "//mediapipe/modules/face_landmark:face_landmark_front_cpu" ,
     "//mediapipe/calculators/core:constant_side_packet_calculator" ,
-        "//mediapipe/calculators/image:color_convert_calculator",
+    "//mediapipe/calculators/image:color_convert_calculator" ,
-        "//mediapipe/calculators/image:image_properties_calculator",
+    "//mediapipe/calculators/image:image_properties_calculator" ,
-
     ],
 )
 
 mediapipe_binary_graph(
-    name = "mobile_gpu_binary_graph",
+    name = "mobile_gpu_binary_graph" ,
-    graph = "deformation_mobile.pbtxt",
+    graph = "deformation_mobile.pbtxt" ,
-    output_name = "mobile_gpu.binarypb",
+    output_name = "mobile_gpu.binarypb" ,
     deps = [":mobile_calculators"],
 )
-

mediapipe/graphs/deformation/README.md

@@ -14,8 +14,7 @@ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/d
 ```
 Run with (using your camera):
 ```
-bazel-bin/mediapipe/examples/desktop/deformation/deformation_cpu 
+bazel-bin/mediapipe/examples/desktop/deformation/deformation_cpu --calculator_graph_config_file=mediapipe/graphs/deformation/deformation_cpu.pbtxt
---calculator_graph_config_file=mediapipe/graphs/deformation/deformation_cpu.pbtxt
 ```
 Run with (using video):
 ```

mediapipe/graphs/deformation/calculators/face_processor_calculator.cc

@@ -137,9 +137,9 @@ namespace mediapipe
 
     vector<string> index_names;
     map<string, vector<int>> indexes;
-
-    map<string, Tensor<double>> masks;
     vector<vector<int>> _trianglesIndexes;
+    map<string, Tensor<double>> masks;
+
     Tensor<double> __facePts;
   };
 
@@ -181,14 +181,13 @@ namespace mediapipe
   absl::Status FaceProcessorCalculator::Open(CalculatorContext *cc)
   {
     cc->SetOffset(TimestampDiff(0));
+    MP_RETURN_IF_ERROR(SetData(cc));
 
         return absl::OkStatus();
   }
 
   absl::Status FaceProcessorCalculator::Process(CalculatorContext *cc)
   {
-    MP_RETURN_IF_ERROR(SetData(cc));
-
     if (cc->Inputs().HasTag(kNormLandmarksTag) &&
         !cc->Inputs().Tag(kNormLandmarksTag).IsEmpty())
     {
@@ -206,11 +205,10 @@ namespace mediapipe
   absl::Status FaceProcessorCalculator::SetData(CalculatorContext *cc)
   {
     masks = {};
-    _trianglesIndexes = {};
+    string line;
 
     string filename = "mediapipe/graphs/deformation/config/triangles.txt";
-    string content_blob;
+    ASSIGN_OR_RETURN(string content_blob,
-    ASSIGN_OR_RETURN(content_blob,
                      ReadContentBlobFromFile(filename),
                      _ << "Failed to read texture blob from file!");
 
@@ -219,7 +217,6 @@ namespace mediapipe
     vector<int> tmp;
     for (int i = 0; i < 854; ++i)
     {
-      string line;
       tmp = {};
       for (int j = 0; j < 3; ++j)
       {
@@ -228,20 +225,20 @@ namespace mediapipe
       }
       _trianglesIndexes.push_back(tmp);
     }
+    stream.clear();
 
     filename = "./mediapipe/graphs/deformation/config/index_names.txt";
     ASSIGN_OR_RETURN(content_blob,
                      ReadContentBlobFromFile(filename),
                      _ << "Failed to read texture blob from file!");
-    istringstream stream2(content_blob);
+    stream.str(content_blob);
 
-    string line;
     vector<int> idxs;
-    while (getline(stream2, line))
+    while (getline(stream, line))
     {
       index_names.push_back(line);
     }
-    stream2.clear();
+    stream.clear();
 
     for (int i = 0; i < index_names.size(); i++)
     {
@@ -250,16 +247,16 @@ namespace mediapipe
       ASSIGN_OR_RETURN(content_blob,
                        ReadContentBlobFromFile(filename),
                        _ << "Failed to read texture blob from file!");
-      stream2.str(content_blob);
+      stream.str(content_blob);
 
-      while (getline(stream2, line))
+      while (getline(stream, line))
       {
         idxs.push_back(stoi(line));
       }
       indexes[index_names[i]] = idxs;
 
       idxs = {};
-      stream2.clear();
+      stream.clear();
     }
 
     double **zero_arr;

mediapipe/graphs/deformation/deformation_mobile.pbtxt

@@ -56,6 +56,16 @@ node {
   output_stream: "SIZE:size"	
 }
 
+node {
+   calculator: "LocalFileContentsCalculator"
+   input_side_packet: "FILE_PATH:0:file_path1"
+   input_side_packet: "FILE_PATH:1:file_path2"
+  input_side_packet: "FILE_PATH:1:file_path2"
+   output_side_packet: "CONTENTS:0:contents1"
+   output_side_packet: "CONTENTS:1:contents2"
+   output_side_packet: "CONTENTS:1:contents2"
+}
+
 node {
   calculator: "FaceProcessorCalculator"
   input_stream: "NORM_LANDMARKS:multi_face_landmarks"

mediapipe/graphs/image_style/README.md

@@ -18,9 +18,7 @@ bazel-bin/mediapipe/examples/desktop/image_style/image_style_cpu --calculator_gr
 ```
 Run with (using video):
 ```
-bazel-bin/mediapipe/examples/desktop/image_style/image_style_cpu --calculator_graph_config_file=mediapipe/graphs/image_style/image_style_cpu.pbtxt
+bazel-bin/mediapipe/examples/desktop/image_style/image_style_cpu --calculator_graph_config_file=mediapipe/graphs/image_style/image_style_cpu.pbtxt --input_video_path=/path/video.mp4 --output_video_path=/path/outvideo.mp4
---input_video_path=/path/video.mp4 
---output_video_path=/path/outvideo.mp4
 ```
 
 2. Mobile (Android)

mediapipe/graphs/image_style/image_style_cpu.pbtxt

@@ -6,6 +6,13 @@ input_stream: "input_video"
 # Output image with rendered results. (ImageFrame)
 output_stream: "output_video"
 
+profiler_config {
+  trace_enabled: true
+  enable_profiler: true
+  trace_log_interval_count: 200
+  trace_log_path: "/home/mslight/Work/clone/mediapipe/mediapipe/logs/imagestyle/"
+}
+
 node {
   calculator: "FlowLimiterCalculator"
   input_stream: "input_video"

mediapipe/graphs/makeup/BUILD

@@ -0,0 +1,63 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "//mediapipe/framework/tool:mediapipe_graph.bzl",
+    "mediapipe_binary_graph",
+)
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "desktop_live_calculators",
+    deps = [
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/graphs/makeup/subgraphs:face_renderer_cpu",
+        "//mediapipe/modules/face_landmark:face_landmark_front_cpu",
+    ],
+)
+
+cc_library(
+    name = "desktop_live_gpu_calculators",
+    deps = [
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/graphs/makeup/subgraphs:face_renderer_gpu",
+        "//mediapipe/modules/face_landmark:face_landmark_front_gpu",
+    ],
+)
+
+cc_library(
+    name = "mobile_calculators",
+    deps = [
+    	"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
+    	"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/graphs/makeup/subgraphs:face_renderer_cpu",
+        "//mediapipe/graphs/makeup/subgraphs:face_renderer_gpu",
+        "//mediapipe/modules/face_landmark:face_landmark_front_gpu",
+    ],
+)
+
+
+mediapipe_binary_graph(
+    name = "beauty_mobile_gpu_binary_graph",
+    graph = "beauty_mobile_gpu.pbtxt",
+    output_name = "beauty_mobile_gpu.binarypb",
+    deps = [":mobile_calculators"],
+)
+

mediapipe/graphs/makeup/makeup_desktop_cpu.pbtxt

@@ -0,0 +1,71 @@
+# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU.
+
+# Input image. (ImageFrame)
+input_stream: "input_video"
+
+# Output image with rendered results. (ImageFrame)
+output_stream: "output_video"
+# Collection of detected/processed faces, each represented as a list of
+# landmarks. (std::vector<NormalizedLandmarkList>)
+output_stream: "multi_face_landmarks"
+
+
+profiler_config {
+  trace_enabled: true
+  enable_profiler: true
+  trace_log_interval_count: 200
+  trace_log_path: "/home/mslight/Work/clone/mediapipe/mediapipe/logs/makeup/"
+}
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+
+
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:output_video"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Defines side packets for further use in the graph.
+node {
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:0:num_faces"
+  output_side_packet: "PACKET:1:with_attention"
+  node_options: {
+    [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
+      packet { int_value: 1 }
+      packet { bool_value: true }
+    }
+  }
+}
+
+# Subgraph that detects faces and corresponding landmarks.
+node {
+  calculator: "FaceLandmarkFrontCpu"
+  input_stream: "IMAGE:throttled_input_video"
+  input_side_packet: "NUM_FACES:num_faces"
+  input_side_packet: "WITH_ATTENTION:with_attention"
+  output_stream: "LANDMARKS:multi_face_landmarks"
+}
+
+# Subgraph that renders face-landmark annotation onto the input image.
+node {
+  calculator: "FaceRendererCpu"
+  input_stream: "IMAGE:throttled_input_video"
+  input_stream: "LANDMARKS:multi_face_landmarks"
+  output_stream: "IMAGE:output_video"
+}

mediapipe/graphs/makeup/makeup_mobile_gpu.pbtxt

@@ -0,0 +1,84 @@
+# MediaPipe graph that performs face mesh with TensorFlow Lite on GPU.
+
+# GPU buffer. (GpuBuffer)
+input_stream: "input_video"
+
+# Max number of faces to detect/process. (int)
+input_side_packet: "num_faces"
+
+# Output image with rendered results. (GpuBuffer)
+output_stream: "output_video"
+# Collection of detected/processed faces, each represented as a list of
+# landmarks. (std::vector<NormalizedLandmarkList>)
+output_stream: "multi_face_landmarks"
+
+profiler_config {
+  trace_enabled: true
+  enable_profiler: true
+  trace_log_interval_count: 200
+}
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:output_video"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Defines side packets for further use in the graph.
+node {
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:with_attention"
+  node_options: {
+    [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
+      packet { bool_value: true }
+    }
+  }
+}
+
+# Defines side packets for further use in the graph.
+node {
+  calculator: "GpuBufferToImageFrameCalculator"
+  input_stream: "throttled_input_video"
+  output_stream: "throttled_input_video_cpu"
+}
+
+# Subgraph that detects faces and corresponding landmarks.
+node {
+  calculator: "FaceLandmarkFrontGpu"
+  input_stream: "IMAGE:throttled_input_video"
+  input_side_packet: "NUM_FACES:num_faces"
+  input_side_packet: "WITH_ATTENTION:with_attention"
+  output_stream: "LANDMARKS:multi_face_landmarks"
+  output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
+  output_stream: "DETECTIONS:face_detections"
+  output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
+}
+
+# Subgraph that renders face-landmark annotation onto the input image.
+node {
+  calculator: "FaceRendererCpu"
+  input_stream: "IMAGE:throttled_input_video_cpu"
+  input_stream: "LANDMARKS:multi_face_landmarks"
+  output_stream: "IMAGE:output_video_cpu"
+}
+
+node {
+  calculator: "ImageFrameToGpuBufferCalculator"
+  input_stream: "output_video_cpu"
+  output_stream: "output_video"
+}

mediapipe/graphs/makeup/subgraphs/BUILD

@@ -0,0 +1,60 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load(
+    "//mediapipe/framework/tool:mediapipe_graph.bzl",
+    "mediapipe_simple_subgraph",
+)
+
+licenses(["notice"])
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "renderer_calculators",
+    deps = [
+        "//mediapipe/calculators/beauty:smooth_face_calculator1",
+        "//mediapipe/calculators/beauty:bilateral_filter_calculator",
+        "//mediapipe/calculators/beauty:smooth_face_calculator2",
+        "//mediapipe/calculators/beauty:draw_lipstick_calculator",
+        "//mediapipe/calculators/beauty:whiten_teeth_calculator",
+        "//mediapipe/calculators/beauty:merging_images_calculator",
+        "//mediapipe/calculators/beauty:imageframe_to_mat_calculator",
+        "//mediapipe/calculators/landmarks:landmarks_to_point_array_calculator",
+        "//mediapipe/calculators/landmarks:points_to_face_box_calculator",
+        "//mediapipe/calculators/landmarks:point_vector_to_mask_calculator",
+        "//mediapipe/calculators/core:round_robin_demux_calculator",
+        "//mediapipe/calculators/core:mux_calculator",    
+    ],
+)
+
+
+mediapipe_simple_subgraph(
+    name = "face_renderer_gpu",
+    graph = "face_renderer_gpu.pbtxt",
+    register_as = "FaceRendererGpu",
+    deps = [
+        ":renderer_calculators",
+    ],
+)
+
+
+mediapipe_simple_subgraph(
+    name = "face_renderer_cpu",
+    graph = "face_renderer_cpu.pbtxt",
+    register_as = "FaceRendererCpu",
+    deps = [
+        ":renderer_calculators",
+    ],
+)

mediapipe/graphs/makeup/subgraphs/face_renderer_cpu.pbtxt

@@ -0,0 +1,152 @@
+# MediaPipe face mesh rendering subgraph.
+
+type: "FaceRendererCpu"
+
+# CPU image. (ImageFrame)
+input_stream: "IMAGE:input_image"
+# Collection of detected/predicted faces, each represented as a list of
+# landmarks. (std::vector<NormalizedLandmarkList>)
+input_stream: "LANDMARKS:multi_face_landmarks"
+
+# CPU image with rendered data. (ImageFrame)
+output_stream: "IMAGE:output_image"
+
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:input_image"
+  output_stream: "SIZE:size"
+}
+
+node {
+  calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
+  input_stream: "ITERABLE:multi_face_landmarks"
+  input_stream: "CLONE:size"
+  output_stream: "ITEM:face_landmarks"
+  output_stream: "CLONE:image_size"
+  output_stream: "BATCH_END:landmark_timestamp"
+}
+
+# Converts landmarks to face part masks.
+node {
+  calculator: "LandmarksToPointArrayCalculator"
+  input_stream: "IMAGE_SIZE:image_size"
+  input_stream: "NORM_LANDMARKS:face_landmarks"
+  output_stream: "POINTS:points"
+}
+
+# Converts landmarks to face part masks.
+node {
+  calculator: "PointVectorToMaskCalculator"
+  input_stream: "POINTS:points"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "MASKS:mask"
+}
+
+# Converts landmarks to face part masks.
+node {
+  calculator: "PointsToFaceBoxCalculator"
+  input_stream: "POINTS:points"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "FACE_BOX:face_box"
+}
+
+node {
+  calculator: "EndLoopMapMaskCalculator"
+  input_stream: "ITEM:mask"
+  input_stream: "BATCH_END:landmark_timestamp"
+  output_stream: "ITERABLE:multi_mask"
+}
+
+node {
+  calculator: "EndLoopFaceBoxCalculator"
+  input_stream: "ITEM:face_box"
+  input_stream: "BATCH_END:landmark_timestamp"
+  output_stream: "ITERABLE:multi_face_box"
+}
+
+node {
+  calculator: "ImageFrameToMatCalculator"
+  input_stream: "IMAGE:input_image"
+  output_stream: "MAT:input_mat"
+}
+
+#Applies lipstick to the face on the IMAGE using MASK.
+node {
+  calculator: "DrawLipstickCalculator"
+  input_stream: "MAT:input_mat"
+  input_stream: "MASK:0:multi_mask"
+  output_stream: "MAT:lipstick_image"
+  output_stream: "MASK:lipstick_mask"
+}
+
+#Whitens teeth of the face on the IMAGE using MASK.
+node {
+  calculator: "WhitenTeethCalculator"
+  input_stream: "MAT:input_mat"
+  input_stream: "MASK:0:multi_mask"
+  output_stream: "MAT:teeth_image"
+  output_stream: "MASK:teeth_mask"
+}
+
+#Smoothes face on the IMAGE using MASK.
+node {
+  calculator: "SmoothFaceCalculator1"
+  input_stream: "MAT:input_mat"
+  input_stream: "MASK:0:multi_mask"
+  input_stream: "FACE:0:multi_face_box"
+  output_stream: "MAT:tmp_image_1"
+  output_stream: "MASK:face_mask"
+  output_stream: "FACE:box_1"
+}
+
+node {
+   calculator: "RoundRobinDemuxCalculator"
+   input_stream: "box_1"
+   output_stream: "OUTPUT:0:box1_0"
+   output_stream: "OUTPUT:1:box1_1"
+   output_stream: "SELECT:select"
+ }
+
+ node {
+  calculator: "BilateralCalculator"
+  input_stream: "IMAGE:box1_0"
+  output_stream: "CVMAT:box14_0"
+}
+ node {
+   calculator: "BilateralCalculator"
+   input_stream: "IMAGE:box1_1"
+   output_stream: "CVMAT:box14_1"
+ }
+
+ node {
+   calculator: "MuxCalculator"
+   input_stream: "INPUT:0:box14_0"
+   input_stream: "INPUT:1:box14_1"
+   input_stream: "SELECT:select"
+  output_stream: "OUTPUT:tmp_image_2"
+   input_stream_handler {
+     input_stream_handler: "MuxInputStreamHandler"
+   }
+ }
+
+#Smoothes face on the IMAGE using MASK.
+node {
+  calculator: "SmoothFaceCalculator2"
+  input_stream: "MAT:input_mat"
+  input_stream: "IMAGE2:tmp_image_2"
+  input_stream: "MASK:face_mask"
+  input_stream: "FACEBOX:box_1"
+  output_stream: "MAT:face_image"
+}
+
+#Merges output images of DrawLipstick, WhitenTeeth and SmoothFace calculators
+node {
+  calculator: "MergeImagesCalculator"
+  input_stream: "MAT:0:lipstick_image"
+  input_stream: "MASK:0:lipstick_mask"
+  input_stream: "MAT:1:teeth_image"
+  input_stream: "MASK:1:teeth_mask"
+  input_stream: "MAT:2:face_image"
+  input_stream: "MASK:2:face_mask"
+  output_stream: "IMAGE:output_image"
+}

mediapipe/graphs/makeup/subgraphs/face_renderer_gpu.pbtxt

@@ -0,0 +1,106 @@
+# MediaPipe face mesh rendering subgraph.
+
+type: "FaceRendererGpu"
+
+# GPU image. (GpuBuffer)
+input_stream: "IMAGE:input_image"
+# Collection of detected/predicted faces, each represented as a list of
+# landmarks. (std::vector<NormalizedLandmarkList>)
+input_stream: "LANDMARKS:multi_face_landmarks"
+# Regions of interest calculated based on palm detections.
+# (std::vector<NormalizedRect>)
+input_stream: "NORM_RECTS:rects"
+# Detected palms. (std::vector<Detection>)
+input_stream: "DETECTIONS:detections"
+
+# GPU image with rendered data. (GpuBuffer)
+output_stream: "IMAGE:output_image"
+
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:input_image"
+  output_stream: "SIZE:image_size"
+}
+# Outputs each element of multi_face_landmarks at a fake timestamp for the rest
+# of the graph to process. At the end of the loop, outputs the BATCH_END
+# timestamp for downstream calculators to inform them that all elements in the
+# vector have been processed.
+node {
+  calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
+  input_stream: "ITERABLE:multi_face_landmarks"
+  input_stream: "IMAGE_GPU:input_image"
+  output_stream: "ITEM:face_landmarks"
+  output_stream: "IMAGE_GPU:loop_image"
+  output_stream: "BATCH_END:landmark_timestamp"
+}
+
+# Converts landmarks to face part masks.
+node {
+  calculator: "LandmarksToMaskCalculator"
+  input_stream: "IMAGE_GPU:loop_image"
+  input_stream: "NORM_LANDMARKS:face_landmarks"
+  output_stream: "FACEBOX:face_box"
+  output_stream: "MASK:mask"
+}
+
+# Collects a MapMask object for each hand into a vector. Upon receiving the
+# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END
+# timestamp.
+node {
+  calculator: "EndLoopMapMaskCalculator"
+  input_stream: "ITEM:mask"
+  input_stream: "BATCH_END:landmark_timestamp"
+  output_stream: "ITERABLE:multi_mask"
+}
+
+node {
+  calculator: "EndLoopFaceBoxCalculator"
+  input_stream: "ITEM:face_box"
+  input_stream: "BATCH_END:landmark_timestamp"
+  output_stream: "ITERABLE:multi_face_box"
+}
+
+#Applies lipstick to the face on the IMAGE using MASK.
+node {
+  calculator: "DrawLipstickCalculator"
+  input_stream: "IMAGE_GPU:input_image"
+  input_stream: "MASK:0:multi_mask"
+  output_stream: "IMAGE_GPU:input_image_1"
+}
+
+#Whitens teeth of the face on the IMAGE using MASK.
+node {
+  calculator: "WhitenTeethCalculator"
+  input_stream: "IMAGE_GPU:input_image_1"
+  input_stream: "MASK:0:multi_mask"
+  output_stream: "IMAGE_GPU:input_image_2"
+}
+
+#Smoothes face on the IMAGE using MASK.
+node {
+  calculator: "SmoothFaceCalculator1"
+  input_stream: "IMAGE:input_image_2"
+  input_stream: "MASK:0:multi_mask"
+  input_stream: "FACEBOX:0:multi_face_box"
+  output_stream: "IMAGE:input_image_3"
+  output_stream: "MASK:not_full_face"
+  output_stream: "FACEBOX:box1"
+}
+
+#Smoothes face on the IMAGE using MASK.
+node {
+  calculator: "BilateralCalculator"
+  input_stream: "IMAGE:input_image_3"
+  input_stream: "FACEBOX:box1"
+  output_stream: "IMAGE2:input_image_4"
+}
+
+#Smoothes face on the IMAGE using MASK.
+node {
+  calculator: "SmoothFaceCalculator2"
+  input_stream: "IMAGE:input_image_2"
+  input_stream: "IMAGE2:input_image_4"
+  input_stream: "MASK:not_full_face"
+  input_stream: "FACEBOX:box1"
+  output_stream: "IMAGE:output_image"
+}