From 61da56f99fe34442f02847a14da93abf7e5b27ca Mon Sep 17 00:00:00 2001 From: mslight Date: Thu, 7 Jul 2022 16:31:16 +0400 Subject: [PATCH] code optimizitation --- mediapipe/calculators/beauty/BUILD | 9 --- .../beauty/draw_lipstick_calculator.cc | 18 ----- .../beauty/smooth_face_calculator.cc | 68 +++++++------------ .../beauty/whiten_teeth_calculator.cc | 45 +++++------- mediapipe/calculators/landmarks/BUILD | 3 - .../landmarks/landmarks_to_mask_calculator.cc | 53 ++++----------- .../landmarks/landmarks_to_mask_calculator.h | 31 +++------ mediapipe/examples/desktop/beauty/BUILD | 20 +++++- mediapipe/graphs/beauty/beauty_desktop.pbtxt | 50 ++++++++++++++ mediapipe/graphs/beauty/subgraphs/BUILD | 3 - .../beauty/subgraphs/face_renderer_cpu.pbtxt | 8 ++- .../subgraphs/face_renderer_cpu_old.pbtxt | 48 +------------ 12 files changed, 138 insertions(+), 218 deletions(-) create mode 100644 mediapipe/graphs/beauty/beauty_desktop.pbtxt diff --git a/mediapipe/calculators/beauty/BUILD b/mediapipe/calculators/beauty/BUILD index 16ff31356..55210c628 100644 --- a/mediapipe/calculators/beauty/BUILD +++ b/mediapipe/calculators/beauty/BUILD @@ -34,11 +34,8 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:opencv_highgui", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", - "//mediapipe/util:annotation_renderer", - "//mediapipe/util:render_data_cc_proto", ], alwayslink = 1, ) @@ -87,11 +84,8 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:opencv_highgui", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", - "//mediapipe/util:annotation_renderer", - "//mediapipe/util:render_data_cc_proto", ], alwayslink = 1, ) @@ -112,11 +106,8 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:opencv_highgui", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", - "//mediapipe/util:annotation_renderer", - "//mediapipe/util:render_data_cc_proto", ], alwayslink = 1, ) diff --git a/mediapipe/calculators/beauty/draw_lipstick_calculator.cc b/mediapipe/calculators/beauty/draw_lipstick_calculator.cc index 971babaee..4dcabb00d 100644 --- a/mediapipe/calculators/beauty/draw_lipstick_calculator.cc +++ b/mediapipe/calculators/beauty/draw_lipstick_calculator.cc @@ -18,7 +18,6 @@ #include #include #include -//#include #include @@ -33,17 +32,13 @@ #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/status.h" -#include "mediapipe/util/annotation_renderer.h" -#include "mediapipe/util/render_data.pb.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/vector.h" -#include "mediapipe/util/color.pb.h" namespace mediapipe { namespace { - constexpr char kMaskTag[] = "MASK"; constexpr char kImageFrameTag[] = "IMAGE"; @@ -54,12 +49,7 @@ namespace mediapipe NUM_ATTRIBUTES }; - // Round up n to next multiple of m. - size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } - - using Point = RenderAnnotation::Point; - } // namespace class DrawLipstickCalculator : public CalculatorBase @@ -91,10 +81,6 @@ namespace mediapipe // Indicates if image frame is available as input. bool image_frame_available_ = false; std::unordered_map all_masks; - int width_ = 0; - int height_ = 0; - int width_canvas_ = 0; // Size of overlay drawing texture canvas. - int height_canvas_ = 0; }; REGISTER_CALCULATOR(DrawLipstickCalculator); @@ -141,9 +127,6 @@ namespace mediapipe { image_frame_available_ = true; } - else - { - } // Set the output header based on the input header (if present). const char *tag = kImageFrameTag; @@ -204,7 +187,6 @@ namespace mediapipe CalculatorContext *cc, const ImageFormat::Format &target_format, uchar *data_image, std::unique_ptr &image_mat) { - cv::Mat mat_image_ = *image_mat.get(); auto output_frame = absl::make_unique( diff --git a/mediapipe/calculators/beauty/smooth_face_calculator.cc b/mediapipe/calculators/beauty/smooth_face_calculator.cc index 4f96020c6..33b578dd1 100644 --- a/mediapipe/calculators/beauty/smooth_face_calculator.cc +++ b/mediapipe/calculators/beauty/smooth_face_calculator.cc @@ -17,7 +17,6 @@ #include #include #include -//#include #include @@ -32,11 +31,8 @@ #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/status.h" -#include "mediapipe/util/annotation_renderer.h" -#include "mediapipe/util/render_data.pb.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/vector.h" -#include "mediapipe/util/color.pb.h" namespace mediapipe { @@ -54,12 +50,7 @@ namespace mediapipe NUM_ATTRIBUTES }; - // Round up n to next multiple of m. - size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } - - using Point = RenderAnnotation::Point; - } // namespace class SmoothFaceCalculator : public CalculatorBase @@ -82,23 +73,22 @@ namespace mediapipe absl::Status RenderToCpu( CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image, std::unique_ptr &image_mat); + uchar *data_image); - absl::Status SmoothFace(CalculatorContext *cc, std::unique_ptr &image_mat, + absl::Status SmoothFace(CalculatorContext *cc, ImageFormat::Format *target_format, const std::unordered_map &mask_vec, const std::tuple &face_box); - cv::Mat predict_forehead_mask(std::unique_ptr &image_mat, - const std::unordered_map &mask_vec, double face_box_min_y); + cv::Mat predict_forehead_mask(const std::unordered_map &mask_vec, double face_box_min_y); // Indicates if image frame is available as input. bool image_frame_available_ = false; - std::unordered_map all_masks; - int width_ = 0; - int height_ = 0; - int width_canvas_ = 0; // Size of overlay drawing texture canvas. - int height_canvas_ = 0; + + int image_width_; + int image_height_; + cv::Mat mat_image_; + std::unique_ptr image_mat; }; REGISTER_CALCULATOR(SmoothFaceCalculator); @@ -186,7 +176,6 @@ namespace mediapipe } // Initialize render target, drawn with OpenCV. - std::unique_ptr image_mat; ImageFormat::Format target_format; if (cc->Outputs().HasTag(kImageFrameTag)) @@ -194,6 +183,10 @@ namespace mediapipe MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); } + mat_image_ = *image_mat.get(); + image_width_ = image_mat->cols; + image_height_ = image_mat->rows; + const std::vector> &mask_vec = cc->Inputs().Tag(kMaskTag).Get>>(); @@ -203,12 +196,12 @@ namespace mediapipe if (mask_vec.size() > 0 && face_box.size() > 0) { for (int i = 0; i < mask_vec.size(); i++) - MP_RETURN_IF_ERROR(SmoothFace(cc, image_mat, &target_format, mask_vec[i], face_box[i])); + MP_RETURN_IF_ERROR(SmoothFace(cc, &target_format, mask_vec[i], face_box[i])); } // Copy the rendered image to output. uchar *image_mat_ptr = image_mat->data; - MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat)); + MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr)); return absl::OkStatus(); } @@ -220,15 +213,12 @@ namespace mediapipe absl::Status SmoothFaceCalculator::RenderToCpu( CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image, std::unique_ptr &image_mat) + uchar *data_image) { - - cv::Mat mat_image__ = *image_mat.get(); - auto output_frame = absl::make_unique( - target_format, mat_image__.cols, mat_image__.rows); + target_format, image_width_, image_height_); - output_frame->CopyPixelData(target_format, mat_image__.cols, mat_image__.rows, data_image, + output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image, ImageFrame::kDefaultAlignmentBoundary); if (cc->Outputs().HasTag(kImageFrameTag)) @@ -290,29 +280,23 @@ namespace mediapipe { image_mat = absl::make_unique( 150, 150, CV_8UC4, - cv::Scalar(255, 255, - 255)); + cv::Scalar::all(255)); *target_format = ImageFormat::SRGBA; } return absl::OkStatus(); } - cv::Mat SmoothFaceCalculator::predict_forehead_mask(std::unique_ptr &image_mat, - const std::unordered_map &mask_vec, double face_box_min_y) + cv::Mat SmoothFaceCalculator::predict_forehead_mask(const std::unordered_map &mask_vec, double face_box_min_y) { - cv::Mat mat_image__ = *image_mat.get(); - int image_width_ = image_mat->cols; - int image_height_ = image_mat->rows; - cv::Mat part_forehead_mask = mask_vec.find("PART_FOREHEAD_B")->second.clone(); part_forehead_mask.convertTo(part_forehead_mask, CV_32F, 1.0 / 255); part_forehead_mask.convertTo(part_forehead_mask, CV_8U); cv::Mat image_sm, image_sm_hsv, skinMask; - cv::resize(mat_image__, image_sm, cv::Size(mat_image__.size().width, mat_image__.size().height)); + cv::resize(mat_image_, image_sm, cv::Size(image_width_, image_height_)); cv::cvtColor(image_sm, image_sm_hsv, cv::COLOR_BGR2HSV); std::vector x, y; @@ -375,20 +359,14 @@ namespace mediapipe } absl::Status SmoothFaceCalculator::SmoothFace(CalculatorContext *cc, - std::unique_ptr &image_mat, ImageFormat::Format *target_format, const std::unordered_map &mask_vec, const std::tuple &face_box) { - - cv::Mat mat_image__ = *image_mat.get(); - cv::Mat mouth_mask, mouth; - int image_width_ = image_mat->cols; - int image_height_ = image_mat->rows; cv::Mat not_full_face = mask_vec.find("FACE_OVAL")->second.clone() + - predict_forehead_mask(image_mat, mask_vec, std::get<1>(face_box)) - + predict_forehead_mask(mask_vec, std::get<1>(face_box)) - mask_vec.find("LEFT_EYE")->second.clone() - mask_vec.find("RIGHT_EYE")->second.clone() - mask_vec.find("LEFT_BROW")->second.clone() - @@ -397,7 +375,7 @@ namespace mediapipe cv::resize(not_full_face, not_full_face, - mat_image__.size(), 0, 0, + mat_image_.size(), 0, 0, cv::INTER_LINEAR); std::vector x, y; @@ -416,7 +394,7 @@ namespace mediapipe cv::minMaxLoc(x, &min_x, &max_x); cv::minMaxLoc(y, &min_y, &max_y); - cv::Mat patch_face = mat_image__(cv::Range(min_y, max_y), cv::Range(min_x, max_x)); + cv::Mat patch_face = mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x)); cv::Mat patch_nff = not_full_face(cv::Range(min_y, max_y), cv::Range(min_x, max_x)); cv::Mat patch_new, patch_wow; cv::cvtColor(patch_face, patch_wow, cv::COLOR_RGBA2RGB); diff --git a/mediapipe/calculators/beauty/whiten_teeth_calculator.cc b/mediapipe/calculators/beauty/whiten_teeth_calculator.cc index 5881ac89b..31c67eb69 100644 --- a/mediapipe/calculators/beauty/whiten_teeth_calculator.cc +++ b/mediapipe/calculators/beauty/whiten_teeth_calculator.cc @@ -31,11 +31,8 @@ #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/status.h" -#include "mediapipe/util/annotation_renderer.h" -#include "mediapipe/util/render_data.pb.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/vector.h" -#include "mediapipe/util/color.pb.h" namespace mediapipe { @@ -52,12 +49,7 @@ namespace mediapipe NUM_ATTRIBUTES }; - // Round up n to next multiple of m. - size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } - - using Point = RenderAnnotation::Point; - } // namespace class WhitenTeethCalculator : public CalculatorBase @@ -82,17 +74,15 @@ namespace mediapipe CalculatorContext *cc, const ImageFormat::Format &target_format, uchar *data_image, std::unique_ptr &image_mat); - absl::Status WhitenTeeth(CalculatorContext *cc, std::unique_ptr &image_mat, - ImageFormat::Format *target_format, + absl::Status WhitenTeeth(CalculatorContext *cc, ImageFormat::Format *target_format, const std::unordered_map &mask_vec); // Indicates if image frame is available as input. bool image_frame_available_ = false; - std::unordered_map all_masks; - int width_ = 0; - int height_ = 0; - int width_canvas_ = 0; // Size of overlay drawing texture canvas. - int height_canvas_ = 0; + std::unique_ptr image_mat; + cv::Mat mat_image_; + int image_width_; + int image_height_; }; REGISTER_CALCULATOR(WhitenTeethCalculator); @@ -170,20 +160,22 @@ namespace mediapipe } // Initialize render target, drawn with OpenCV. - std::unique_ptr image_mat; ImageFormat::Format target_format; if (cc->Outputs().HasTag(kImageFrameTag)) { MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); } - + mat_image_ = *image_mat.get(); + image_width_ = image_mat->cols; + image_height_ = image_mat->rows; + const std::vector> &mask_vec = cc->Inputs().Tag(kMaskTag).Get>>(); if (mask_vec.size() > 0) { for (auto mask : mask_vec) - MP_RETURN_IF_ERROR(WhitenTeeth(cc, image_mat, &target_format, mask)); + MP_RETURN_IF_ERROR(WhitenTeeth(cc, &target_format, mask)); } // Copy the rendered image to output. @@ -206,9 +198,9 @@ namespace mediapipe cv::Mat mat_image_ = *image_mat.get(); auto output_frame = absl::make_unique( - target_format, mat_image_.cols, mat_image_.rows); + target_format, image_width_, image_height_); - output_frame->CopyPixelData(target_format, mat_image_.cols, mat_image_.rows, data_image, + output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image, ImageFrame::kDefaultAlignmentBoundary); if (cc->Outputs().HasTag(kImageFrameTag)) @@ -279,20 +271,15 @@ namespace mediapipe } absl::Status WhitenTeethCalculator::WhitenTeeth(CalculatorContext *cc, - std::unique_ptr &image_mat, ImageFormat::Format *target_format, const std::unordered_map &mask_vec) { - cv::Mat mat_image__ = *image_mat.get(); - cv::Mat mouth_mask, mouth; - int image_width_ = image_mat->cols; - int image_height_ = image_mat->rows; - mouth_mask = cv::Mat::zeros(mat_image__.size(), CV_32F); + mouth_mask = cv::Mat::zeros(mat_image_.size(), CV_32F); mouth_mask = mask_vec.find("MOUTH_INSIDE")->second.clone(); - cv::resize(mouth_mask, mouth, mat_image__.size(), cv::INTER_LINEAR); + cv::resize(mouth_mask, mouth, mat_image_.size(), cv::INTER_LINEAR); std::vector x, y; std::vector location; @@ -323,7 +310,7 @@ namespace mediapipe mouth_max_x = static_cast(std::min(mouth_max_x + mw * 0.1, (double)image_width_)); mouth_crop_mask = mouth(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)); cv::Mat img_hsv, tmp_mask, img_hls; - cv::cvtColor(mat_image__(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)), img_hsv, + cv::cvtColor(mat_image_(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)), img_hsv, cv::COLOR_RGBA2RGB); cv::cvtColor(img_hsv, img_hsv, cv::COLOR_RGB2HSV); @@ -357,7 +344,7 @@ namespace mediapipe cv::cvtColor(img_hls, img_hls, cv::COLOR_HLS2RGB); cv::cvtColor(img_hls, img_hls, cv::COLOR_RGB2RGBA); - cv::Mat slice = mat_image__(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)); + cv::Mat slice = mat_image_(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)); img_hls.copyTo(slice); } } diff --git a/mediapipe/calculators/landmarks/BUILD b/mediapipe/calculators/landmarks/BUILD index 00a86f465..6f05ce11f 100644 --- a/mediapipe/calculators/landmarks/BUILD +++ b/mediapipe/calculators/landmarks/BUILD @@ -28,9 +28,6 @@ cc_library( "//mediapipe/framework:calculator_options_cc_proto", "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:location_data_cc_proto", - "//mediapipe/framework/port:ret_check", - "//mediapipe/util:color_cc_proto", - "//mediapipe/util:render_data_cc_proto", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", "//mediapipe/framework/formats:image_format_cc_proto", diff --git a/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.cc b/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.cc index 600cd8ac3..cdacca72c 100644 --- a/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.cc +++ b/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.cc @@ -17,28 +17,9 @@ #include #include -#include -#include #include -#include - #include "absl/memory/memory.h" -#include "absl/strings/str_cat.h" -#include "absl/strings/str_join.h" -#include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" -#include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/color.pb.h" -#include "mediapipe/util/render_data.pb.h" -#include "absl/strings/str_cat.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/vector.h" @@ -49,7 +30,6 @@ namespace mediapipe { constexpr char kLandmarksTag[] = "LANDMARKS"; constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS"; - constexpr char kLandmarkLabel[] = "KEYPOINT"; constexpr char kVectorTag[] = "VECTOR"; constexpr char kMaskTag[] = "MASK"; constexpr char kFaceBoxTag[] = "FACEBOX"; @@ -111,6 +91,11 @@ namespace mediapipe std::tuple face_box; + std::unique_ptr image_mat; + + int image_width_; + int image_height_; + float scale_factor_ = 1.0; bool image_frame_available_ = false; @@ -161,9 +146,6 @@ namespace mediapipe { image_frame_available_ = true; } - else - { - } return absl::OkStatus(); } @@ -190,15 +172,18 @@ namespace mediapipe // Initialize render target, drawn with OpenCV. - std::unique_ptr image_mat; ImageFormat::Format target_format; std::unordered_map all_masks; MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); - MP_RETURN_IF_ERROR(GetMasks(cc, all_masks, image_mat)); + cv::Mat mat_image_ = *image_mat.get(); + image_width_ = image_mat->cols; + image_height_ = image_mat->rows; - MP_RETURN_IF_ERROR(GetFaceBox(cc, image_mat)); + MP_RETURN_IF_ERROR(GetMasks(cc, all_masks)); + + MP_RETURN_IF_ERROR(GetFaceBox(cc)); MP_RETURN_IF_ERROR(RenderToCpu(cc, all_masks)); @@ -289,12 +274,8 @@ namespace mediapipe } absl::Status LandmarksToMaskCalculator::GetMasks(CalculatorContext *cc, - std::unordered_map &all_masks, std::unique_ptr &image_mat) + std::unordered_map &all_masks) { - - int image_width_ = image_mat->cols; - int image_height_ = image_mat->rows; - if (cc->Inputs().HasTag(kLandmarksTag)) { const LandmarkList &landmarks = @@ -326,10 +307,11 @@ namespace mediapipe std::vector> point_vec; point_vec.push_back(point_array); + mask = cv::Mat::zeros(image_mat->size(), CV_32FC1); cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA); mask.convertTo(mask, CV_8U); - all_masks.insert(make_pair(key, mask)); + all_masks.insert({key, mask}); point_vec.clear(); point_array.clear(); } @@ -377,13 +359,8 @@ namespace mediapipe return absl::OkStatus(); } - absl::Status LandmarksToMaskCalculator::GetFaceBox(CalculatorContext *cc, std::unique_ptr &image_mat) + absl::Status LandmarksToMaskCalculator::GetFaceBox(CalculatorContext *cc) { - cv::Mat mat_image_ = *image_mat.get(); - - int image_width_ = image_mat->cols; - int image_height_ = image_mat->rows; - std::vector x_s, y_s; double box_min_y, box_max_y, box_max_x, box_min_x; if (cc->Inputs().HasTag(kLandmarksTag)) diff --git a/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h b/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h index 7454f206c..49b82944c 100644 --- a/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h +++ b/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h @@ -14,46 +14,33 @@ #ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_TO_MASK_CALCULATOR_H_ #define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_TO_MASK_CALCULATOR_H_ -#include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_options.pb.h" #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/location_data.pb.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/util/color.pb.h" -#include "mediapipe/util/render_data.pb.h" #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/vector.h" namespace mediapipe { - // A calculator that converts Landmark proto to RenderData proto for - // visualization. The input should be LandmarkList proto. It is also possible - // to specify the connections between landmarks. + // A calculator that uses face landmarks to create face part masks and facebox for + // visualization. The input should be LandmarkList proto and ImageFrame. // // Example config: // node { // calculator: "LandmarksToMaskCalculator" - // input_stream: "NORM_LANDMARKS:landmarks" - // output_stream: "RENDER_DATA:render_data" - // options { - // [LandmarksToRenderDataCalculatorOptions.ext] { - // landmark_connections: [0, 1, 1, 2] - // landmark_color { r: 0 g: 255 b: 0 } - // connection_color { r: 0 g: 255 b: 0 } - // thickness: 4.0 - // } - // } + // input_stream: "IMAGE:image" + // input_stream: "NORM_LANDMARKS:face_landmarks" + // output_stream: "FACEBOX:face_box" + // output_stream: "MASK:mask" // } + class LandmarksToMaskCalculator : public CalculatorBase { public: @@ -74,9 +61,9 @@ namespace mediapipe absl::Status RenderToCpu(CalculatorContext *cc, std::unordered_map &all_masks); - absl::Status GetFaceBox(CalculatorContext *cc, std::unique_ptr &image_mat); + absl::Status GetFaceBox(CalculatorContext *cc); - absl::Status GetMasks(CalculatorContext *cc, std::unordered_map &all_masks, std::unique_ptr &image_mat); + absl::Status GetMasks(CalculatorContext *cc, std::unordered_map &all_masks); absl::Status CreateRenderTargetCpu( CalculatorContext *cc, std::unique_ptr &image_mat, diff --git a/mediapipe/examples/desktop/beauty/BUILD b/mediapipe/examples/desktop/beauty/BUILD index e6d4f6909..f09321e79 100644 --- a/mediapipe/examples/desktop/beauty/BUILD +++ b/mediapipe/examples/desktop/beauty/BUILD @@ -17,7 +17,7 @@ licenses(["notice"]) package(default_visibility = ["//mediapipe/examples:__subpackages__"]) cc_binary( - name = "face_mesh_tflite", + name = "beauty_tflite", deps = [ "//mediapipe/examples/desktop:simple_run_graph_main", "//mediapipe/graphs/beauty:desktop_calculators", @@ -25,9 +25,25 @@ cc_binary( ) cc_binary( - name = "face_mesh_cpu", + name = "beauty_cpu", deps = [ "//mediapipe/examples/desktop:demo_run_graph_main", "//mediapipe/graphs/beauty:desktop_live_calculators", ], ) + +cc_binary( + name = "beauty_cpu_over", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main", + "//mediapipe/graphs/beauty:desktop_live_over_calculators", + ], +) + +cc_binary( + name = "beauty_gpu_over", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main", + "//mediapipe/graphs/beauty:desktop_gpu_over", + ], +) diff --git a/mediapipe/graphs/beauty/beauty_desktop.pbtxt b/mediapipe/graphs/beauty/beauty_desktop.pbtxt new file mode 100644 index 000000000..f6d407303 --- /dev/null +++ b/mediapipe/graphs/beauty/beauty_desktop.pbtxt @@ -0,0 +1,50 @@ +# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU. + +# Input image. (ImageFrame) +input_stream: "input_video" + +# Output image with rendered results. (ImageFrame) +output_stream: "output_video" +# Collection of detected/processed faces, each represented as a list of +# landmarks. (std::vector) +output_stream: "multi_face_landmarks" + +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:output_video" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +node { + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:0:num_faces" + output_side_packet: "PACKET:1:with_attention" + node_options: { + [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { + packet { int_value: 1 } + packet { bool_value: true } + } + } +} + +# Subgraph that detects faces and corresponding landmarks. +node { + calculator: "FaceLandmarkFrontCpu" + input_stream: "IMAGE:throttled_input_video" + input_side_packet: "NUM_FACES:num_faces" + input_side_packet: "WITH_ATTENTION:with_attention" + output_stream: "LANDMARKS:multi_face_landmarks" +} + +# Subgraph that renders onto the input image. +node { + calculator: "FaceRendererCpu" + input_stream: "IMAGE:throttled_input_video" + input_stream: "LANDMARKS:multi_face_landmarks" + output_stream: "IMAGE:output_video" +} diff --git a/mediapipe/graphs/beauty/subgraphs/BUILD b/mediapipe/graphs/beauty/subgraphs/BUILD index d96ef2ec5..8001e0fc7 100644 --- a/mediapipe/graphs/beauty/subgraphs/BUILD +++ b/mediapipe/graphs/beauty/subgraphs/BUILD @@ -27,15 +27,12 @@ cc_library( "//mediapipe/calculators/core:split_proto_list_calculator", "//mediapipe/util:annotation_renderer", "//mediapipe/calculators/util:annotation_overlay_calculator", - "//mediapipe/calculators/beauty:form_face_mask_calculator", "//mediapipe/calculators/beauty:smooth_face_calculator", "//mediapipe/calculators/beauty:draw_lipstick_calculator", "//mediapipe/calculators/beauty:whiten_teeth_calculator", "//mediapipe/calculators/util:detections_to_render_data_calculator", "//mediapipe/calculators/util:landmarks_to_render_data_calculator", "//mediapipe/calculators/landmarks:landmarks_to_mask_calculator", - "//mediapipe/calculators/util:rect_to_render_data_calculator", - "//mediapipe/graphs/beauty/calculators:face_landmarks_to_render_data_calculator", ], ) diff --git a/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt b/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt index 8d6d9ddd3..866754f72 100644 --- a/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt +++ b/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt @@ -30,7 +30,7 @@ node { output_stream: "BATCH_END:landmark_timestamp" } -# Converts landmarks to drawing primitives for annotation overlay. +# Converts landmarks to face part masks. node { calculator: "LandmarksToMaskCalculator" input_stream: "IMAGE:loop_image" @@ -39,7 +39,7 @@ node { output_stream: "MASK:mask" } -# Collects a RenderData object for each hand into a vector. Upon receiving the +# Collects a MapMask object for each hand into a vector. Upon receiving the # BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END # timestamp. node { @@ -56,7 +56,7 @@ node { output_stream: "ITERABLE:multi_face_box" } - +#Applies lipstick to the face on the IMAGE using MASK. node { calculator: "DrawLipstickCalculator" input_stream: "IMAGE:input_image" @@ -64,6 +64,7 @@ node { output_stream: "IMAGE:input_image_1" } +#Whitens teeth of the face on the IMAGE using MASK. node { calculator: "WhitenTeethCalculator" input_stream: "IMAGE:input_image_1" @@ -71,6 +72,7 @@ node { output_stream: "IMAGE:input_image_2" } +#Smoothes face on the IMAGE using MASK. node { calculator: "SmoothFaceCalculator" input_stream: "IMAGE:input_image_2" diff --git a/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu_old.pbtxt b/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu_old.pbtxt index 0b866b21b..5052208b6 100644 --- a/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu_old.pbtxt +++ b/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu_old.pbtxt @@ -53,57 +53,13 @@ node { output_stream: "ITERABLE:multi_face_landmarks_render_data" } -# Converts normalized rects to drawing primitives for annotation overlay. -#node { -# calculator: "RectToRenderDataCalculator" -# input_stream: "NORM_RECTS:rects" -# output_stream: "RENDER_DATA:rects_render_data" -# node_options: { -# [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] { -# filled: false -# color { r: 255 g: 0 b: 0 } -# thickness: 4.0 -# } -# } -#} - +# Draws annotations and overlays them on top of the input images. node { - calculator: "FormFaceMaskCalculator" + calculator: "AnnotationOverlayCalculator" input_stream: "IMAGE:input_image" input_stream: "VECTOR:0:multi_face_landmarks_render_data" - output_stream: "FACEBOX:face_box" - output_stream: "MASK:multi_mask" -} - -node { - calculator: "DrawLipstickCalculator" - input_stream: "IMAGE:input_image" - input_stream: "MASK:0:multi_mask" - output_stream: "IMAGE:input_image_1" -} - -node { - calculator: "WhitenTeethCalculator" - input_stream: "IMAGE:input_image_1" - input_stream: "MASK:0:multi_mask" - output_stream: "IMAGE:input_image_2" -} - -node { - calculator: "SmoothFaceCalculator" - input_stream: "IMAGE:input_image_2" - input_stream: "MASK:0:multi_mask" - input_stream: "FACEBOX:face_box" output_stream: "IMAGE:output_image" } -# Draws annotations and overlays them on top of the input images. -#node { -# calculator: "AnnotationOverlayCalculator" -# input_stream: "IMAGE:input_image" -# input_stream: "VECTOR:0:multi_face_landmarks_render_data" -# output_stream: "IMAGE:output_image" -#} -