diff --git a/mediapipe/calculators/beauty/BUILD b/mediapipe/calculators/beauty/BUILD index 79a8e2549..f05421907 100644 --- a/mediapipe/calculators/beauty/BUILD +++ b/mediapipe/calculators/beauty/BUILD @@ -22,29 +22,6 @@ cc_library( name = "draw_lipstick_calculator", srcs = ["draw_lipstick_calculator.cc"], visibility = ["//visibility:public"], - deps = [ - "//mediapipe/framework:calculator_options_cc_proto", - "//mediapipe/framework/formats:image_format_cc_proto", - "//mediapipe/util:color_cc_proto", - "@com_google_absl//absl/strings", - "//mediapipe/framework:calculator_framework", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", - "//mediapipe/framework/formats:video_stream_header", - "//mediapipe/framework/port:logging", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:status", - "//mediapipe/framework/port:vector", - ], - alwayslink = 1, -) - - -cc_library( - name = "form_face_mask_calculator", - srcs = ["form_face_mask_calculator.cc"], - visibility = ["//visibility:public"], deps = [ "//mediapipe/framework:calculator_options_cc_proto", "//mediapipe/framework/formats:image_format_cc_proto", @@ -60,14 +37,10 @@ cc_library( "//mediapipe/framework/port:opencv_highgui", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", - "//mediapipe/util:annotation_renderer", - "//mediapipe/util:render_data_cc_proto", ], alwayslink = 1, ) - - cc_library( name = "smooth_face_calculator1", srcs = ["smooth_face1_calculator.cc"], @@ -84,6 +57,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_highgui", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", ], @@ -106,6 +80,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_highgui", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", ], @@ -128,6 +103,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_highgui", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", ], @@ -150,14 +126,49 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_highgui", "//mediapipe/framework/port:status", "//mediapipe/framework/port:vector", ], alwayslink = 1, ) - - - +cc_library( + name = "merging_images_calculator", + srcs = ["merging_images_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/framework/formats:image_format_cc_proto", + "@com_google_absl//absl/strings", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:video_stream_header", + "//mediapipe/framework/port:logging", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_highgui", + "//mediapipe/framework/port:status", + "//mediapipe/framework/port:vector", + ], + alwayslink = 1, +) + +cc_library( + name = "imageframe_to_mat_calculator", + srcs = ["imageframe_to_mat_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_highgui", + ], + alwayslink = 1, +) diff --git a/mediapipe/calculators/beauty/bilateral_filter_calculator.cc b/mediapipe/calculators/beauty/bilateral_filter_calculator.cc index 18f671ba9..d5ecfd19e 100644 --- a/mediapipe/calculators/beauty/bilateral_filter_calculator.cc +++ b/mediapipe/calculators/beauty/bilateral_filter_calculator.cc @@ -12,14 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - -#include -#include -#include - -#include - #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_options.pb.h" @@ -30,26 +22,18 @@ #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/vector.h" +using namespace std; namespace mediapipe { namespace { - - constexpr char kMaskTag[] = "MASK"; - constexpr char kFaceBoxTag[] = "FACEBOX"; constexpr char kImageFrameTag[] = "IMAGE"; - constexpr char kImageNewTag[] = "IMAGE2"; - - enum - { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - NUM_ATTRIBUTES - }; + constexpr char kOutTag[] = "CVMAT"; inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } } // namespace @@ -68,24 +52,18 @@ namespace mediapipe absl::Status Close(CalculatorContext *cc) override; private: - absl::Status CreateRenderTargetCpu(CalculatorContext *cc, - std::unique_ptr &image_mat, - ImageFormat::Format *target_format); - - absl::Status RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image); - absl::Status BilateralFilter(CalculatorContext *cc, const std::vector &face_box); + absl::Status RenderToCpu(CalculatorContext *cc); + // Indicates if image frame is available as input. bool image_frame_available_ = false; int image_width_; int image_height_; cv::Mat mat_image_; - std::unique_ptr image_mat; + cv::Mat out_mat; }; REGISTER_CALCULATOR(BilateralCalculator); @@ -95,30 +73,13 @@ namespace mediapipe if (cc->Inputs().HasTag(kImageFrameTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - CHECK(cc->Outputs().HasTag(kImageNewTag)); + cc->Inputs().Tag(kImageFrameTag).Set>>(); + CHECK(cc->Outputs().HasTag(kOutTag)); } - // Data streams to render. - for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId(); - ++id) + if (cc->Outputs().HasTag(kOutTag)) { - auto tag_and_index = cc->Inputs().TagAndIndexFromId(id); - std::string tag = tag_and_index.first; - if (tag == kFaceBoxTag) - { - cc->Inputs().Get(id).Set>(); - } - else if (tag.empty()) - { - // Empty tag defaults to accepting a single object of Mat type. - cc->Inputs().Get(id).Set(); - } - } - - if (cc->Outputs().HasTag(kImageNewTag)) - { - cc->Outputs().Tag(kImageNewTag).Set(); + cc->Outputs().Tag(kOutTag).Set(); } return absl::OkStatus(); @@ -135,7 +96,7 @@ namespace mediapipe // Set the output header based on the input header (if present). const char *tag = kImageFrameTag; - const char *out_tag = kImageNewTag; + const char *out_tag = kOutTag; if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) { const auto &input_header = @@ -154,32 +115,21 @@ namespace mediapipe { return absl::OkStatus(); } - if (cc->Inputs().HasTag(kFaceBoxTag) && - cc->Inputs().Tag(kFaceBoxTag).IsEmpty()) - { - return absl::OkStatus(); - } + const std::pair> &face = + cc->Inputs().Tag(kImageFrameTag).Get>>(); // Initialize render target, drawn with OpenCV. ImageFormat::Format target_format; + const std::vector &face_box = face.second; + mat_image_ = face.first.clone(); - if (cc->Outputs().HasTag(kImageNewTag)) - { - MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); - } + image_width_ = mat_image_.cols; + image_height_ = mat_image_.rows; - mat_image_ = *image_mat.get(); - image_width_ = image_mat->cols; - image_height_ = image_mat->rows; + if (!face_box.empty()) + MP_RETURN_IF_ERROR(BilateralFilter(cc, face_box)); - const std::vector &face_box = - cc->Inputs().Tag(kFaceBoxTag).Get>(); - - MP_RETURN_IF_ERROR(BilateralFilter(cc, face_box)); - - // Copy the rendered image to output. - uchar *image_mat_ptr = image_mat->data; - MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr)); + MP_RETURN_IF_ERROR(RenderToCpu(cc)); return absl::OkStatus(); } @@ -189,92 +139,28 @@ namespace mediapipe return absl::OkStatus(); } - absl::Status BilateralCalculator::RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image) + absl::Status BilateralCalculator::BilateralFilter(CalculatorContext *cc, + const std::vector &face_box) { - auto output_frame = absl::make_unique( - target_format, image_width_, image_height_); + cv::Mat patch_wow = mat_image_(cv::Range(face_box[1], face_box[3]), + cv::Range(face_box[0], face_box[2])); - output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image, - ImageFrame::kDefaultAlignmentBoundary); + cv::cvtColor(patch_wow, patch_wow, CV_RGBA2RGB); + cv::bilateralFilter(patch_wow, out_mat, 12, 50, 50); + cv::cvtColor(out_mat, out_mat, CV_RGB2RGBA); + + return absl::OkStatus(); + } + absl::Status BilateralCalculator::RenderToCpu(CalculatorContext *cc) + { + auto out_mat_ptr = absl::make_unique(out_mat); - if (cc->Outputs().HasTag(kImageNewTag)) + if (cc->Outputs().HasTag(kOutTag)) { cc->Outputs() - .Tag(kImageNewTag) - .Add(output_frame.release(), cc->InputTimestamp()); + .Tag(kOutTag) + .Add(out_mat_ptr.release(), cc->InputTimestamp()); } - - return absl::OkStatus(); - } - - absl::Status BilateralCalculator::CreateRenderTargetCpu( - CalculatorContext *cc, std::unique_ptr &image_mat, - ImageFormat::Format *target_format) - { - if (image_frame_available_) - { - const auto &input_frame = - cc->Inputs().Tag(kImageFrameTag).Get(); - - int target_mat_type; - switch (input_frame.Format()) - { - case ImageFormat::SRGBA: - *target_format = ImageFormat::SRGBA; - target_mat_type = CV_8UC4; - break; - case ImageFormat::SRGB: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - case ImageFormat::GRAY8: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - default: - return absl::UnknownError("Unexpected image frame format."); - break; - } - - image_mat = absl::make_unique( - input_frame.Height(), input_frame.Width(), target_mat_type); - - auto input_mat = formats::MatView(&input_frame); - - if (input_frame.Format() == ImageFormat::GRAY8) - { - cv::Mat rgb_mat; - cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB); - rgb_mat.copyTo(*image_mat); - } - else - { - input_mat.copyTo(*image_mat); - } - } - else - { - image_mat = absl::make_unique( - 150, 150, CV_8UC4, - cv::Scalar::all(255)); - *target_format = ImageFormat::SRGBA; - } - - return absl::OkStatus(); - } - - absl::Status BilateralCalculator::BilateralFilter(CalculatorContext *cc, - const std::vector &face_box) - { - cv::Mat patch_face = mat_image_(cv::Range(face_box[1], face_box[3]), - cv::Range(face_box[0], face_box[2])); - cv::Mat patch_new, patch_wow; - cv::cvtColor(patch_face, patch_wow, cv::COLOR_RGBA2RGB); - cv::bilateralFilter(patch_wow, patch_new, 12, 50, 50); - cv::cvtColor(patch_new, patch_new, cv::COLOR_RGB2RGBA); - return absl::OkStatus(); } diff --git a/mediapipe/calculators/beauty/draw_lipstick_calculator.cc b/mediapipe/calculators/beauty/draw_lipstick_calculator.cc index f19816e63..85a942eb3 100644 --- a/mediapipe/calculators/beauty/draw_lipstick_calculator.cc +++ b/mediapipe/calculators/beauty/draw_lipstick_calculator.cc @@ -12,18 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - -#include -#include -#include -#include - -#include - #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" @@ -31,16 +21,18 @@ #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/vector.h" +using namespace std; namespace mediapipe { namespace { constexpr char kMaskTag[] = "MASK"; - constexpr char kImageFrameTag[] = "IMAGE"; + constexpr char kMatTag[] = "MAT"; inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } } // namespace @@ -59,21 +51,16 @@ namespace mediapipe absl::Status Close(CalculatorContext *cc) override; private: - absl::Status CreateRenderTargetCpu(CalculatorContext *cc, - std::unique_ptr &image_mat, - ImageFormat::Format *target_format); - absl::Status RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image, std::unique_ptr &image_mat); - - absl::Status DrawLipstick(CalculatorContext *cc, std::unique_ptr &image_mat, - ImageFormat::Format *target_format, - const std::unordered_map &mask_vec); + CalculatorContext *cc); + absl::Status DrawLipstick(CalculatorContext *cc, + std::unordered_map &mask_vec); // Indicates if image frame is available as input. bool image_frame_available_ = false; std::unordered_map all_masks; + cv::Mat spec_lips_mask; + cv::Mat mat_image_; }; REGISTER_CALCULATOR(DrawLipstickCalculator); @@ -81,10 +68,10 @@ namespace mediapipe { CHECK_GE(cc->Inputs().NumEntries(), 1); - if (cc->Inputs().HasTag(kImageFrameTag)) + if (cc->Inputs().HasTag(kMatTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - CHECK(cc->Outputs().HasTag(kImageFrameTag)); + cc->Inputs().Tag(kMatTag).Set(); + CHECK(cc->Outputs().HasTag(kMatTag)); } // Data streams to render. @@ -104,9 +91,13 @@ namespace mediapipe } } - if (cc->Outputs().HasTag(kImageFrameTag)) + if (cc->Outputs().HasTag(kMatTag)) { - cc->Outputs().Tag(kImageFrameTag).Set(); + cc->Outputs().Tag(kMatTag).Set(); + } + if (cc->Outputs().HasTag(kMaskTag)) + { + cc->Outputs().Tag(kMaskTag).Set(); } return absl::OkStatus(); @@ -116,13 +107,13 @@ namespace mediapipe { cc->SetOffset(TimestampDiff(0)); - if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc)) + if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc)) { image_frame_available_ = true; } // Set the output header based on the input header (if present). - const char *tag = kImageFrameTag; + const char *tag = kMatTag; if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) { const auto &input_header = @@ -136,35 +127,34 @@ namespace mediapipe absl::Status DrawLipstickCalculator::Process(CalculatorContext *cc) { - if (cc->Inputs().HasTag(kImageFrameTag) && - cc->Inputs().Tag(kImageFrameTag).IsEmpty()) + if (cc->Inputs().HasTag(kMatTag) && + cc->Inputs().Tag(kMatTag).IsEmpty()) { return absl::OkStatus(); } - // Initialize render target, drawn with OpenCV. - std::unique_ptr image_mat; ImageFormat::Format target_format; - if (cc->Outputs().HasTag(kImageFrameTag)) - { - MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); - } + const cv::Mat &input_mat = + cc->Inputs().Tag(kMatTag).Get(); + + mat_image_ = input_mat.clone(); if (cc->Inputs().HasTag(kMaskTag) && !cc->Inputs().Tag(kMaskTag).IsEmpty()) { const std::vector> &mask_vec = cc->Inputs().Tag(kMaskTag).Get>>(); + if (mask_vec.size() > 0) { - for (auto mask : mask_vec) - MP_RETURN_IF_ERROR(DrawLipstick(cc, image_mat, &target_format, mask)); + for (auto mask : mask_vec){ + MP_RETURN_IF_ERROR(DrawLipstick(cc, mask)); + } } } - // Copy the rendered image to output. - uchar *image_mat_ptr = image_mat->data; - MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat)); + + MP_RETURN_IF_ERROR(RenderToCpu(cc)); return absl::OkStatus(); } @@ -175,163 +165,71 @@ namespace mediapipe } absl::Status DrawLipstickCalculator::RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image, std::unique_ptr &image_mat) + CalculatorContext *cc) { - cv::Mat mat_image_ = *image_mat.get(); - - auto output_frame = absl::make_unique( - target_format, mat_image_.cols, mat_image_.rows); - - output_frame->CopyPixelData(target_format, mat_image_.cols, mat_image_.rows, data_image, - ImageFrame::kDefaultAlignmentBoundary); - - if (cc->Outputs().HasTag(kImageFrameTag)) + auto output_frame = absl::make_unique(mat_image_); + + if (cc->Outputs().HasTag(kMatTag)) { cc->Outputs() - .Tag(kImageFrameTag) + .Tag(kMatTag) .Add(output_frame.release(), cc->InputTimestamp()); } - return absl::OkStatus(); - } + spec_lips_mask.convertTo(spec_lips_mask, CV_32F, 1.0 / 255); + auto output_frame2 = absl::make_unique(spec_lips_mask); - absl::Status DrawLipstickCalculator::CreateRenderTargetCpu( - CalculatorContext *cc, std::unique_ptr &image_mat, - ImageFormat::Format *target_format) - { - if (image_frame_available_) + if (cc->Outputs().HasTag(kMaskTag)) { - const auto &input_frame = - cc->Inputs().Tag(kImageFrameTag).Get(); - - int target_mat_type; - switch (input_frame.Format()) - { - case ImageFormat::SRGBA: - *target_format = ImageFormat::SRGBA; - target_mat_type = CV_8UC4; - break; - case ImageFormat::SRGB: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - case ImageFormat::GRAY8: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - default: - return absl::UnknownError("Unexpected image frame format."); - break; - } - - image_mat = absl::make_unique( - input_frame.Height(), input_frame.Width(), target_mat_type); - - auto input_mat = formats::MatView(&input_frame); - - if (input_frame.Format() == ImageFormat::GRAY8) - { - cv::Mat rgb_mat; - cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB); - rgb_mat.copyTo(*image_mat); - } - else - { - input_mat.copyTo(*image_mat); - } - } - else - { - image_mat = absl::make_unique( - 150, 150, CV_8UC4, - cv::Scalar(cv::Scalar::all(255))); - *target_format = ImageFormat::SRGBA; + cc->Outputs() + .Tag(kMaskTag) + .Add(output_frame2.release(), cc->InputTimestamp()); } return absl::OkStatus(); } absl::Status DrawLipstickCalculator::DrawLipstick(CalculatorContext *cc, - std::unique_ptr &image_mat, - ImageFormat::Format *target_format, - const std::unordered_map &mask_vec) + std::unordered_map &mask_vec) { - cv::Mat mat_image__ = *image_mat.get(); - - cv::Mat spec_lips_mask, upper_lips_mask, lower_lips_mask; - spec_lips_mask = cv::Mat::zeros(mat_image__.size(), CV_32F); - upper_lips_mask = cv::Mat::zeros(mat_image__.size(), CV_32F); - lower_lips_mask = cv::Mat::zeros(mat_image__.size(), CV_32F); - - upper_lips_mask = mask_vec.find("UPPER_LIP")->second; - lower_lips_mask = mask_vec.find("LOWER_LIP")->second; + cv::Mat upper_lips_mask = mask_vec.find("UPPER_LIP")->second; + cv::Mat lower_lips_mask = mask_vec.find("LOWER_LIP")->second; spec_lips_mask = upper_lips_mask + lower_lips_mask; - spec_lips_mask.convertTo(spec_lips_mask, CV_8U); + cv::resize(spec_lips_mask, spec_lips_mask, mat_image_.size(), cv::INTER_LINEAR); - cv::resize(spec_lips_mask, spec_lips_mask, mat_image__.size(), cv::INTER_LINEAR); + cv::Rect rect = cv::boundingRect(spec_lips_mask); - std::vector x, y; - std::vector location; - - cv::findNonZero(spec_lips_mask, location); - - for (auto &i : location) + if (!rect.empty()) { - x.push_back(i.x); - y.push_back(i.y); - } - - if (!(x.empty()) && !(y.empty())) - { - - double min_y, max_y, max_x, min_x; - cv::minMaxLoc(y, &min_y, &max_y); - cv::minMaxLoc(x, &min_x, &max_x); + double min_y = rect.y, max_y = rect.y + rect.height, + max_x = rect.x + rect.width, min_x = rect.x; cv::Mat lips_crop_mask = spec_lips_mask(cv::Range(min_y, max_y), cv::Range(min_x, max_x)); lips_crop_mask.convertTo(lips_crop_mask, CV_32F, 1.0 / 255); - cv::Mat lips_crop = cv::Mat(mat_image__(cv::Range(min_y, max_y), cv::Range(min_x, max_x))); - + cv::Mat lips_crop = cv::Mat(mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x))); cv::Mat lips_blend = cv::Mat(lips_crop.size().height, lips_crop.size().width, CV_32FC4, cv::Scalar(255.0, 0, 0, 0)); - std::vector channels(4); cv::split(lips_blend, channels); channels[3] = lips_crop_mask * 20; - cv::merge(channels, lips_blend); cv::Mat tmp_lip_mask; + channels[3].convertTo(tmp_lip_mask, CV_32F, 1.0 / 255); - channels[3].convertTo(tmp_lip_mask, CV_32FC1, 1.0 / 255); - - cv::split(lips_blend, channels); - for (auto &ch : channels) - { - cv::multiply(ch, tmp_lip_mask, ch, 1.0, CV_32F); - } - cv::merge(channels, lips_blend); - + cv::merge(std::vector{tmp_lip_mask, tmp_lip_mask, tmp_lip_mask, tmp_lip_mask}, tmp_lip_mask); + cv::multiply(lips_blend, tmp_lip_mask, lips_blend, 1.0, CV_32F); cv::subtract(1.0, tmp_lip_mask, tmp_lip_mask, cv::noArray(), CV_32F); - - cv::split(lips_crop, channels); - for (auto &ch : channels) - { - cv::multiply(ch, tmp_lip_mask, ch, 1.0, CV_8U); - } - cv::merge(channels, lips_crop); - + cv::multiply(lips_crop, tmp_lip_mask, lips_crop, 1.0, CV_8U); cv::add(lips_blend, lips_crop, lips_crop, cv::noArray(), CV_8U); - lips_crop = cv::abs(lips_crop); cvtColor(lips_crop, lips_crop, cv::COLOR_RGBA2RGB); - cv::Mat slice = mat_image__(cv::Range(min_y, max_y), cv::Range(min_x, max_x)); + cv::Mat slice = mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x)); lips_crop_mask.convertTo(lips_crop_mask, slice.type()); slice.copyTo(slice, lips_crop_mask); diff --git a/mediapipe/calculators/beauty/imageframe_to_mat_calculator.cc b/mediapipe/calculators/beauty/imageframe_to_mat_calculator.cc new file mode 100644 index 000000000..62d799a8b --- /dev/null +++ b/mediapipe/calculators/beauty/imageframe_to_mat_calculator.cc @@ -0,0 +1,131 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/formats/image_format.pb.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/status.h" + +using namespace std; +namespace mediapipe +{ + namespace api2 + { + class ImageFrameToMatCalculator : public Node + { + public: + static constexpr Input kImageFrame{"IMAGE"}; + static constexpr Output kOut{"MAT"}; + + MEDIAPIPE_NODE_CONTRACT(kImageFrame, kOut); + + static absl::Status UpdateContract(CalculatorContract *cc) + { + RET_CHECK(kOut(cc).IsConnected()) + << "At least one output stream is expected."; + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext *cc) + { + // Initialize render target, drawn with OpenCV. + std::unique_ptr image_mat; + ImageFormat::Format target_format; + + MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); + + MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat)); + + return absl::OkStatus(); + } + + absl::Status Close(CalculatorContext *cc) + { + return absl::OkStatus(); + } + + absl::Status RenderToCpu( + CalculatorContext *cc, const ImageFormat::Format &target_format, std::unique_ptr &image_mat) + { + auto output_frame = absl::make_unique(*image_mat); + + kOut(cc).Send(std::move(output_frame)); + + return absl::OkStatus(); + } + + absl::Status CreateRenderTargetCpu( + CalculatorContext *cc, std::unique_ptr &image_mat, + ImageFormat::Format *target_format) + { + if (!kImageFrame(cc).IsEmpty()) + { + const auto &input_frame = *kImageFrame(cc); + + int target_mat_type; + switch (input_frame.Format()) + { + case ImageFormat::SRGBA: + *target_format = ImageFormat::SRGBA; + target_mat_type = CV_8UC4; + break; + case ImageFormat::SRGB: + *target_format = ImageFormat::SRGB; + target_mat_type = CV_8UC3; + break; + case ImageFormat::GRAY8: + *target_format = ImageFormat::SRGB; + target_mat_type = CV_8UC3; + break; + default: + return absl::UnknownError("Unexpected image frame format."); + break; + } + + image_mat = absl::make_unique( + input_frame.Height(), input_frame.Width(), target_mat_type); + + auto input_mat = formats::MatView(&input_frame); + + if (input_frame.Format() == ImageFormat::GRAY8) + { + cv::Mat rgb_mat; + cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB); + rgb_mat.copyTo(*image_mat); + } + else + { + input_mat.copyTo(*image_mat); + } + } + else + { + image_mat = absl::make_unique( + 1920, 1080, CV_8UC4, + cv::Scalar(cv::Scalar::all(255))); + *target_format = ImageFormat::SRGBA; + } + + return absl::OkStatus(); + } + }; + MEDIAPIPE_REGISTER_NODE(ImageFrameToMatCalculator); // namespace mediapipe + } +} diff --git a/mediapipe/calculators/beauty/merging_images_calculator.cc b/mediapipe/calculators/beauty/merging_images_calculator.cc new file mode 100644 index 000000000..001c37588 --- /dev/null +++ b/mediapipe/calculators/beauty/merging_images_calculator.cc @@ -0,0 +1,179 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/str_cat.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_options.pb.h" +#include "mediapipe/framework/formats/image_format.pb.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/video_stream_header.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/status.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/vector.h" + +using namespace std; + +namespace mediapipe +{ + namespace + { + + constexpr char kMaskTag[] = "MASK"; + constexpr char kMatTag[] = "MAT"; + constexpr char kImageFrameTag[] = "IMAGE"; + } // namespace + + class MergeImagesCalculator : public CalculatorBase + { + public: + MergeImagesCalculator() = default; + ~MergeImagesCalculator() override = default; + + static absl::Status GetContract(CalculatorContract *cc); + + // From Calculator. + absl::Status Open(CalculatorContext *cc) override; + absl::Status Process(CalculatorContext *cc) override; + absl::Status Close(CalculatorContext *cc) override; + + private: + absl::Status SmoothEnd(CalculatorContext *cc, + const std::vector &face_box); + + cv::Mat mat_image_; + }; + REGISTER_CALCULATOR(MergeImagesCalculator); + + absl::Status MergeImagesCalculator::GetContract(CalculatorContract *cc) + { + CHECK_GE(cc->Inputs().NumEntries(), 1); + CHECK(cc->Outputs().HasTag(kImageFrameTag)); + + // Data streams to render. + for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId(); + ++id) + { + auto tag_and_index = cc->Inputs().TagAndIndexFromId(id); + std::string tag = tag_and_index.first; + if (tag == kMatTag) + { + cc->Inputs().Get(id).Set(); + } + else + { + // Every other tag defaults to accepting a single object of Mat type. + cc->Inputs().Get(id).Set(); + } + } + + if (cc->Outputs().HasTag(kImageFrameTag)) + { + cc->Outputs().Tag(kImageFrameTag).Set(); + } + + return absl::OkStatus(); + } + + absl::Status MergeImagesCalculator::Open(CalculatorContext *cc) + { + cc->SetOffset(TimestampDiff(0)); + + // Set the output header based on the input header (if present). + const char *tag = kMatTag; + if (!cc->Inputs().Tag(tag).Header().IsEmpty()) + { + const auto &input_header = + cc->Inputs().Tag(tag).Header().Get(); + auto *output_video_header = new VideoHeader(input_header); + cc->Outputs().Tag(tag).SetHeader(Adopt(output_video_header)); + } + + return absl::OkStatus(); + } + + absl::Status MergeImagesCalculator::Process(CalculatorContext *cc) + { + if (cc->Inputs().HasTag(kMatTag) && + cc->Inputs().Tag(kMatTag).IsEmpty()) + { + return absl::OkStatus(); + } + + const auto &input_mat_c = cc->Inputs().Get(kMatTag, 0).Get(); + cv::Mat input_mat = input_mat_c.clone(); + cv::Mat input_mat2 = input_mat.clone(); + cv::Mat mask = cc->Inputs().Get(kMaskTag, 0).Get(); + + cv::cvtColor(mask, mask, cv::COLOR_GRAY2RGBA); + + input_mat.convertTo(input_mat, CV_32F); + mat_image_ = mask.mul(input_mat); + int image_number = cc->Inputs().NumEntries() / 2; + cv::Mat all_masks = mask; + + for (int i = 1; i < image_number; i++) + { + const auto &input_mat_c = cc->Inputs().Get(kMatTag, i).Get(); + cv::Mat input_mat = input_mat_c.clone(); + mask = cc->Inputs().Get(kMaskTag, i).Get(); + cv::cvtColor(mask, mask, cv::COLOR_GRAY2RGBA); + input_mat.convertTo(input_mat, CV_32F); + mat_image_ += mask.mul(input_mat); + all_masks += mask; + + } + + input_mat2.convertTo(input_mat2, CV_32F); + cv::threshold(all_masks, all_masks, 1, 255, CV_THRESH_TRUNC); + input_mat2 = input_mat2.mul(1-(all_masks*255)); + + input_mat2.convertTo(input_mat2, CV_8U); + mat_image_.convertTo(mat_image_, CV_8U); + + mat_image_ += input_mat2; + + std::unique_ptr image_mat = absl::make_unique( + mat_image_.rows, mat_image_.cols, ImageFormat::SRGBA); + mat_image_.convertTo(mat_image_, CV_8U); + mat_image_.copyTo(*image_mat); + + uchar *image_mat_ptr = image_mat->data; + + // Copy the rendered image to output. + auto output_frame = absl::make_unique( + ImageFormat::SRGBA, mat_image_.cols, mat_image_.rows); + + output_frame->CopyPixelData(ImageFormat::SRGBA, mat_image_.cols, mat_image_.rows, image_mat_ptr, + ImageFrame::kDefaultAlignmentBoundary); + + if (cc->Outputs().HasTag(kImageFrameTag)) + { + cc->Outputs() + .Tag(kImageFrameTag) + .Add(output_frame.release(), cc->InputTimestamp()); + } + + return absl::OkStatus(); + } + + absl::Status MergeImagesCalculator::Close(CalculatorContext *cc) + { + return absl::OkStatus(); + } +} // namespace mediapipe diff --git a/mediapipe/calculators/beauty/smooth_face1_calculator.cc b/mediapipe/calculators/beauty/smooth_face1_calculator.cc index 416e9d424..24be75bf8 100644 --- a/mediapipe/calculators/beauty/smooth_face1_calculator.cc +++ b/mediapipe/calculators/beauty/smooth_face1_calculator.cc @@ -12,15 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - -#include -#include -#include -#include - -#include - #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_options.pb.h" @@ -31,25 +22,20 @@ #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/vector.h" +using namespace std; namespace mediapipe { namespace { constexpr char kMaskTag[] = "MASK"; - constexpr char kFaceBoxTag[] = "FACEBOX"; - constexpr char kImageFrameTag[] = "IMAGE"; - - enum - { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - NUM_ATTRIBUTES - }; + constexpr char kMatTag[] = "MAT"; + constexpr char kFaceTag[] = "FACE"; inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } } // namespace @@ -68,13 +54,8 @@ namespace mediapipe absl::Status Close(CalculatorContext *cc) override; private: - absl::Status CreateRenderTargetCpu(CalculatorContext *cc, - std::unique_ptr &image_mat, - ImageFormat::Format *target_format); - absl::Status RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image); + CalculatorContext *cc); absl::Status SmoothFace(CalculatorContext *cc, const std::unordered_map &mask_vec, @@ -90,7 +71,7 @@ namespace mediapipe cv::Mat mat_image_; cv::Mat not_full_face; std::vector face_box; - std::unique_ptr image_mat; + std::pair> face; }; REGISTER_CALCULATOR(SmoothFaceCalculator1); @@ -98,10 +79,10 @@ namespace mediapipe { CHECK_GE(cc->Inputs().NumEntries(), 1); - if (cc->Inputs().HasTag(kImageFrameTag)) + if (cc->Inputs().HasTag(kMatTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - CHECK(cc->Outputs().HasTag(kImageFrameTag)); + cc->Inputs().Tag(kMatTag).Set(); + CHECK(cc->Outputs().HasTag(kMatTag)); CHECK(cc->Outputs().HasTag(kMaskTag)); } @@ -121,23 +102,23 @@ namespace mediapipe cc->Inputs().Get(id).Set(); } - if (tag == kFaceBoxTag) + if (tag == kFaceTag) { cc->Inputs().Get(id).Set>>(); } } - if (cc->Outputs().HasTag(kImageFrameTag)) + if (cc->Outputs().HasTag(kMatTag)) { - cc->Outputs().Tag(kImageFrameTag).Set(); + cc->Outputs().Tag(kMatTag).Set(); } if (cc->Outputs().HasTag(kMaskTag)) { cc->Outputs().Tag(kMaskTag).Set(); } - if (cc->Outputs().HasTag(kFaceBoxTag)) + if (cc->Outputs().HasTag(kFaceTag)) { - cc->Outputs().Tag(kFaceBoxTag).Set>(); + cc->Outputs().Tag(kFaceTag).Set>>(); } return absl::OkStatus(); @@ -147,13 +128,13 @@ namespace mediapipe { cc->SetOffset(TimestampDiff(0)); - if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc)) + if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc)) { image_frame_available_ = true; } // Set the output header based on the input header (if present). - const char *tag = kImageFrameTag; + const char *tag = kMatTag; if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) { const auto &input_header = @@ -167,8 +148,8 @@ namespace mediapipe absl::Status SmoothFaceCalculator1::Process(CalculatorContext *cc) { - if (cc->Inputs().HasTag(kImageFrameTag) && - cc->Inputs().Tag(kImageFrameTag).IsEmpty()) + if (cc->Inputs().HasTag(kMatTag) && + cc->Inputs().Tag(kMatTag).IsEmpty()) { return absl::OkStatus(); } @@ -176,35 +157,33 @@ namespace mediapipe // Initialize render target, drawn with OpenCV. ImageFormat::Format target_format; - if (cc->Outputs().HasTag(kImageFrameTag)) - { - MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); - } + const cv::Mat &input_mat = + cc->Inputs().Tag(kMatTag).Get(); - mat_image_ = *image_mat.get(); - image_width_ = image_mat->cols; - image_height_ = image_mat->rows; + mat_image_ = input_mat.clone(); + + image_width_ = input_mat.cols; + image_height_ = input_mat.rows; if (cc->Inputs().HasTag(kMaskTag) && !cc->Inputs().Tag(kMaskTag).IsEmpty() && - cc->Inputs().HasTag(kFaceBoxTag) && - !cc->Inputs().Tag(kFaceBoxTag).IsEmpty()) + cc->Inputs().HasTag(kFaceTag) && + !cc->Inputs().Tag(kFaceTag).IsEmpty()) { - const std::vector> &mask_vec = - cc->Inputs().Tag(kMaskTag).Get>>(); + const std::vector> &mask_vec = + cc->Inputs().Tag(kMaskTag).Get>>(); - const std::vector> &face_boxes = - cc->Inputs().Tag(kFaceBoxTag).Get>>(); + const std::vector> &face_boxes = + cc->Inputs().Tag(kFaceTag).Get>>(); - if (mask_vec.size() > 0 && face_boxes.size() > 0) - { - for (int i = 0; i < mask_vec.size(); i++) - MP_RETURN_IF_ERROR(SmoothFace(cc, mask_vec[i], face_boxes[i])); - } + if (mask_vec.size() > 0 && face_boxes.size() > 0) + { + for (int i = 0; i < mask_vec.size(); i++) + MP_RETURN_IF_ERROR(SmoothFace(cc, mask_vec[i], face_boxes[i])); + } } - // Copy the rendered image to output. - uchar *image_mat_ptr = image_mat->data; - MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr)); + + MP_RETURN_IF_ERROR(RenderToCpu(cc)); return absl::OkStatus(); } @@ -215,22 +194,18 @@ namespace mediapipe } absl::Status SmoothFaceCalculator1::RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image) + CalculatorContext *cc) { - auto output_frame1 = absl::make_unique( - target_format, image_width_, image_height_); + auto output_frame1 = absl::make_unique(mat_image_); - output_frame1->CopyPixelData(target_format, image_width_, image_height_, data_image, - ImageFrame::kDefaultAlignmentBoundary); - - if (cc->Outputs().HasTag(kImageFrameTag)) + if (cc->Outputs().HasTag(kMatTag)) { cc->Outputs() - .Tag(kImageFrameTag) + .Tag(kMatTag) .Add(output_frame1.release(), cc->InputTimestamp()); } + not_full_face.convertTo(not_full_face, CV_32F, 1.0 / 255); auto output_frame2 = absl::make_unique(not_full_face); if (cc->Outputs().HasTag(kMaskTag)) @@ -240,71 +215,15 @@ namespace mediapipe .Add(output_frame2.release(), cc->InputTimestamp()); } - auto output_frame3 = absl::make_unique>(face_box); + auto output_frame3 = absl::make_unique>>( + face); - if (cc->Outputs().HasTag(kFaceBoxTag)) + if (cc->Outputs().HasTag(kFaceTag)) { cc->Outputs() - .Tag(kFaceBoxTag) + .Tag(kFaceTag) .Add(output_frame3.release(), cc->InputTimestamp()); } - - return absl::OkStatus(); - } - - absl::Status SmoothFaceCalculator1::CreateRenderTargetCpu( - CalculatorContext *cc, std::unique_ptr &image_mat, - ImageFormat::Format *target_format) - { - if (image_frame_available_) - { - const auto &input_frame = - cc->Inputs().Tag(kImageFrameTag).Get(); - - int target_mat_type; - switch (input_frame.Format()) - { - case ImageFormat::SRGBA: - *target_format = ImageFormat::SRGBA; - target_mat_type = CV_8UC4; - break; - case ImageFormat::SRGB: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - case ImageFormat::GRAY8: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - default: - return absl::UnknownError("Unexpected image frame format."); - break; - } - - image_mat = absl::make_unique( - input_frame.Height(), input_frame.Width(), target_mat_type); - - auto input_mat = formats::MatView(&input_frame); - - if (input_frame.Format() == ImageFormat::GRAY8) - { - cv::Mat rgb_mat; - cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB); - rgb_mat.copyTo(*image_mat); - } - else - { - input_mat.copyTo(*image_mat); - } - } - else - { - image_mat = absl::make_unique( - 150, 150, CV_8UC4, - cv::Scalar::all(255)); - *target_format = ImageFormat::SRGBA; - } - return absl::OkStatus(); } @@ -379,11 +298,11 @@ namespace mediapipe } absl::Status SmoothFaceCalculator1::SmoothFace(CalculatorContext *cc, - const std::unordered_map &mask_vec, - const std::tuple &face_boxx) + const std::unordered_map &mask_vec, + const std::tuple &face_boxx) { not_full_face = mask_vec.find("FACE_OVAL")->second.clone() - -// predict_forehead_mask(mask_vec, std::get<1>(face_boxx)) - + // predict_forehead_mask(mask_vec, std::get<1>(face_boxx)) - mask_vec.find("LEFT_EYE")->second.clone() - mask_vec.find("RIGHT_EYE")->second.clone() - mask_vec.find("LEFT_BROW")->second.clone() - @@ -395,26 +314,18 @@ namespace mediapipe mat_image_.size(), 0, 0, cv::INTER_LINEAR); - std::vector x, y; - std::vector location; + cv::Rect rect = cv::boundingRect(not_full_face); - cv::findNonZero(not_full_face, location); - - double min_y, min_x, max_x, max_y; - - for (auto &i : location) + if (!rect.empty()) { - x.push_back(i.x); - y.push_back(i.y); + double min_y = rect.y, max_y = rect.y + rect.height, + max_x = rect.x + rect.width, min_x = rect.x; + face.second.push_back(min_x); + face.second.push_back(min_y); + face.second.push_back(max_x); + face.second.push_back(max_y); + face.first = mat_image_; } - - cv::minMaxLoc(x, &min_x, &max_x); - cv::minMaxLoc(y, &min_y, &max_y); - face_box.push_back(min_x); - face_box.push_back(min_y); - face_box.push_back(max_x); - face_box.push_back(max_y); - return absl::OkStatus(); } diff --git a/mediapipe/calculators/beauty/smooth_face2_calculator.cc b/mediapipe/calculators/beauty/smooth_face2_calculator.cc index 7d9700125..48a946f53 100644 --- a/mediapipe/calculators/beauty/smooth_face2_calculator.cc +++ b/mediapipe/calculators/beauty/smooth_face2_calculator.cc @@ -12,15 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - -#include -#include -#include -#include - -#include - #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_options.pb.h" @@ -31,10 +22,13 @@ #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/vector.h" +using namespace std; + namespace mediapipe { namespace @@ -42,16 +36,9 @@ namespace mediapipe constexpr char kMaskTag[] = "MASK"; constexpr char kFaceBoxTag[] = "FACEBOX"; - constexpr char kImageFrameTag[] = "IMAGE"; + constexpr char kMatTag[] = "MAT"; constexpr char kImageNewTag[] = "IMAGE2"; - enum - { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - NUM_ATTRIBUTES - }; - inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } } // namespace @@ -69,17 +56,11 @@ namespace mediapipe absl::Status Close(CalculatorContext *cc) override; private: - absl::Status CreateRenderTargetCpu(CalculatorContext *cc, - std::unique_ptr &image_mat, - std::unique_ptr &new_mat, - ImageFormat::Format *target_format); - absl::Status RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image); + CalculatorContext *cc); absl::Status SmoothEnd(CalculatorContext *cc, - const std::vector &face_box); + const std::vector &face_box); // Indicates if image frame is available as input. bool image_frame_available_ = false; @@ -89,9 +70,6 @@ namespace mediapipe cv::Mat mat_image_; cv::Mat new_image_; cv::Mat not_full_face; - std::unique_ptr image_mat; - std::unique_ptr new_mat; - std::unique_ptr nff_mat; }; REGISTER_CALCULATOR(SmoothFaceCalculator2); @@ -99,40 +77,28 @@ namespace mediapipe { CHECK_GE(cc->Inputs().NumEntries(), 1); - if (cc->Inputs().HasTag(kImageFrameTag)) + if (cc->Inputs().HasTag(kMatTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - CHECK(cc->Outputs().HasTag(kImageFrameTag)); + cc->Inputs().Tag(kMatTag).Set(); + CHECK(cc->Outputs().HasTag(kMatTag)); } if (cc->Inputs().HasTag(kImageNewTag)) { - cc->Inputs().Tag(kImageNewTag).Set(); + cc->Inputs().Tag(kImageNewTag).Set(); } if (cc->Inputs().HasTag(kMaskTag)) { cc->Inputs().Tag(kMaskTag).Set(); } - // Data streams to render. - for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId(); - ++id) + if (cc->Inputs().HasTag(kFaceBoxTag)) { - auto tag_and_index = cc->Inputs().TagAndIndexFromId(id); - std::string tag = tag_and_index.first; - if (tag == kFaceBoxTag) - { - cc->Inputs().Get(id).Set>(); - } - else if (tag.empty()) - { - // Empty tag defaults to accepting a single object of Mat type. - cc->Inputs().Get(id).Set(); - } + cc->Inputs().Tag(kFaceBoxTag).Set>>(); } - if (cc->Outputs().HasTag(kImageFrameTag)) + if (cc->Outputs().HasTag(kMatTag)) { - cc->Outputs().Tag(kImageFrameTag).Set(); + cc->Outputs().Tag(kMatTag).Set(); } return absl::OkStatus(); @@ -142,13 +108,13 @@ namespace mediapipe { cc->SetOffset(TimestampDiff(0)); - if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc)) + if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc)) { image_frame_available_ = true; } // Set the output header based on the input header (if present). - const char *tag = kImageFrameTag; + const char *tag = kMatTag; if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) { const auto &input_header = @@ -162,8 +128,9 @@ namespace mediapipe absl::Status SmoothFaceCalculator2::Process(CalculatorContext *cc) { - if (cc->Inputs().HasTag(kImageFrameTag) && - cc->Inputs().Tag(kImageFrameTag).IsEmpty()) + + if (cc->Inputs().HasTag(kMatTag) && + cc->Inputs().Tag(kMatTag).IsEmpty()) { return absl::OkStatus(); } @@ -183,28 +150,33 @@ namespace mediapipe return absl::OkStatus(); } - // Initialize render target, drawn with OpenCV. - ImageFormat::Format target_format; + not_full_face = cc->Inputs().Tag(kMaskTag).Get(); + not_full_face.convertTo(not_full_face, CV_8U, 255); - if (cc->Outputs().HasTag(kImageFrameTag)) + const cv::Mat &input_mat = + cc->Inputs().Tag(kMatTag).Get(); + + mat_image_ = input_mat.clone(); + + const cv::Mat &input_new = + cc->Inputs().Tag(kImageNewTag).Get(); + + new_image_ = input_new.clone(); + + image_width_ = input_mat.cols; + image_height_ = input_mat.rows; + + const auto &face_box_pair = + cc->Inputs().Tag(kFaceBoxTag).Get>>(); + + const auto &face_box = face_box_pair.second; + + if (!face_box.empty()) { - MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, new_mat, &target_format)); + MP_RETURN_IF_ERROR(SmoothEnd(cc, face_box)); } - not_full_face = cc->Inputs().Tag(kMaskTag).Get(); - new_image_ = *new_mat.get(); - mat_image_ = *image_mat.get(); - image_width_ = image_mat->cols; - image_height_ = image_mat->rows; - - const std::vector &face_box = - cc->Inputs().Tag(kFaceBoxTag).Get>(); - - MP_RETURN_IF_ERROR(SmoothEnd(cc, face_box)); - - // Copy the rendered image to output. - uchar *image_mat_ptr = image_mat->data; - MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr)); + MP_RETURN_IF_ERROR(RenderToCpu(cc)); return absl::OkStatus(); } @@ -215,119 +187,36 @@ namespace mediapipe } absl::Status SmoothFaceCalculator2::RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image) + CalculatorContext *cc) { - auto output_frame = absl::make_unique( - target_format, image_width_, image_height_); + auto output_frame = absl::make_unique(mat_image_); - output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image, - ImageFrame::kDefaultAlignmentBoundary); - - if (cc->Outputs().HasTag(kImageFrameTag)) + if (cc->Outputs().HasTag(kMatTag)) { cc->Outputs() - .Tag(kImageFrameTag) + .Tag(kMatTag) .Add(output_frame.release(), cc->InputTimestamp()); } return absl::OkStatus(); } - absl::Status SmoothFaceCalculator2::CreateRenderTargetCpu( - CalculatorContext *cc, - std::unique_ptr &image_mat, - std::unique_ptr &new_mat, - ImageFormat::Format *target_format) - { - if (image_frame_available_) - { - const auto &input_frame = - cc->Inputs().Tag(kImageFrameTag).Get(); - const auto &new_frame = - cc->Inputs().Tag(kImageNewTag).Get(); - - int target_mat_type; - switch (input_frame.Format()) - { - case ImageFormat::SRGBA: - *target_format = ImageFormat::SRGBA; - target_mat_type = CV_8UC4; - break; - case ImageFormat::SRGB: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - case ImageFormat::GRAY8: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - default: - return absl::UnknownError("Unexpected image frame format."); - break; - } - - image_mat = absl::make_unique( - input_frame.Height(), input_frame.Width(), target_mat_type); - - auto input_mat = formats::MatView(&input_frame); - - new_mat = absl::make_unique( - new_frame.Height(), new_frame.Width(), target_mat_type); - - auto new_input_mat = formats::MatView(&new_frame); - - if (input_frame.Format() == ImageFormat::GRAY8) - { - cv::Mat rgb_mat; - cv::Mat rgb_mat2; - cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB); - rgb_mat.copyTo(*image_mat); - cv::cvtColor(new_input_mat, rgb_mat2, CV_GRAY2RGB); - rgb_mat2.copyTo(*new_mat); - } - else - { - input_mat.copyTo(*image_mat); - new_input_mat.copyTo(*new_mat); - } - } - else - { - image_mat = absl::make_unique( - 150, 150, CV_8UC4, - cv::Scalar::all(255)); - nff_mat = absl::make_unique( - 150, 150, CV_8UC4, - cv::Scalar::all(255)); - new_mat = absl::make_unique( - 150, 150, CV_8UC4, - cv::Scalar::all(255)); - *target_format = ImageFormat::SRGBA; - } - - return absl::OkStatus(); - } - - absl::Status SmoothFaceCalculator2::SmoothEnd(CalculatorContext *cc, const std::vector &face_box) { - cv::Mat patch_face = mat_image_(cv::Range(face_box[1], face_box[3]), + cv::Mat patch_face = mat_image_(cv::Range(face_box[1], face_box[3]), cv::Range(face_box[0], face_box[2])); - cv::Mat patch_new = new_image_(cv::Range(face_box[1], face_box[3]), - cv::Range(face_box[0], face_box[2])); - cv::Mat patch_nff = not_full_face(cv::Range(face_box[1], face_box[3]), + cv::Mat patch_nff = not_full_face(cv::Range(face_box[1], face_box[3]), cv::Range(face_box[0], face_box[2])); cv::Mat patch_new_nff, patch_new_mask, patch, patch_face_nff; - patch_new.copyTo(patch_new_nff, patch_nff); + new_image_.copyTo(patch_new_nff, patch_nff); patch_face.copyTo(patch_face_nff, patch_nff); cv::cvtColor(patch_face_nff, patch_face_nff, cv::COLOR_RGBA2RGB); cv::cvtColor(patch_new_nff, patch_new_nff, cv::COLOR_RGBA2RGB); - + patch_new_mask = 0.85 * patch_new_nff + 0.15 * patch_face_nff; patch = cv::min(255, patch_new_mask); diff --git a/mediapipe/calculators/beauty/whiten_teeth_calculator.cc b/mediapipe/calculators/beauty/whiten_teeth_calculator.cc index a96666d9d..48e374619 100644 --- a/mediapipe/calculators/beauty/whiten_teeth_calculator.cc +++ b/mediapipe/calculators/beauty/whiten_teeth_calculator.cc @@ -12,16 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include - -#include -#include - -#include - #include "absl/strings/str_cat.h" #include "mediapipe/framework/calculator_framework.h" -#include "mediapipe/framework/calculator_options.pb.h" #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" @@ -29,17 +21,19 @@ #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/vector.h" +using namespace std; namespace mediapipe { namespace { constexpr char kMaskTag[] = "MASK"; - constexpr char kImageFrameTag[] = "IMAGE"; + constexpr char kMatTag[] = "MAT"; inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } } // namespace @@ -58,20 +52,15 @@ namespace mediapipe absl::Status Close(CalculatorContext *cc) override; private: - absl::Status CreateRenderTargetCpu(CalculatorContext *cc, - std::unique_ptr &image_mat, - ImageFormat::Format *target_format); - absl::Status RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image, std::unique_ptr &image_mat); + CalculatorContext *cc); - absl::Status WhitenTeeth(CalculatorContext *cc, ImageFormat::Format *target_format, + absl::Status WhitenTeeth(CalculatorContext *cc, const std::unordered_map &mask_vec); // Indicates if image frame is available as input. bool image_frame_available_ = false; - std::unique_ptr image_mat; + cv::Mat mouth; cv::Mat mat_image_; int image_width_; int image_height_; @@ -82,10 +71,10 @@ namespace mediapipe { CHECK_GE(cc->Inputs().NumEntries(), 1); - if (cc->Inputs().HasTag(kImageFrameTag)) + if (cc->Inputs().HasTag(kMatTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); - CHECK(cc->Outputs().HasTag(kImageFrameTag)); + cc->Inputs().Tag(kMatTag).Set(); + CHECK(cc->Outputs().HasTag(kMatTag)); } // Data streams to render. @@ -105,9 +94,13 @@ namespace mediapipe } } - if (cc->Outputs().HasTag(kImageFrameTag)) + if (cc->Outputs().HasTag(kMatTag)) { - cc->Outputs().Tag(kImageFrameTag).Set(); + cc->Outputs().Tag(kMatTag).Set(); + } + if (cc->Outputs().HasTag(kMaskTag)) + { + cc->Outputs().Tag(kMaskTag).Set(); } return absl::OkStatus(); @@ -117,16 +110,13 @@ namespace mediapipe { cc->SetOffset(TimestampDiff(0)); - if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc)) + if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc)) { image_frame_available_ = true; } - else - { - } // Set the output header based on the input header (if present). - const char *tag = kImageFrameTag; + const char *tag = kMatTag; if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) { const auto &input_header = @@ -140,22 +130,21 @@ namespace mediapipe absl::Status WhitenTeethCalculator::Process(CalculatorContext *cc) { - if (cc->Inputs().HasTag(kImageFrameTag) && - cc->Inputs().Tag(kImageFrameTag).IsEmpty()) + if (cc->Inputs().HasTag(kMatTag) && + cc->Inputs().Tag(kMatTag).IsEmpty()) { return absl::OkStatus(); } - // Initialize render target, drawn with OpenCV. ImageFormat::Format target_format; - if (cc->Outputs().HasTag(kImageFrameTag)) - { - MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); - } - mat_image_ = *image_mat.get(); - image_width_ = image_mat->cols; - image_height_ = image_mat->rows; + const cv::Mat &input_mat = + cc->Inputs().Tag(kMatTag).Get(); + + mat_image_ = input_mat.clone(); + + image_width_ = input_mat.cols; + image_height_ = input_mat.rows; if (cc->Inputs().HasTag(kMaskTag) && !cc->Inputs().Tag(kMaskTag).IsEmpty()) @@ -165,12 +154,11 @@ namespace mediapipe if (mask_vec.size() > 0) { for (auto mask : mask_vec) - MP_RETURN_IF_ERROR(WhitenTeeth(cc, &target_format, mask)); + MP_RETURN_IF_ERROR(WhitenTeeth(cc, mask)); } } - // Copy the rendered image to output. - uchar *image_mat_ptr = image_mat->data; - MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat)); + + MP_RETURN_IF_ERROR(RenderToCpu(cc)); return absl::OkStatus(); } @@ -181,124 +169,57 @@ namespace mediapipe } absl::Status WhitenTeethCalculator::RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image, std::unique_ptr &image_mat) + CalculatorContext *cc) { + auto output_frame = absl::make_unique(mat_image_); - cv::Mat mat_image_ = *image_mat.get(); - - auto output_frame = absl::make_unique( - target_format, image_width_, image_height_); - - output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image, - ImageFrame::kDefaultAlignmentBoundary); - - if (cc->Outputs().HasTag(kImageFrameTag)) + if (cc->Outputs().HasTag(kMatTag)) { cc->Outputs() - .Tag(kImageFrameTag) + .Tag(kMatTag) .Add(output_frame.release(), cc->InputTimestamp()); } - return absl::OkStatus(); - } + mouth.convertTo(mouth, CV_32F, 255); + auto output_frame2 = absl::make_unique(mouth); - absl::Status WhitenTeethCalculator::CreateRenderTargetCpu( - CalculatorContext *cc, std::unique_ptr &image_mat, - ImageFormat::Format *target_format) - { - if (image_frame_available_) + if (cc->Outputs().HasTag(kMaskTag)) { - const auto &input_frame = - cc->Inputs().Tag(kImageFrameTag).Get(); - - int target_mat_type; - switch (input_frame.Format()) - { - case ImageFormat::SRGBA: - *target_format = ImageFormat::SRGBA; - target_mat_type = CV_8UC4; - break; - case ImageFormat::SRGB: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - case ImageFormat::GRAY8: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - default: - return absl::UnknownError("Unexpected image frame format."); - break; - } - - image_mat = absl::make_unique( - input_frame.Height(), input_frame.Width(), target_mat_type); - - auto input_mat = formats::MatView(&input_frame); - - if (input_frame.Format() == ImageFormat::GRAY8) - { - cv::Mat rgb_mat; - cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB); - rgb_mat.copyTo(*image_mat); - } - else - { - input_mat.copyTo(*image_mat); - } - } - else - { - image_mat = absl::make_unique( - 150, 150, CV_8UC4, - cv::Scalar(255, 255, - 255)); - *target_format = ImageFormat::SRGBA; + cc->Outputs() + .Tag(kMaskTag) + .Add(output_frame2.release(), cc->InputTimestamp()); } return absl::OkStatus(); } absl::Status WhitenTeethCalculator::WhitenTeeth(CalculatorContext *cc, - ImageFormat::Format *target_format, const std::unordered_map &mask_vec) { - cv::Mat mouth_mask, mouth; - mouth_mask = cv::Mat::zeros(mat_image_.size(), CV_32F); - - mouth_mask = mask_vec.find("MOUTH_INSIDE")->second.clone(); + cv::Mat mouth_mask = mask_vec.find("MOUTH_INSIDE")->second.clone(); cv::resize(mouth_mask, mouth, mat_image_.size(), cv::INTER_LINEAR); - std::vector x, y; - std::vector location; + cv::Rect rect = cv::boundingRect(mouth); - cv::findNonZero(mouth, location); - - for (auto &i : location) + if (!rect.empty()) { - x.push_back(i.x); - y.push_back(i.y); - } + double mouth_min_y = rect.y, mouth_max_y = rect.y + rect.height, + mouth_max_x = rect.x + rect.width, mouth_min_x = rect.x; - if (!(x.empty()) && !(y.empty())) - { - double mouth_min_y, mouth_max_y, mouth_max_x, mouth_min_x; - cv::minMaxLoc(y, &mouth_min_y, &mouth_max_y); - cv::minMaxLoc(x, &mouth_min_x, &mouth_max_x); double mh = mouth_max_y - mouth_min_y; double mw = mouth_max_x - mouth_min_x; - cv::Mat mouth_crop_mask; + mouth.convertTo(mouth, CV_32F, 1.0 / 255); mouth.convertTo(mouth, CV_32F, 1.0 / 255); + if (mh / mw > 0.17) { mouth_min_y = static_cast(std::max(mouth_min_y - mh * 0.1, 0.0)); mouth_max_y = static_cast(std::min(mouth_max_y + mh * 0.1, (double)image_height_)); mouth_min_x = static_cast(std::max(mouth_min_x - mw * 0.1, 0.0)); mouth_max_x = static_cast(std::min(mouth_max_x + mw * 0.1, (double)image_width_)); - mouth_crop_mask = mouth(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)); + cv::Mat mouth_crop_mask = mouth(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)); cv::Mat img_hsv, tmp_mask, img_hls; cv::cvtColor(mat_image_(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)), img_hsv, cv::COLOR_RGBA2RGB); diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index 05b5934b5..404a0383b 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -194,6 +194,7 @@ cc_library( "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:opencv_core", "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", diff --git a/mediapipe/calculators/core/begin_loop_calculator.cc b/mediapipe/calculators/core/begin_loop_calculator.cc index 5bf8e65fc..20e995866 100644 --- a/mediapipe/calculators/core/begin_loop_calculator.cc +++ b/mediapipe/calculators/core/begin_loop_calculator.cc @@ -16,6 +16,7 @@ #include +#include "mediapipe/framework/port/opencv_core_inc.h" #include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/matrix.h" @@ -46,6 +47,9 @@ REGISTER_CALCULATOR(BeginLoopDetectionCalculator); typedef BeginLoopCalculator> BeginLoopMatrixCalculator; REGISTER_CALCULATOR(BeginLoopMatrixCalculator); +typedef BeginLoopCalculator>> BeginLoopPointsCalculator; +REGISTER_CALCULATOR(BeginLoopPointsCalculator); + // A calculator to process std::vector>. typedef BeginLoopCalculator>> BeginLoopMatrixVectorCalculator; diff --git a/mediapipe/calculators/core/end_loop_calculator.cc b/mediapipe/calculators/core/end_loop_calculator.cc index fd2fd89ae..999c40cc1 100644 --- a/mediapipe/calculators/core/end_loop_calculator.cc +++ b/mediapipe/calculators/core/end_loop_calculator.cc @@ -57,6 +57,10 @@ typedef EndLoopCalculator EndLoopFaceBoxCalculator; REGISTER_CALCULATOR(EndLoopFaceBoxCalculator); +typedef EndLoopCalculator>> + EndLoopPointsCalculator; +REGISTER_CALCULATOR(EndLoopPointsCalculator); + typedef EndLoopCalculator> EndLoopTensorCalculator; REGISTER_CALCULATOR(EndLoopTensorCalculator); diff --git a/mediapipe/calculators/image_style/fast_utils_calculator.cc b/mediapipe/calculators/image_style/fast_utils_calculator.cc index 721107bc9..3400db88c 100644 --- a/mediapipe/calculators/image_style/fast_utils_calculator.cc +++ b/mediapipe/calculators/image_style/fast_utils_calculator.cc @@ -329,7 +329,7 @@ namespace mediapipe orig_width = size.first; orig_height = size.second; CHECK_GT(size.first, 0); - CHECK_GT(orig_height, 0); + CHECK_GT(size.second, 0); MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); mat_image_ = *image_mat.get(); diff --git a/mediapipe/calculators/landmarks/BUILD b/mediapipe/calculators/landmarks/BUILD index 6f05ce11f..6ef14e25c 100644 --- a/mediapipe/calculators/landmarks/BUILD +++ b/mediapipe/calculators/landmarks/BUILD @@ -40,3 +40,54 @@ cc_library( ], alwayslink = 1, ) + +cc_library( + name = "points_to_face_box_calculator", + srcs = ["points_to_face_box_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_highgui", + ], + alwayslink = 1, +) + +cc_library( + name = "landmarks_to_point_array_calculator", + srcs = ["landmarks_to_point_array_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_highgui", + "//mediapipe/framework/formats:landmark_cc_proto", + ], + alwayslink = 1, +) + +cc_library( + name = "point_vector_to_mask_calculator", + srcs = ["point_vector_to_mask_calculator.cc"], + hdrs = ["point_vector_to_mask_calculator.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:node", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_highgui", + "//mediapipe/framework/formats:landmark_cc_proto", + ], + alwayslink = 1, +) diff --git a/mediapipe/calculators/landmarks/forehead_mask_calculator.cc b/mediapipe/calculators/landmarks/forehead_mask_calculator.cc new file mode 100644 index 000000000..a20d51d7e --- /dev/null +++ b/mediapipe/calculators/landmarks/forehead_mask_calculator.cc @@ -0,0 +1,149 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/landmarks/point_vector_to_mask_calculator.h" + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/formats/image_format.pb.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/status.h" + +using namespace std; +using namespace cv; + +namespace mediapipe +{ + namespace api2 + { + class PointVectorToMaskCalculator : public Node + { + public: + static constexpr Input kImage{"IMAGE"}; + static constexpr Input> kMasks{"MASKS"}; + static constexpr Input> kFaceBox{"FACE_BOX"}; + static constexpr Output kOut{"FOREHEAD_MASK"}; + + MEDIAPIPE_NODE_CONTRACT(kImage, kMasks, kFaceBox, kOut); + + static absl::Status UpdateContract(CalculatorContract *cc) + { + RET_CHECK(kOut(cc).IsConnected()) + << "At least one output stream is expected."; + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext *cc) override + { + if (kImage(cc).IsEmpty() || kMasks(cc).IsEmpty() || kFaceBox(cc).IsEmpty()) + { + return absl::OkStatus(); + } + + const auto &mat_image_ = *kImage(cc); + const auto &masks = *kMasks(cc); + const auto &face_box = *kFaceBox(cc); + + MP_RETURN_IF_ERROR(PredictForeheadMask(cc, mat_image_, masks, face_box)); + + auto output_frame = absl::make_unique(new_skin_mask); + + kOut(cc).Send(std::move(output_frame)); + + return absl::OkStatus(); + } + + absl::Status PredictForeheadMask(CalculatorContext *cc, Mat mat_image_, + unordered_map mask_vec, tuple face_box) + { + + cv::Mat part_forehead_mask = mask_vec.find("PART_FOREHEAD_B")->second.clone(); + part_forehead_mask.convertTo(part_forehead_mask, CV_32F, 1.0 / 255); + part_forehead_mask.convertTo(part_forehead_mask, CV_8U); + + cv::Mat image_sm, image_sm_hsv, skinMask; + + cv::resize(mat_image_, image_sm, cv::Size(mat_image_.cols, mat_image_.rows)); + cv::cvtColor(image_sm, image_sm_hsv, cv::COLOR_BGR2HSV); + + std::vector x, y; + std::vector location; + + cv::Vec3d hsv_min, hsv_max; + + std::vector channels(3); + cv::split(image_sm_hsv, channels); + std::vector> minx(3), maxx(3); + int c = 0; + for (auto ch : channels) + { + cv::Mat row, mask_row; + double min, max; + for (int i = 0; i < ch.rows; i++) + { + row = ch.row(i); + mask_row = part_forehead_mask.row(i); + cv::minMaxLoc(row, &min, &max, 0, 0, mask_row); + minx[c].push_back(min); + maxx[c].push_back(max); + } + c++; + } + for (int i = 0; i < 3; i++) + { + hsv_min[i] = *std::min_element(minx[i].begin(), minx[i].end()); + } + for (int i = 0; i < 3; i++) + { + hsv_max[i] = *std::max_element(maxx[i].begin(), maxx[i].end()); + } + + cv::Mat _forehead_kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(1, 1)); + cv::inRange(image_sm_hsv, hsv_min, hsv_max, skinMask); + cv::erode(skinMask, skinMask, _forehead_kernel, cv::Point(-1, -1), 2); + cv::dilate(skinMask, skinMask, _forehead_kernel, cv::Point(-1, -1), 2); + skinMask.convertTo(skinMask, CV_8U, 1.0 / 255); + + cv::findNonZero(skinMask, location); + + double max_part_f, x_min_part, x_max_part; + + for (auto &i : location) + { + x.push_back(i.x); + y.push_back(i.y); + } + + cv::minMaxLoc(y, NULL, &max_part_f); + cv::minMaxLoc(x, &x_min_part, &x_max_part); + + cv::Mat new_skin_mask = cv::Mat::zeros(skinMask.size(), CV_8U); + + new_skin_mask(cv::Range(get<1>(face_box), max_part_f), cv::Range(x_min_part, x_max_part)) = + skinMask(cv::Range(get<1>(face_box), max_part_f), cv::Range(x_min_part, x_max_part)); + + return absl::OkStatus(); + } + + private: + Mat new_skin_mask; + }; + MEDIAPIPE_REGISTER_NODE(PointVectorToMaskCalculator); + } +} // namespace mediapipe \ No newline at end of file diff --git a/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.cc b/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.cc index cdacca72c..f941e6b4b 100644 --- a/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.cc +++ b/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.cc @@ -13,17 +13,12 @@ // limitations under the License. #include "mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h" -#include - -#include -#include -#include - #include "absl/memory/memory.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/vector.h" +using namespace std; namespace mediapipe { namespace @@ -33,20 +28,7 @@ namespace mediapipe constexpr char kVectorTag[] = "VECTOR"; constexpr char kMaskTag[] = "MASK"; constexpr char kFaceBoxTag[] = "FACEBOX"; - constexpr char kImageFrameTag[] = "IMAGE"; - - std::unordered_map> orderList = { - {"UPPER_LIP", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78}}, - {"LOWER_LIP", {61, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}}, - {"FACE_OVAL", {10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356, 454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378, 378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150, 136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162, 21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10}}, - {"MOUTH_INSIDE", {78, 191, 80, 81, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95}}, - {"LEFT_EYE", {130, 33, 246, 161, 160, 159, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7}}, - {"RIGHT_EYE", {362, 398, 384, 385, 386, 387, 388, 466, 263, 249, 390, 373, 374, 380, 381, 382}}, - {"LEFT_BROW", {70, 63, 105, 66, 107, 55, 65, 52, 53, 46}}, - {"RIGHT_BROW", {336, 296, 334, 293, 301, 300, 283, 282, 295, 285}}, - {"LIPS", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}}, - {"PART_FOREHEAD_B", {21, 54, 103, 67, 109, 10, 338, 297, 332, 284, 251, 301, 293, 334, 296, 336, 9, 107, 66, 105, 63, 71}}, - }; + constexpr char kImageSizeTag[] = "SIZE"; template bool IsLandmarkVisibleAndPresent(const LandmarkType &landmark, @@ -91,15 +73,10 @@ namespace mediapipe std::tuple face_box; - std::unique_ptr image_mat; - int image_width_; int image_height_; float scale_factor_ = 1.0; - - bool image_frame_available_ = false; - } // namespace absl::Status LandmarksToMaskCalculator::GetContract( @@ -113,9 +90,9 @@ namespace mediapipe << "Can only one type of landmark can be taken. Either absolute or " "normalized landmarks."; - if (cc->Inputs().HasTag(kImageFrameTag)) + if (cc->Inputs().HasTag(kImageSizeTag)) { - cc->Inputs().Tag(kImageFrameTag).Set(); + cc->Inputs().Tag(kImageSizeTag).Set>(); } if (cc->Inputs().HasTag(kLandmarksTag)) @@ -142,18 +119,11 @@ namespace mediapipe { cc->SetOffset(TimestampDiff(0)); - if (cc->Inputs().HasTag(kImageFrameTag)) - { - image_frame_available_ = true; - } - return absl::OkStatus(); } absl::Status LandmarksToMaskCalculator::Process(CalculatorContext *cc) { - // Check that landmarks are not empty and skip rendering if so. - // Don't emit an empty packet for this timestamp. if (cc->Inputs().HasTag(kLandmarksTag) && cc->Inputs().Tag(kLandmarksTag).IsEmpty()) { @@ -164,27 +134,17 @@ namespace mediapipe { return absl::OkStatus(); } - if (cc->Inputs().HasTag(kImageFrameTag) && - cc->Inputs().Tag(kImageFrameTag).IsEmpty()) - { - return absl::OkStatus(); - } - // Initialize render target, drawn with OpenCV. - - ImageFormat::Format target_format; std::unordered_map all_masks; - MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); - - cv::Mat mat_image_ = *image_mat.get(); - image_width_ = image_mat->cols; - image_height_ = image_mat->rows; + const auto size = cc->Inputs().Tag(kImageSizeTag).Get>(); + image_width_ = size.first; + image_height_ = size.second; + CHECK_GT(image_width_, 0); + CHECK_GT(image_height_, 0); MP_RETURN_IF_ERROR(GetMasks(cc, all_masks)); - MP_RETURN_IF_ERROR(GetFaceBox(cc)); - MP_RETURN_IF_ERROR(RenderToCpu(cc, all_masks)); return absl::OkStatus(); @@ -216,116 +176,23 @@ namespace mediapipe return absl::OkStatus(); } - absl::Status LandmarksToMaskCalculator::CreateRenderTargetCpu( - CalculatorContext *cc, std::unique_ptr &image_mat, - ImageFormat::Format *target_format) - { - if (image_frame_available_) - { - const auto &input_frame = - cc->Inputs().Tag(kImageFrameTag).Get(); - - int target_mat_type; - switch (input_frame.Format()) - { - case ImageFormat::SRGBA: - *target_format = ImageFormat::SRGBA; - target_mat_type = CV_8UC4; - break; - case ImageFormat::SRGB: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - case ImageFormat::GRAY8: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - default: - return absl::UnknownError("Unexpected image frame format."); - break; - } - - image_mat = absl::make_unique( - input_frame.Height(), input_frame.Width(), target_mat_type); - - auto input_mat = formats::MatView(&input_frame); - - if (input_frame.Format() == ImageFormat::GRAY8) - { - cv::Mat rgb_mat; - cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB); - rgb_mat.copyTo(*image_mat); - } - else - { - input_mat.copyTo(*image_mat); - } - } - else - { - image_mat = absl::make_unique( - 150, 150, CV_8UC4, - cv::Scalar(255, 255, - 255)); - *target_format = ImageFormat::SRGBA; - } - - return absl::OkStatus(); - } - absl::Status LandmarksToMaskCalculator::GetMasks(CalculatorContext *cc, std::unordered_map &all_masks) { - if (cc->Inputs().HasTag(kLandmarksTag)) - { - const LandmarkList &landmarks = - cc->Inputs().Tag(kNormLandmarksTag).Get(); - - cv::Mat mask; - std::vector point_array; - for (const auto &[key, value] : orderList) - { - for (auto order : value) - { - const Landmark &landmark = landmarks.landmark(order); - - if (!IsLandmarkVisibleAndPresent( - landmark, false, - 0.0, false, - 0.0)) - { - continue; - } - - const auto &point = landmark; - int x = -1; - int y = -1; - CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_, - image_height_, &x, &y)); - point_array.push_back(cv::Point(x, y)); - } - - std::vector> point_vec; - point_vec.push_back(point_array); - - mask = cv::Mat::zeros(image_mat->size(), CV_32FC1); - cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA); - mask.convertTo(mask, CV_8U); - all_masks.insert({key, mask}); - point_vec.clear(); - point_array.clear(); - } - } - if (cc->Inputs().HasTag(kNormLandmarksTag)) { + std::vector x_s, y_s; + double box_min_y, box_max_y, box_max_x, box_min_x; const NormalizedLandmarkList &landmarks = cc->Inputs().Tag(kNormLandmarksTag).Get(); cv::Mat mask; std::vector point_array; + std::vector> point_vec; for (const auto &[key, value] : orderList) { + point_vec = {}; + point_array = {}; for (auto order : value) { const NormalizedLandmark &landmark = landmarks.landmark(order); @@ -345,88 +212,25 @@ namespace mediapipe image_height_, &x, &y)); point_array.push_back(cv::Point(x, y)); } + cv::Mat po(point_array); + po.convertTo(po, CV_32F); + cv::Mat min, max; + + cv::reduce(po, min, 0, CV_REDUCE_MIN, CV_32F); + cv::reduce(po, max, 0, CV_REDUCE_MAX, CV_32F); + + min.at(0,1)*=0.9; + face_box = {min.at(0,0), min.at(0,1), max.at(0,0), max.at(0,1)}; - std::vector> point_vec; point_vec.push_back(point_array); - mask = cv::Mat::zeros(image_mat->size(), CV_32FC1); + mask = cv::Mat::zeros({image_width_, image_height_}, CV_32FC1); cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA); mask.convertTo(mask, CV_8U); - all_masks.insert(make_pair(key, mask)); - point_vec.clear(); - point_array.clear(); + all_masks.insert({key, mask}); } } return absl::OkStatus(); } - absl::Status LandmarksToMaskCalculator::GetFaceBox(CalculatorContext *cc) - { - std::vector x_s, y_s; - double box_min_y, box_max_y, box_max_x, box_min_x; - if (cc->Inputs().HasTag(kLandmarksTag)) - { - const LandmarkList &landmarks = - cc->Inputs().Tag(kLandmarksTag).Get(); - - for (int i = 0; i < landmarks.landmark_size(); ++i) - { - const Landmark &landmark = landmarks.landmark(i); - - if (!IsLandmarkVisibleAndPresent( - landmark, false, - 0.0, false, - 0.0)) - { - continue; - } - - const auto &point = landmark; - int x = -1; - int y = -1; - CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_, - image_height_, &x, &y)); - x_s.push_back(point.x()); - x_s.push_back(point.y()); - } - cv::minMaxLoc(y_s, &box_min_y, &box_max_y); - cv::minMaxLoc(x_s, &box_min_x, &box_max_x); - box_min_y = box_min_y * 0.9; - face_box = std::make_tuple(box_min_x, box_min_y, box_max_x, box_max_y); - } - - if (cc->Inputs().HasTag(kNormLandmarksTag)) - { - const NormalizedLandmarkList &landmarks = - cc->Inputs().Tag(kNormLandmarksTag).Get(); - - for (int i = 0; i < landmarks.landmark_size(); ++i) - { - const NormalizedLandmark &landmark = landmarks.landmark(i); - - if (!IsLandmarkVisibleAndPresent( - landmark, false, - 0.0, false, - 0.0)) - { - continue; - } - - const auto &point = landmark; - int x = -1; - int y = -1; - CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_, - image_height_, &x, &y)); - x_s.push_back(point.x()); - x_s.push_back(point.y()); - } - cv::minMaxLoc(y_s, &box_min_y, &box_max_y); - cv::minMaxLoc(x_s, &box_min_x, &box_max_x); - box_min_y = box_min_y * 0.9; - face_box = std::make_tuple(box_min_x, box_min_y, box_max_x, box_max_y); - } - - return absl::OkStatus(); - } - REGISTER_CALCULATOR(LandmarksToMaskCalculator); } // namespace mediapipe diff --git a/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h b/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h index 49b82944c..dd0b3abb1 100644 --- a/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h +++ b/mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h @@ -30,7 +30,7 @@ namespace mediapipe { // A calculator that uses face landmarks to create face part masks and facebox for - // visualization. The input should be LandmarkList proto and ImageFrame. + // visualization. The input should be LandmarkList proto and ImageFrame. // // Example config: // node { @@ -41,6 +41,19 @@ namespace mediapipe // output_stream: "MASK:mask" // } + std::unordered_map> orderList = { + {"UPPER_LIP", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78}}, + {"LOWER_LIP", {61, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}}, + {"FACE_OVAL", {10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356, 454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378, 378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150, 136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162, 21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10}}, + {"MOUTH_INSIDE", {78, 191, 80, 81, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95}}, + {"LEFT_EYE", {130, 33, 246, 161, 160, 159, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7}}, + {"RIGHT_EYE", {362, 398, 384, 385, 386, 387, 388, 466, 263, 249, 390, 373, 374, 380, 381, 382}}, + {"LEFT_BROW", {70, 63, 105, 66, 107, 55, 65, 52, 53, 46}}, + {"RIGHT_BROW", {336, 296, 334, 293, 301, 300, 283, 282, 295, 285}}, + {"LIPS", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}}, + {"PART_FOREHEAD_B", {21, 54, 103, 67, 109, 10, 338, 297, 332, 284, 251, 301, 293, 334, 296, 336, 9, 107, 66, 105, 63, 71}}, + }; + class LandmarksToMaskCalculator : public CalculatorBase { public: diff --git a/mediapipe/calculators/landmarks/landmarks_to_point_array_calculator.cc b/mediapipe/calculators/landmarks/landmarks_to_point_array_calculator.cc new file mode 100644 index 000000000..3a0494eec --- /dev/null +++ b/mediapipe/calculators/landmarks/landmarks_to_point_array_calculator.cc @@ -0,0 +1,152 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/image_format.pb.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/status.h" + +using namespace std; +using namespace cv; + +namespace mediapipe +{ + namespace + { + template + bool IsLandmarkVisibleAndPresent(const LandmarkType &landmark, + bool utilize_visibility, + float visibility_threshold, + bool utilize_presence, + float presence_threshold) + { + if (utilize_visibility && landmark.has_visibility() && + landmark.visibility() < visibility_threshold) + { + return false; + } + if (utilize_presence && landmark.has_presence() && + landmark.presence() < presence_threshold) + { + return false; + } + return true; + } + + bool NormalizedtoPixelCoordinates(double normalized_x, double normalized_y, double normalized_z, + int image_width, int image_height, double *x_px, + double *y_px, double *z_px) + { + CHECK(x_px != nullptr); + CHECK(y_px != nullptr); + CHECK_GT(image_width, 0); + CHECK_GT(image_height, 0); + + if (normalized_x < 0 || normalized_x > 1.0 || normalized_y < 0 || + normalized_y > 1.0 || normalized_z < 0 || + normalized_z > 1.0) + { + VLOG(1) << "Normalized coordinates must be between 0.0 and 1.0"; + } + + *x_px = static_cast(normalized_x) * image_width; + *y_px = static_cast(normalized_y) * image_height; + *z_px = static_cast(normalized_z) * image_width; + + return true; + } + } + namespace api2 + { + class LandmarksToPointArrayCalculator : public Node + { + public: + static constexpr Input kNormLandmarks{"NORM_LANDMARKS"}; + static constexpr Input> kImageSize{"IMAGE_SIZE"}; + static constexpr Output> kOut{"POINTS"}; + + MEDIAPIPE_NODE_CONTRACT(kNormLandmarks, kImageSize, kOut); + + static absl::Status UpdateContract(CalculatorContract *cc) + { + RET_CHECK(kOut(cc).IsConnected()) + << "At least one output stream is expected."; + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext *cc) override + { + if (kNormLandmarks(cc).IsEmpty()) + { + return absl::OkStatus(); + } + if (kImageSize(cc).IsEmpty()) + { + return absl::OkStatus(); + } + + const auto &size = *kImageSize(cc); + + MP_RETURN_IF_ERROR(GetPoints(cc, size)); + + auto output_frame = absl::make_unique>(point_array); + + kOut(cc).Send(std::move(output_frame)); + return absl::OkStatus(); + } + + absl::Status GetPoints(CalculatorContext *cc, pair size) + { + + const auto &landmarks = *kNormLandmarks(cc); + point_array = {}; + for (int i = 0; i < landmarks.landmark_size(); i++) + { + const NormalizedLandmark &landmark = landmarks.landmark(i); + + if (!IsLandmarkVisibleAndPresent( + landmark, false, + 0.0, false, + 0.0)) + { + continue; + } + + const auto &point = landmark; + + double x = -1; + double y = -1; + double z = -1; + + CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), point.z(), size.first, + size.second, &x, &y, &z)); + + point_array.push_back({x, y, z}); + } + + return absl::OkStatus(); + } + + private: + std::vector point_array; + }; + MEDIAPIPE_REGISTER_NODE(LandmarksToPointArrayCalculator); + } +} // namespace mediapipe \ No newline at end of file diff --git a/mediapipe/calculators/landmarks/point_vector_to_mask_calculator.cc b/mediapipe/calculators/landmarks/point_vector_to_mask_calculator.cc new file mode 100644 index 000000000..4be046688 --- /dev/null +++ b/mediapipe/calculators/landmarks/point_vector_to_mask_calculator.cc @@ -0,0 +1,91 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/landmarks/point_vector_to_mask_calculator.h" + +using namespace std; +using namespace cv; + +namespace mediapipe +{ + namespace api2 + { + class PointVectorToMaskCalculator : public Node + { + public: + static constexpr Input> kPoints{"POINTS"}; + static constexpr Input> kImageSize{"IMAGE_SIZE"}; + static constexpr Output> kOut{"MASKS"}; + + MEDIAPIPE_NODE_CONTRACT(kPoints, kImageSize, kOut); + + static absl::Status UpdateContract(CalculatorContract *cc) + { + RET_CHECK(kOut(cc).IsConnected()) + << "At least one output stream is expected."; + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext *cc) override + { + if (kPoints(cc).IsEmpty()) + { + return absl::OkStatus(); + } + if (kImageSize(cc).IsEmpty()) + { + return absl::OkStatus(); + } + + const auto &size = *kImageSize(cc); + const auto &points = *kPoints(cc); + + MP_RETURN_IF_ERROR(GetMasks(cc, size, points)); + + auto output_frame = absl::make_unique>(all_masks); + + kOut(cc).Send(std::move(output_frame), cc->InputTimestamp()); + + return absl::OkStatus(); + } + + absl::Status GetMasks(CalculatorContext *cc, pair size, vector points) + { + all_masks= {}; + + for (const auto &[key, value] : orderList) + { + vector> point_vec = {}; + vector point_array = {}; + for (auto order : value) + { + point_array.push_back({points[order].x, points[order].y}); + } + point_vec.push_back(point_array); + + cv::Mat mask = cv::Mat::zeros({size.first, size.second}, CV_32FC1); + cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA); + mask.convertTo(mask, CV_8U); + + all_masks.insert({key, mask}); + } + return absl::OkStatus(); + } + + private: + std::unordered_map all_masks; + }; + MEDIAPIPE_REGISTER_NODE(PointVectorToMaskCalculator); + } +} // namespace mediapipe \ No newline at end of file diff --git a/mediapipe/calculators/landmarks/point_vector_to_mask_calculator.h b/mediapipe/calculators/landmarks/point_vector_to_mask_calculator.h new file mode 100644 index 000000000..4438333f6 --- /dev/null +++ b/mediapipe/calculators/landmarks/point_vector_to_mask_calculator.h @@ -0,0 +1,81 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/formats/image_format.pb.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe +{ + + // A calculator that uses face landmarks to create face part masks and facebox for + // visualization. The input should be LandmarkList proto and ImageFrame. + // + // Example config: + // node { + // calculator: "LandmarksToMaskCalculator" + // input_stream: "IMAGE:image" + // input_stream: "NORM_LANDMARKS:face_landmarks" + // output_stream: "FACEBOX:face_box" + // output_stream: "MASK:mask" + // } + + std::unordered_map> orderList = { + {"UPPER_LIP", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78}}, + {"LOWER_LIP", {61, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}}, + {"FACE_OVAL", {10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356, 454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378, 378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150, 136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162, 21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10}}, + {"MOUTH_INSIDE", {78, 191, 80, 81, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95}}, + {"LEFT_EYE", {33, 246, 161, 160, 159, 158, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7}}, + {"RIGHT_EYE", {362, 398, 384, 385, 386, 387, 388, 466, 263, 249, 390, 373, 374, 380, 381, 382}}, + {"LEFT_IRIS", {468, 469, 470, 471, 472}}, + {"RIGHT_IRIS", {473, 474, 475, 476, 477}}, + {"LEFT_BROW", {468, 469, 470, 471, 472}}, + {"RIGHT_BROW", {336, 296, 334, 293, 301, 300, 283, 282, 295, 285}}, + {"LIPS", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}}, + {"PART_FOREHEAD_B", { + 21, + 54, + 103, + 67, + 109, + 10, + 338, + 297, + 332, + 284, + 251, + 301, + 293, + 334, + 296, + 336, + 9, + 107, + 66, + 105, + 63, + 71, + }}, + }; + +} // namespace mediapipe diff --git a/mediapipe/calculators/landmarks/points_to_face_box_calculator.cc b/mediapipe/calculators/landmarks/points_to_face_box_calculator.cc new file mode 100644 index 000000000..f57577688 --- /dev/null +++ b/mediapipe/calculators/landmarks/points_to_face_box_calculator.cc @@ -0,0 +1,93 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/api2/node.h" +#include "mediapipe/framework/formats/image_format.pb.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/status.h" + +using namespace std; +using namespace cv; + +namespace mediapipe +{ + namespace api2 + { + class PointsToFaceBoxCalculator : public Node + { + public: + static constexpr Input> kPoints{"POINTS"}; + static constexpr Input> kImageSize{"IMAGE_SIZE"}; + static constexpr Output> kOut{"FACE_BOX"}; + + MEDIAPIPE_NODE_CONTRACT(kPoints, kImageSize, kOut); + + static absl::Status UpdateContract(CalculatorContract *cc) + { + RET_CHECK(kOut(cc).IsConnected()) + << "At least one output stream is expected."; + return absl::OkStatus(); + } + + absl::Status Process(CalculatorContext *cc) override + { + if (kPoints(cc).IsEmpty()) + { + return absl::OkStatus(); + } + if (kImageSize(cc).IsEmpty()) + { + return absl::OkStatus(); + } + + const auto &size = *kImageSize(cc); + const auto &points = *kPoints(cc); + + MP_RETURN_IF_ERROR(GetFaceBox(cc, size, points)); + + auto output_frame = absl::make_unique>(face_box); + + kOut(cc).Send(std::move(output_frame)); + + return absl::OkStatus(); + } + + absl::Status GetFaceBox(CalculatorContext *cc, pair size, vector points) + { + cv::Mat points_mat(points); + + points_mat.convertTo(points_mat, CV_32F); + cv::Mat min, max; + + cv::reduce(points_mat, min, 0, CV_REDUCE_MIN, CV_32F); + cv::reduce(points_mat, max, 0, CV_REDUCE_MAX, CV_32F); + + min.at(0, 1) *= 0.9; + face_box = {min.at(0, 0), min.at(0, 1), max.at(0, 0), max.at(0, 1)}; + + return absl::OkStatus(); + } + + private: + tuple face_box; + }; + MEDIAPIPE_REGISTER_NODE(PointsToFaceBoxCalculator); + } +} // namespace mediapipe \ No newline at end of file diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/AndroidManifest.xml b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/AndroidManifest.xml index 528a03a3a..f364b7ec8 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/AndroidManifest.xml +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/AndroidManifest.xml @@ -12,7 +12,8 @@ - + + index_names; map> indexes; - - map> masks; vector> _trianglesIndexes; + map> masks; + Tensor __facePts; }; @@ -181,14 +181,13 @@ namespace mediapipe absl::Status FaceProcessorCalculator::Open(CalculatorContext *cc) { cc->SetOffset(TimestampDiff(0)); + MP_RETURN_IF_ERROR(SetData(cc)); - return absl::OkStatus(); + return absl::OkStatus(); } absl::Status FaceProcessorCalculator::Process(CalculatorContext *cc) { - MP_RETURN_IF_ERROR(SetData(cc)); - if (cc->Inputs().HasTag(kNormLandmarksTag) && !cc->Inputs().Tag(kNormLandmarksTag).IsEmpty()) { @@ -206,11 +205,10 @@ namespace mediapipe absl::Status FaceProcessorCalculator::SetData(CalculatorContext *cc) { masks = {}; - _trianglesIndexes = {}; + string line; string filename = "mediapipe/graphs/deformation/config/triangles.txt"; - string content_blob; - ASSIGN_OR_RETURN(content_blob, + ASSIGN_OR_RETURN(string content_blob, ReadContentBlobFromFile(filename), _ << "Failed to read texture blob from file!"); @@ -219,7 +217,6 @@ namespace mediapipe vector tmp; for (int i = 0; i < 854; ++i) { - string line; tmp = {}; for (int j = 0; j < 3; ++j) { @@ -228,20 +225,20 @@ namespace mediapipe } _trianglesIndexes.push_back(tmp); } + stream.clear(); filename = "./mediapipe/graphs/deformation/config/index_names.txt"; ASSIGN_OR_RETURN(content_blob, ReadContentBlobFromFile(filename), _ << "Failed to read texture blob from file!"); - istringstream stream2(content_blob); + stream.str(content_blob); - string line; vector idxs; - while (getline(stream2, line)) + while (getline(stream, line)) { index_names.push_back(line); } - stream2.clear(); + stream.clear(); for (int i = 0; i < index_names.size(); i++) { @@ -250,16 +247,16 @@ namespace mediapipe ASSIGN_OR_RETURN(content_blob, ReadContentBlobFromFile(filename), _ << "Failed to read texture blob from file!"); - stream2.str(content_blob); + stream.str(content_blob); - while (getline(stream2, line)) + while (getline(stream, line)) { idxs.push_back(stoi(line)); } indexes[index_names[i]] = idxs; idxs = {}; - stream2.clear(); + stream.clear(); } double **zero_arr; diff --git a/mediapipe/graphs/deformation/deformation_mobile.pbtxt b/mediapipe/graphs/deformation/deformation_mobile.pbtxt index 8a8abab6c..bff8fa74c 100644 --- a/mediapipe/graphs/deformation/deformation_mobile.pbtxt +++ b/mediapipe/graphs/deformation/deformation_mobile.pbtxt @@ -56,6 +56,16 @@ node { output_stream: "SIZE:size" } +node { + calculator: "LocalFileContentsCalculator" + input_side_packet: "FILE_PATH:0:file_path1" + input_side_packet: "FILE_PATH:1:file_path2" + input_side_packet: "FILE_PATH:1:file_path2" + output_side_packet: "CONTENTS:0:contents1" + output_side_packet: "CONTENTS:1:contents2" + output_side_packet: "CONTENTS:1:contents2" +} + node { calculator: "FaceProcessorCalculator" input_stream: "NORM_LANDMARKS:multi_face_landmarks" diff --git a/mediapipe/graphs/image_style/README.md b/mediapipe/graphs/image_style/README.md index afeb457cd..9902021e1 100644 --- a/mediapipe/graphs/image_style/README.md +++ b/mediapipe/graphs/image_style/README.md @@ -18,9 +18,7 @@ bazel-bin/mediapipe/examples/desktop/image_style/image_style_cpu --calculator_gr ``` Run with (using video): ``` -bazel-bin/mediapipe/examples/desktop/image_style/image_style_cpu --calculator_graph_config_file=mediapipe/graphs/image_style/image_style_cpu.pbtxt ---input_video_path=/path/video.mp4 ---output_video_path=/path/outvideo.mp4 +bazel-bin/mediapipe/examples/desktop/image_style/image_style_cpu --calculator_graph_config_file=mediapipe/graphs/image_style/image_style_cpu.pbtxt --input_video_path=/path/video.mp4 --output_video_path=/path/outvideo.mp4 ``` 2. Mobile (Android) diff --git a/mediapipe/graphs/image_style/image_style_cpu.pbtxt b/mediapipe/graphs/image_style/image_style_cpu.pbtxt index cad9858d8..236d5fdb5 100644 --- a/mediapipe/graphs/image_style/image_style_cpu.pbtxt +++ b/mediapipe/graphs/image_style/image_style_cpu.pbtxt @@ -6,6 +6,13 @@ input_stream: "input_video" # Output image with rendered results. (ImageFrame) output_stream: "output_video" +profiler_config { + trace_enabled: true + enable_profiler: true + trace_log_interval_count: 200 + trace_log_path: "/home/mslight/Work/clone/mediapipe/mediapipe/logs/imagestyle/" +} + node { calculator: "FlowLimiterCalculator" input_stream: "input_video" diff --git a/mediapipe/graphs/makeup/BUILD b/mediapipe/graphs/makeup/BUILD new file mode 100644 index 000000000..1178cea43 --- /dev/null +++ b/mediapipe/graphs/makeup/BUILD @@ -0,0 +1,63 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load( + "//mediapipe/framework/tool:mediapipe_graph.bzl", + "mediapipe_binary_graph", +) + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "desktop_live_calculators", + deps = [ + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/graphs/makeup/subgraphs:face_renderer_cpu", + "//mediapipe/modules/face_landmark:face_landmark_front_cpu", + ], +) + +cc_library( + name = "desktop_live_gpu_calculators", + deps = [ + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/graphs/makeup/subgraphs:face_renderer_gpu", + "//mediapipe/modules/face_landmark:face_landmark_front_gpu", + ], +) + +cc_library( + name = "mobile_calculators", + deps = [ + "//mediapipe/gpu:gpu_buffer_to_image_frame_calculator", + "//mediapipe/gpu:image_frame_to_gpu_buffer_calculator", + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/graphs/makeup/subgraphs:face_renderer_cpu", + "//mediapipe/graphs/makeup/subgraphs:face_renderer_gpu", + "//mediapipe/modules/face_landmark:face_landmark_front_gpu", + ], +) + + +mediapipe_binary_graph( + name = "beauty_mobile_gpu_binary_graph", + graph = "beauty_mobile_gpu.pbtxt", + output_name = "beauty_mobile_gpu.binarypb", + deps = [":mobile_calculators"], +) + diff --git a/mediapipe/graphs/makeup/makeup_desktop_cpu.pbtxt b/mediapipe/graphs/makeup/makeup_desktop_cpu.pbtxt new file mode 100644 index 000000000..b14b2f4b7 --- /dev/null +++ b/mediapipe/graphs/makeup/makeup_desktop_cpu.pbtxt @@ -0,0 +1,71 @@ +# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU. + +# Input image. (ImageFrame) +input_stream: "input_video" + +# Output image with rendered results. (ImageFrame) +output_stream: "output_video" +# Collection of detected/processed faces, each represented as a list of +# landmarks. (std::vector) +output_stream: "multi_face_landmarks" + + +profiler_config { + trace_enabled: true + enable_profiler: true + trace_log_interval_count: 200 + trace_log_path: "/home/mslight/Work/clone/mediapipe/mediapipe/logs/makeup/" +} + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for downstream nodes +# (calculators and subgraphs) in the graph to finish their tasks before it +# passes through another image. All images that come in while waiting are +# dropped, limiting the number of in-flight images in most part of the graph to +# 1. This prevents the downstream nodes from queuing up incoming images and data +# excessively, which leads to increased latency and memory usage, unwanted in +# real-time mobile applications. It also eliminates unnecessarily computation, +# e.g., the output produced by a node may get dropped downstream if the +# subsequent nodes are still busy processing previous inputs. + + +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:output_video" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Defines side packets for further use in the graph. +node { + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:0:num_faces" + output_side_packet: "PACKET:1:with_attention" + node_options: { + [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { + packet { int_value: 1 } + packet { bool_value: true } + } + } +} + +# Subgraph that detects faces and corresponding landmarks. +node { + calculator: "FaceLandmarkFrontCpu" + input_stream: "IMAGE:throttled_input_video" + input_side_packet: "NUM_FACES:num_faces" + input_side_packet: "WITH_ATTENTION:with_attention" + output_stream: "LANDMARKS:multi_face_landmarks" +} + +# Subgraph that renders face-landmark annotation onto the input image. +node { + calculator: "FaceRendererCpu" + input_stream: "IMAGE:throttled_input_video" + input_stream: "LANDMARKS:multi_face_landmarks" + output_stream: "IMAGE:output_video" +} diff --git a/mediapipe/graphs/makeup/makeup_mobile_gpu.pbtxt b/mediapipe/graphs/makeup/makeup_mobile_gpu.pbtxt new file mode 100644 index 000000000..ebed553b2 --- /dev/null +++ b/mediapipe/graphs/makeup/makeup_mobile_gpu.pbtxt @@ -0,0 +1,84 @@ +# MediaPipe graph that performs face mesh with TensorFlow Lite on GPU. + +# GPU buffer. (GpuBuffer) +input_stream: "input_video" + +# Max number of faces to detect/process. (int) +input_side_packet: "num_faces" + +# Output image with rendered results. (GpuBuffer) +output_stream: "output_video" +# Collection of detected/processed faces, each represented as a list of +# landmarks. (std::vector) +output_stream: "multi_face_landmarks" + +profiler_config { + trace_enabled: true + enable_profiler: true + trace_log_interval_count: 200 +} + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for downstream nodes +# (calculators and subgraphs) in the graph to finish their tasks before it +# passes through another image. All images that come in while waiting are +# dropped, limiting the number of in-flight images in most part of the graph to +# 1. This prevents the downstream nodes from queuing up incoming images and data +# excessively, which leads to increased latency and memory usage, unwanted in +# real-time mobile applications. It also eliminates unnecessarily computation, +# e.g., the output produced by a node may get dropped downstream if the +# subsequent nodes are still busy processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:output_video" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Defines side packets for further use in the graph. +node { + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:with_attention" + node_options: { + [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { + packet { bool_value: true } + } + } +} + +# Defines side packets for further use in the graph. +node { + calculator: "GpuBufferToImageFrameCalculator" + input_stream: "throttled_input_video" + output_stream: "throttled_input_video_cpu" +} + +# Subgraph that detects faces and corresponding landmarks. +node { + calculator: "FaceLandmarkFrontGpu" + input_stream: "IMAGE:throttled_input_video" + input_side_packet: "NUM_FACES:num_faces" + input_side_packet: "WITH_ATTENTION:with_attention" + output_stream: "LANDMARKS:multi_face_landmarks" + output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" + output_stream: "DETECTIONS:face_detections" + output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" +} + +# Subgraph that renders face-landmark annotation onto the input image. +node { + calculator: "FaceRendererCpu" + input_stream: "IMAGE:throttled_input_video_cpu" + input_stream: "LANDMARKS:multi_face_landmarks" + output_stream: "IMAGE:output_video_cpu" +} + +node { + calculator: "ImageFrameToGpuBufferCalculator" + input_stream: "output_video_cpu" + output_stream: "output_video" +} diff --git a/mediapipe/graphs/makeup/subgraphs/BUILD b/mediapipe/graphs/makeup/subgraphs/BUILD new file mode 100644 index 000000000..ba7c82426 --- /dev/null +++ b/mediapipe/graphs/makeup/subgraphs/BUILD @@ -0,0 +1,60 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load( + "//mediapipe/framework/tool:mediapipe_graph.bzl", + "mediapipe_simple_subgraph", +) + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "renderer_calculators", + deps = [ + "//mediapipe/calculators/beauty:smooth_face_calculator1", + "//mediapipe/calculators/beauty:bilateral_filter_calculator", + "//mediapipe/calculators/beauty:smooth_face_calculator2", + "//mediapipe/calculators/beauty:draw_lipstick_calculator", + "//mediapipe/calculators/beauty:whiten_teeth_calculator", + "//mediapipe/calculators/beauty:merging_images_calculator", + "//mediapipe/calculators/beauty:imageframe_to_mat_calculator", + "//mediapipe/calculators/landmarks:landmarks_to_point_array_calculator", + "//mediapipe/calculators/landmarks:points_to_face_box_calculator", + "//mediapipe/calculators/landmarks:point_vector_to_mask_calculator", + "//mediapipe/calculators/core:round_robin_demux_calculator", + "//mediapipe/calculators/core:mux_calculator", + ], +) + + +mediapipe_simple_subgraph( + name = "face_renderer_gpu", + graph = "face_renderer_gpu.pbtxt", + register_as = "FaceRendererGpu", + deps = [ + ":renderer_calculators", + ], +) + + +mediapipe_simple_subgraph( + name = "face_renderer_cpu", + graph = "face_renderer_cpu.pbtxt", + register_as = "FaceRendererCpu", + deps = [ + ":renderer_calculators", + ], +) diff --git a/mediapipe/graphs/makeup/subgraphs/face_renderer_cpu.pbtxt b/mediapipe/graphs/makeup/subgraphs/face_renderer_cpu.pbtxt new file mode 100644 index 000000000..d2731c4f7 --- /dev/null +++ b/mediapipe/graphs/makeup/subgraphs/face_renderer_cpu.pbtxt @@ -0,0 +1,152 @@ +# MediaPipe face mesh rendering subgraph. + +type: "FaceRendererCpu" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:input_image" +# Collection of detected/predicted faces, each represented as a list of +# landmarks. (std::vector) +input_stream: "LANDMARKS:multi_face_landmarks" + +# CPU image with rendered data. (ImageFrame) +output_stream: "IMAGE:output_image" + +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:input_image" + output_stream: "SIZE:size" +} + +node { + calculator: "BeginLoopNormalizedLandmarkListVectorCalculator" + input_stream: "ITERABLE:multi_face_landmarks" + input_stream: "CLONE:size" + output_stream: "ITEM:face_landmarks" + output_stream: "CLONE:image_size" + output_stream: "BATCH_END:landmark_timestamp" +} + +# Converts landmarks to face part masks. +node { + calculator: "LandmarksToPointArrayCalculator" + input_stream: "IMAGE_SIZE:image_size" + input_stream: "NORM_LANDMARKS:face_landmarks" + output_stream: "POINTS:points" +} + +# Converts landmarks to face part masks. +node { + calculator: "PointVectorToMaskCalculator" + input_stream: "POINTS:points" + input_stream: "IMAGE_SIZE:image_size" + output_stream: "MASKS:mask" +} + +# Converts landmarks to face part masks. +node { + calculator: "PointsToFaceBoxCalculator" + input_stream: "POINTS:points" + input_stream: "IMAGE_SIZE:image_size" + output_stream: "FACE_BOX:face_box" +} + +node { + calculator: "EndLoopMapMaskCalculator" + input_stream: "ITEM:mask" + input_stream: "BATCH_END:landmark_timestamp" + output_stream: "ITERABLE:multi_mask" +} + +node { + calculator: "EndLoopFaceBoxCalculator" + input_stream: "ITEM:face_box" + input_stream: "BATCH_END:landmark_timestamp" + output_stream: "ITERABLE:multi_face_box" +} + +node { + calculator: "ImageFrameToMatCalculator" + input_stream: "IMAGE:input_image" + output_stream: "MAT:input_mat" +} + +#Applies lipstick to the face on the IMAGE using MASK. +node { + calculator: "DrawLipstickCalculator" + input_stream: "MAT:input_mat" + input_stream: "MASK:0:multi_mask" + output_stream: "MAT:lipstick_image" + output_stream: "MASK:lipstick_mask" +} + +#Whitens teeth of the face on the IMAGE using MASK. +node { + calculator: "WhitenTeethCalculator" + input_stream: "MAT:input_mat" + input_stream: "MASK:0:multi_mask" + output_stream: "MAT:teeth_image" + output_stream: "MASK:teeth_mask" +} + +#Smoothes face on the IMAGE using MASK. +node { + calculator: "SmoothFaceCalculator1" + input_stream: "MAT:input_mat" + input_stream: "MASK:0:multi_mask" + input_stream: "FACE:0:multi_face_box" + output_stream: "MAT:tmp_image_1" + output_stream: "MASK:face_mask" + output_stream: "FACE:box_1" +} + +node { + calculator: "RoundRobinDemuxCalculator" + input_stream: "box_1" + output_stream: "OUTPUT:0:box1_0" + output_stream: "OUTPUT:1:box1_1" + output_stream: "SELECT:select" + } + + node { + calculator: "BilateralCalculator" + input_stream: "IMAGE:box1_0" + output_stream: "CVMAT:box14_0" +} + node { + calculator: "BilateralCalculator" + input_stream: "IMAGE:box1_1" + output_stream: "CVMAT:box14_1" + } + + node { + calculator: "MuxCalculator" + input_stream: "INPUT:0:box14_0" + input_stream: "INPUT:1:box14_1" + input_stream: "SELECT:select" + output_stream: "OUTPUT:tmp_image_2" + input_stream_handler { + input_stream_handler: "MuxInputStreamHandler" + } + } + +#Smoothes face on the IMAGE using MASK. +node { + calculator: "SmoothFaceCalculator2" + input_stream: "MAT:input_mat" + input_stream: "IMAGE2:tmp_image_2" + input_stream: "MASK:face_mask" + input_stream: "FACEBOX:box_1" + output_stream: "MAT:face_image" +} + +#Merges output images of DrawLipstick, WhitenTeeth and SmoothFace calculators +node { + calculator: "MergeImagesCalculator" + input_stream: "MAT:0:lipstick_image" + input_stream: "MASK:0:lipstick_mask" + input_stream: "MAT:1:teeth_image" + input_stream: "MASK:1:teeth_mask" + input_stream: "MAT:2:face_image" + input_stream: "MASK:2:face_mask" + output_stream: "IMAGE:output_image" +} diff --git a/mediapipe/graphs/makeup/subgraphs/face_renderer_gpu.pbtxt b/mediapipe/graphs/makeup/subgraphs/face_renderer_gpu.pbtxt new file mode 100644 index 000000000..b8c4871dd --- /dev/null +++ b/mediapipe/graphs/makeup/subgraphs/face_renderer_gpu.pbtxt @@ -0,0 +1,106 @@ +# MediaPipe face mesh rendering subgraph. + +type: "FaceRendererGpu" + +# GPU image. (GpuBuffer) +input_stream: "IMAGE:input_image" +# Collection of detected/predicted faces, each represented as a list of +# landmarks. (std::vector) +input_stream: "LANDMARKS:multi_face_landmarks" +# Regions of interest calculated based on palm detections. +# (std::vector) +input_stream: "NORM_RECTS:rects" +# Detected palms. (std::vector) +input_stream: "DETECTIONS:detections" + +# GPU image with rendered data. (GpuBuffer) +output_stream: "IMAGE:output_image" + +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE_GPU:input_image" + output_stream: "SIZE:image_size" +} +# Outputs each element of multi_face_landmarks at a fake timestamp for the rest +# of the graph to process. At the end of the loop, outputs the BATCH_END +# timestamp for downstream calculators to inform them that all elements in the +# vector have been processed. +node { + calculator: "BeginLoopNormalizedLandmarkListVectorCalculator" + input_stream: "ITERABLE:multi_face_landmarks" + input_stream: "IMAGE_GPU:input_image" + output_stream: "ITEM:face_landmarks" + output_stream: "IMAGE_GPU:loop_image" + output_stream: "BATCH_END:landmark_timestamp" +} + +# Converts landmarks to face part masks. +node { + calculator: "LandmarksToMaskCalculator" + input_stream: "IMAGE_GPU:loop_image" + input_stream: "NORM_LANDMARKS:face_landmarks" + output_stream: "FACEBOX:face_box" + output_stream: "MASK:mask" +} + +# Collects a MapMask object for each hand into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END +# timestamp. +node { + calculator: "EndLoopMapMaskCalculator" + input_stream: "ITEM:mask" + input_stream: "BATCH_END:landmark_timestamp" + output_stream: "ITERABLE:multi_mask" +} + +node { + calculator: "EndLoopFaceBoxCalculator" + input_stream: "ITEM:face_box" + input_stream: "BATCH_END:landmark_timestamp" + output_stream: "ITERABLE:multi_face_box" +} + +#Applies lipstick to the face on the IMAGE using MASK. +node { + calculator: "DrawLipstickCalculator" + input_stream: "IMAGE_GPU:input_image" + input_stream: "MASK:0:multi_mask" + output_stream: "IMAGE_GPU:input_image_1" +} + +#Whitens teeth of the face on the IMAGE using MASK. +node { + calculator: "WhitenTeethCalculator" + input_stream: "IMAGE_GPU:input_image_1" + input_stream: "MASK:0:multi_mask" + output_stream: "IMAGE_GPU:input_image_2" +} + +#Smoothes face on the IMAGE using MASK. +node { + calculator: "SmoothFaceCalculator1" + input_stream: "IMAGE:input_image_2" + input_stream: "MASK:0:multi_mask" + input_stream: "FACEBOX:0:multi_face_box" + output_stream: "IMAGE:input_image_3" + output_stream: "MASK:not_full_face" + output_stream: "FACEBOX:box1" +} + +#Smoothes face on the IMAGE using MASK. +node { + calculator: "BilateralCalculator" + input_stream: "IMAGE:input_image_3" + input_stream: "FACEBOX:box1" + output_stream: "IMAGE2:input_image_4" +} + +#Smoothes face on the IMAGE using MASK. +node { + calculator: "SmoothFaceCalculator2" + input_stream: "IMAGE:input_image_2" + input_stream: "IMAGE2:input_image_4" + input_stream: "MASK:not_full_face" + input_stream: "FACEBOX:box1" + output_stream: "IMAGE:output_image" +}