cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Makeup alpha

Browse Source
This commit is contained in:
mslight 2022-08-19 11:24:40 +04:00
parent ffa2853b79
commit 007c097237
43 changed files with 2026 additions and 1090 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
mediapipe/calculators/beauty/BUILD
View File

@ -22,29 +22,6 @@ cc_library(
name = "draw_lipstick_calculator",
srcs = ["draw_lipstick_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/util:color_cc_proto",
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
],
alwayslink = 1,
)
cc_library(
name = "form_face_mask_calculator",
srcs = ["form_face_mask_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework/formats:image_format_cc_proto",
@ -60,14 +37,10 @@ cc_library(
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
"//mediapipe/util:annotation_renderer",
"//mediapipe/util:render_data_cc_proto",
],
alwayslink = 1,
)
cc_library(
name = "smooth_face_calculator1",
srcs = ["smooth_face1_calculator.cc"],
@ -84,6 +57,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
],
@ -106,6 +80,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
],
@ -128,6 +103,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
],
@ -150,14 +126,49 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
],
alwayslink = 1,
)
cc_library(
name = "merging_images_calculator",
srcs = ["merging_images_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework/formats:image_format_cc_proto",
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
],
alwayslink = 1,
)
cc_library(
name = "imageframe_to_mat_calculator",
srcs = ["imageframe_to_mat_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_highgui",
],
alwayslink = 1,
)

186
mediapipe/calculators/beauty/bilateral_filter_calculator.cc
View File

@ -12,14 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <math.h>
#include <algorithm>
#include <cmath>
#include <iostream>
#include <memory>
#include "absl/strings/str_cat.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
@ -30,26 +22,18 @@
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/vector.h"
using namespace std;
namespace mediapipe
{
namespace
{
constexpr char kMaskTag[] = "MASK";
constexpr char kFaceBoxTag[] = "FACEBOX";
constexpr char kImageFrameTag[] = "IMAGE";
constexpr char kImageNewTag[] = "IMAGE2";
enum
{
ATTRIB_VERTEX,
ATTRIB_TEXTURE_POSITION,
NUM_ATTRIBUTES
};
constexpr char kOutTag[] = "CVMAT";
inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
} // namespace
@ -68,24 +52,18 @@ namespace mediapipe
absl::Status Close(CalculatorContext *cc) override;
private:
absl::Status CreateRenderTargetCpu(CalculatorContext *cc,
std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format);
absl::Status RenderToCpu(
CalculatorContext *cc, const ImageFormat::Format &target_format,
uchar *data_image);
absl::Status BilateralFilter(CalculatorContext *cc,
const std::vector<double> &face_box);
absl::Status RenderToCpu(CalculatorContext *cc);
// Indicates if image frame is available as input.
bool image_frame_available_ = false;
int image_width_;
int image_height_;
cv::Mat mat_image_;
std::unique_ptr<cv::Mat> image_mat;
cv::Mat out_mat;
};
REGISTER_CALCULATOR(BilateralCalculator);
@ -95,30 +73,13 @@ namespace mediapipe
if (cc->Inputs().HasTag(kImageFrameTag))
{
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
CHECK(cc->Outputs().HasTag(kImageNewTag));
cc->Inputs().Tag(kImageFrameTag).Set<std::pair<cv::Mat, std::vector<double>>>();
CHECK(cc->Outputs().HasTag(kOutTag));
}
// Data streams to render.
for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId();
++id)
if (cc->Outputs().HasTag(kOutTag))
{
auto tag_and_index = cc->Inputs().TagAndIndexFromId(id);
std::string tag = tag_and_index.first;
if (tag == kFaceBoxTag)
{
cc->Inputs().Get(id).Set<std::vector<double>>();
}
else if (tag.empty())
{
// Empty tag defaults to accepting a single object of Mat type.
cc->Inputs().Get(id).Set<cv::Mat>();
}
}
if (cc->Outputs().HasTag(kImageNewTag))
{
cc->Outputs().Tag(kImageNewTag).Set<ImageFrame>();
cc->Outputs().Tag(kOutTag).Set<cv::Mat>();
}
return absl::OkStatus();
@ -135,7 +96,7 @@ namespace mediapipe
// Set the output header based on the input header (if present).
const char *tag = kImageFrameTag;
const char *out_tag = kImageNewTag;
const char *out_tag = kOutTag;
if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty())
{
const auto &input_header =
@ -154,32 +115,21 @@ namespace mediapipe
{
return absl::OkStatus();
}
if (cc->Inputs().HasTag(kFaceBoxTag) &&
cc->Inputs().Tag(kFaceBoxTag).IsEmpty())
{
return absl::OkStatus();
}
const std::pair<cv::Mat, std::vector<double>> &face =
cc->Inputs().Tag(kImageFrameTag).Get<std::pair<cv::Mat, std::vector<double>>>();
// Initialize render target, drawn with OpenCV.
ImageFormat::Format target_format;
const std::vector<double> &face_box = face.second;
mat_image_ = face.first.clone();
if (cc->Outputs().HasTag(kImageNewTag))
{
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
}
mat_image_ = *image_mat.get();
image_width_ = image_mat->cols;
image_height_ = image_mat->rows;
const std::vector<double> &face_box =
cc->Inputs().Tag(kFaceBoxTag).Get<std::vector<double>>();
image_width_ = mat_image_.cols;
image_height_ = mat_image_.rows;
if (!face_box.empty())
MP_RETURN_IF_ERROR(BilateralFilter(cc, face_box));
// Copy the rendered image to output.
uchar *image_mat_ptr = image_mat->data;
MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr));
MP_RETURN_IF_ERROR(RenderToCpu(cc));
return absl::OkStatus();
}
@ -189,92 +139,28 @@ namespace mediapipe
return absl::OkStatus();
}
absl::Status BilateralCalculator::RenderToCpu(
CalculatorContext *cc, const ImageFormat::Format &target_format,
uchar *data_image)
{
auto output_frame = absl::make_unique<ImageFrame>(
target_format, image_width_, image_height_);
output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image,
ImageFrame::kDefaultAlignmentBoundary);
if (cc->Outputs().HasTag(kImageNewTag))
{
cc->Outputs()
.Tag(kImageNewTag)
.Add(output_frame.release(), cc->InputTimestamp());
}
return absl::OkStatus();
}
absl::Status BilateralCalculator::CreateRenderTargetCpu(
CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format)
{
if (image_frame_available_)
{
const auto &input_frame =
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
int target_mat_type;
switch (input_frame.Format())
{
case ImageFormat::SRGBA:
*target_format = ImageFormat::SRGBA;
target_mat_type = CV_8UC4;
break;
case ImageFormat::SRGB:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
case ImageFormat::GRAY8:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
default:
return absl::UnknownError("Unexpected image frame format.");
break;
}
image_mat = absl::make_unique<cv::Mat>(
input_frame.Height(), input_frame.Width(), target_mat_type);
auto input_mat = formats::MatView(&input_frame);
if (input_frame.Format() == ImageFormat::GRAY8)
{
cv::Mat rgb_mat;
cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
rgb_mat.copyTo(*image_mat);
}
else
{
input_mat.copyTo(*image_mat);
}
}
else
{
image_mat = absl::make_unique<cv::Mat>(
150, 150, CV_8UC4,
cv::Scalar::all(255));
*target_format = ImageFormat::SRGBA;
}
return absl::OkStatus();
}
absl::Status BilateralCalculator::BilateralFilter(CalculatorContext *cc,
const std::vector<double> &face_box)
{
cv::Mat patch_face = mat_image_(cv::Range(face_box[1], face_box[3]),
cv::Mat patch_wow = mat_image_(cv::Range(face_box[1], face_box[3]),
cv::Range(face_box[0], face_box[2]));
cv::Mat patch_new, patch_wow;
cv::cvtColor(patch_face, patch_wow, cv::COLOR_RGBA2RGB);
cv::bilateralFilter(patch_wow, patch_new, 12, 50, 50);
cv::cvtColor(patch_new, patch_new, cv::COLOR_RGB2RGBA);
cv::cvtColor(patch_wow, patch_wow, CV_RGBA2RGB);
cv::bilateralFilter(patch_wow, out_mat, 12, 50, 50);
cv::cvtColor(out_mat, out_mat, CV_RGB2RGBA);
return absl::OkStatus();
}
absl::Status BilateralCalculator::RenderToCpu(CalculatorContext *cc)
{
auto out_mat_ptr = absl::make_unique<cv::Mat>(out_mat);
if (cc->Outputs().HasTag(kOutTag))
{
cc->Outputs()
.Tag(kOutTag)
.Add(out_mat_ptr.release(), cc->InputTimestamp());
}
return absl::OkStatus();
}

212
mediapipe/calculators/beauty/draw_lipstick_calculator.cc
View File

@ -12,18 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <math.h>
#include <algorithm>
#include <cmath>
#include <map>
#include <string>
#include <memory>
#include "absl/strings/str_cat.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
@ -31,16 +21,18 @@
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/vector.h"
using namespace std;
namespace mediapipe
{
namespace
{
constexpr char kMaskTag[] = "MASK";
constexpr char kImageFrameTag[] = "IMAGE";
constexpr char kMatTag[] = "MAT";
inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
} // namespace
@ -59,21 +51,16 @@ namespace mediapipe
absl::Status Close(CalculatorContext *cc) override;
private:
absl::Status CreateRenderTargetCpu(CalculatorContext *cc,
std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format);
absl::Status RenderToCpu(
CalculatorContext *cc, const ImageFormat::Format &target_format,
uchar *data_image, std::unique_ptr<cv::Mat> &image_mat);
absl::Status DrawLipstick(CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format,
const std::unordered_map<std::string, cv::Mat> &mask_vec);
CalculatorContext *cc);
absl::Status DrawLipstick(CalculatorContext *cc,
std::unordered_map<std::string, cv::Mat> &mask_vec);
// Indicates if image frame is available as input.
bool image_frame_available_ = false;
std::unordered_map<std::string, cv::Mat> all_masks;
cv::Mat spec_lips_mask;
cv::Mat mat_image_;
};
REGISTER_CALCULATOR(DrawLipstickCalculator);
@ -81,10 +68,10 @@ namespace mediapipe
{
CHECK_GE(cc->Inputs().NumEntries(), 1);
if (cc->Inputs().HasTag(kImageFrameTag))
if (cc->Inputs().HasTag(kMatTag))
{
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
CHECK(cc->Outputs().HasTag(kImageFrameTag));
cc->Inputs().Tag(kMatTag).Set<cv::Mat>();
CHECK(cc->Outputs().HasTag(kMatTag));
}
// Data streams to render.
@ -104,9 +91,13 @@ namespace mediapipe
}
}
if (cc->Outputs().HasTag(kImageFrameTag))
if (cc->Outputs().HasTag(kMatTag))
{
cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
cc->Outputs().Tag(kMatTag).Set<cv::Mat>();
}
if (cc->Outputs().HasTag(kMaskTag))
{
cc->Outputs().Tag(kMaskTag).Set<cv::Mat>();
}
return absl::OkStatus();
@ -116,13 +107,13 @@ namespace mediapipe
{
cc->SetOffset(TimestampDiff(0));
if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc))
if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc))
{
image_frame_available_ = true;
}
// Set the output header based on the input header (if present).
const char *tag = kImageFrameTag;
const char *tag = kMatTag;
if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty())
{
const auto &input_header =
@ -136,35 +127,34 @@ namespace mediapipe
absl::Status DrawLipstickCalculator::Process(CalculatorContext *cc)
{
if (cc->Inputs().HasTag(kImageFrameTag) &&
cc->Inputs().Tag(kImageFrameTag).IsEmpty())
if (cc->Inputs().HasTag(kMatTag) &&
cc->Inputs().Tag(kMatTag).IsEmpty())
{
return absl::OkStatus();
}
// Initialize render target, drawn with OpenCV.
std::unique_ptr<cv::Mat> image_mat;
ImageFormat::Format target_format;
if (cc->Outputs().HasTag(kImageFrameTag))
{
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
}
const cv::Mat &input_mat =
cc->Inputs().Tag(kMatTag).Get<cv::Mat>();
mat_image_ = input_mat.clone();
if (cc->Inputs().HasTag(kMaskTag) &&
!cc->Inputs().Tag(kMaskTag).IsEmpty())
{
const std::vector<std::unordered_map<std::string, cv::Mat>> &mask_vec =
cc->Inputs().Tag(kMaskTag).Get<std::vector<std::unordered_map<std::string, cv::Mat>>>();
if (mask_vec.size() > 0)
{
for (auto mask : mask_vec)
MP_RETURN_IF_ERROR(DrawLipstick(cc, image_mat, &target_format, mask));
for (auto mask : mask_vec){
MP_RETURN_IF_ERROR(DrawLipstick(cc, mask));
}
}
// Copy the rendered image to output.
uchar *image_mat_ptr = image_mat->data;
MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat));
}
MP_RETURN_IF_ERROR(RenderToCpu(cc));
return absl::OkStatus();
}
@ -175,163 +165,71 @@ namespace mediapipe
}
absl::Status DrawLipstickCalculator::RenderToCpu(
CalculatorContext *cc, const ImageFormat::Format &target_format,
uchar *data_image, std::unique_ptr<cv::Mat> &image_mat)
CalculatorContext *cc)
{
cv::Mat mat_image_ = *image_mat.get();
auto output_frame = absl::make_unique<cv::Mat>(mat_image_);
auto output_frame = absl::make_unique<ImageFrame>(
target_format, mat_image_.cols, mat_image_.rows);
output_frame->CopyPixelData(target_format, mat_image_.cols, mat_image_.rows, data_image,
ImageFrame::kDefaultAlignmentBoundary);
if (cc->Outputs().HasTag(kImageFrameTag))
if (cc->Outputs().HasTag(kMatTag))
{
cc->Outputs()
.Tag(kImageFrameTag)
.Tag(kMatTag)
.Add(output_frame.release(), cc->InputTimestamp());
}
return absl::OkStatus();
}
spec_lips_mask.convertTo(spec_lips_mask, CV_32F, 1.0 / 255);
auto output_frame2 = absl::make_unique<cv::Mat>(spec_lips_mask);
absl::Status DrawLipstickCalculator::CreateRenderTargetCpu(
CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format)
if (cc->Outputs().HasTag(kMaskTag))
{
if (image_frame_available_)
{
const auto &input_frame =
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
int target_mat_type;
switch (input_frame.Format())
{
case ImageFormat::SRGBA:
*target_format = ImageFormat::SRGBA;
target_mat_type = CV_8UC4;
break;
case ImageFormat::SRGB:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
case ImageFormat::GRAY8:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
default:
return absl::UnknownError("Unexpected image frame format.");
break;
}
image_mat = absl::make_unique<cv::Mat>(
input_frame.Height(), input_frame.Width(), target_mat_type);
auto input_mat = formats::MatView(&input_frame);
if (input_frame.Format() == ImageFormat::GRAY8)
{
cv::Mat rgb_mat;
cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
rgb_mat.copyTo(*image_mat);
}
else
{
input_mat.copyTo(*image_mat);
}
}
else
{
image_mat = absl::make_unique<cv::Mat>(
150, 150, CV_8UC4,
cv::Scalar(cv::Scalar::all(255)));
*target_format = ImageFormat::SRGBA;
cc->Outputs()
.Tag(kMaskTag)
.Add(output_frame2.release(), cc->InputTimestamp());
}
return absl::OkStatus();
}
absl::Status DrawLipstickCalculator::DrawLipstick(CalculatorContext *cc,
std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format,
const std::unordered_map<std::string, cv::Mat> &mask_vec)
std::unordered_map<std::string, cv::Mat> &mask_vec)
{
cv::Mat mat_image__ = *image_mat.get();
cv::Mat spec_lips_mask, upper_lips_mask, lower_lips_mask;
spec_lips_mask = cv::Mat::zeros(mat_image__.size(), CV_32F);
upper_lips_mask = cv::Mat::zeros(mat_image__.size(), CV_32F);
lower_lips_mask = cv::Mat::zeros(mat_image__.size(), CV_32F);
upper_lips_mask = mask_vec.find("UPPER_LIP")->second;
lower_lips_mask = mask_vec.find("LOWER_LIP")->second;
cv::Mat upper_lips_mask = mask_vec.find("UPPER_LIP")->second;
cv::Mat lower_lips_mask = mask_vec.find("LOWER_LIP")->second;
spec_lips_mask = upper_lips_mask + lower_lips_mask;
spec_lips_mask.convertTo(spec_lips_mask, CV_8U);
cv::resize(spec_lips_mask, spec_lips_mask, mat_image_.size(), cv::INTER_LINEAR);
cv::resize(spec_lips_mask, spec_lips_mask, mat_image__.size(), cv::INTER_LINEAR);
cv::Rect rect = cv::boundingRect(spec_lips_mask);
std::vector<int> x, y;
std::vector<cv::Point> location;
cv::findNonZero(spec_lips_mask, location);
for (auto &i : location)
if (!rect.empty())
{
x.push_back(i.x);
y.push_back(i.y);
}
if (!(x.empty()) && !(y.empty()))
{
double min_y, max_y, max_x, min_x;
cv::minMaxLoc(y, &min_y, &max_y);
cv::minMaxLoc(x, &min_x, &max_x);
double min_y = rect.y, max_y = rect.y + rect.height,
max_x = rect.x + rect.width, min_x = rect.x;
cv::Mat lips_crop_mask = spec_lips_mask(cv::Range(min_y, max_y), cv::Range(min_x, max_x));
lips_crop_mask.convertTo(lips_crop_mask, CV_32F, 1.0 / 255);
cv::Mat lips_crop = cv::Mat(mat_image__(cv::Range(min_y, max_y), cv::Range(min_x, max_x)));
cv::Mat lips_crop = cv::Mat(mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x)));
cv::Mat lips_blend = cv::Mat(lips_crop.size().height, lips_crop.size().width, CV_32FC4, cv::Scalar(255.0, 0, 0, 0));
std::vector<cv::Mat> channels(4);
cv::split(lips_blend, channels);
channels[3] = lips_crop_mask * 20;
cv::merge(channels, lips_blend);
cv::Mat tmp_lip_mask;
channels[3].convertTo(tmp_lip_mask, CV_32F, 1.0 / 255);
channels[3].convertTo(tmp_lip_mask, CV_32FC1, 1.0 / 255);
cv::split(lips_blend, channels);
for (auto &ch : channels)
{
cv::multiply(ch, tmp_lip_mask, ch, 1.0, CV_32F);
}
cv::merge(channels, lips_blend);
cv::merge(std::vector{tmp_lip_mask, tmp_lip_mask, tmp_lip_mask, tmp_lip_mask}, tmp_lip_mask);
cv::multiply(lips_blend, tmp_lip_mask, lips_blend, 1.0, CV_32F);
cv::subtract(1.0, tmp_lip_mask, tmp_lip_mask, cv::noArray(), CV_32F);
cv::split(lips_crop, channels);
for (auto &ch : channels)
{
cv::multiply(ch, tmp_lip_mask, ch, 1.0, CV_8U);
}
cv::merge(channels, lips_crop);
cv::multiply(lips_crop, tmp_lip_mask, lips_crop, 1.0, CV_8U);
cv::add(lips_blend, lips_crop, lips_crop, cv::noArray(), CV_8U);
lips_crop = cv::abs(lips_crop);
cvtColor(lips_crop, lips_crop, cv::COLOR_RGBA2RGB);
cv::Mat slice = mat_image__(cv::Range(min_y, max_y), cv::Range(min_x, max_x));
cv::Mat slice = mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x));
lips_crop_mask.convertTo(lips_crop_mask, slice.type());
slice.copyTo(slice, lips_crop_mask);

131
mediapipe/calculators/beauty/imageframe_to_mat_calculator.cc Normal file
View File

@ -0,0 +1,131 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/status.h"
using namespace std;
namespace mediapipe
{
namespace api2
{
class ImageFrameToMatCalculator : public Node
{
public:
static constexpr Input<ImageFrame> kImageFrame{"IMAGE"};
static constexpr Output<cv::Mat> kOut{"MAT"};
MEDIAPIPE_NODE_CONTRACT(kImageFrame, kOut);
static absl::Status UpdateContract(CalculatorContract *cc)
{
RET_CHECK(kOut(cc).IsConnected())
<< "At least one output stream is expected.";
return absl::OkStatus();
}
absl::Status Process(CalculatorContext *cc)
{
// Initialize render target, drawn with OpenCV.
std::unique_ptr<cv::Mat> image_mat;
ImageFormat::Format target_format;
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat));
return absl::OkStatus();
}
absl::Status Close(CalculatorContext *cc)
{
return absl::OkStatus();
}
absl::Status RenderToCpu(
CalculatorContext *cc, const ImageFormat::Format &target_format, std::unique_ptr<cv::Mat> &image_mat)
{
auto output_frame = absl::make_unique<cv::Mat>(*image_mat);
kOut(cc).Send(std::move(output_frame));
return absl::OkStatus();
}
absl::Status CreateRenderTargetCpu(
CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format)
{
if (!kImageFrame(cc).IsEmpty())
{
const auto &input_frame = *kImageFrame(cc);
int target_mat_type;
switch (input_frame.Format())
{
case ImageFormat::SRGBA:
*target_format = ImageFormat::SRGBA;
target_mat_type = CV_8UC4;
break;
case ImageFormat::SRGB:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
case ImageFormat::GRAY8:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
default:
return absl::UnknownError("Unexpected image frame format.");
break;
}
image_mat = absl::make_unique<cv::Mat>(
input_frame.Height(), input_frame.Width(), target_mat_type);
auto input_mat = formats::MatView(&input_frame);
if (input_frame.Format() == ImageFormat::GRAY8)
{
cv::Mat rgb_mat;
cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
rgb_mat.copyTo(*image_mat);
}
else
{
input_mat.copyTo(*image_mat);
}
}
else
{
image_mat = absl::make_unique<cv::Mat>(
1920, 1080, CV_8UC4,
cv::Scalar(cv::Scalar::all(255)));
*target_format = ImageFormat::SRGBA;
}
return absl::OkStatus();
}
};
MEDIAPIPE_REGISTER_NODE(ImageFrameToMatCalculator); // namespace mediapipe
}
}

179
mediapipe/calculators/beauty/merging_images_calculator.cc Normal file
View File

@ -0,0 +1,179 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/strings/str_cat.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/video_stream_header.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/vector.h"
using namespace std;
namespace mediapipe
{
namespace
{
constexpr char kMaskTag[] = "MASK";
constexpr char kMatTag[] = "MAT";
constexpr char kImageFrameTag[] = "IMAGE";
} // namespace
class MergeImagesCalculator : public CalculatorBase
{
public:
MergeImagesCalculator() = default;
~MergeImagesCalculator() override = default;
static absl::Status GetContract(CalculatorContract *cc);
// From Calculator.
absl::Status Open(CalculatorContext *cc) override;
absl::Status Process(CalculatorContext *cc) override;
absl::Status Close(CalculatorContext *cc) override;
private:
absl::Status SmoothEnd(CalculatorContext *cc,
const std::vector<double> &face_box);
cv::Mat mat_image_;
};
REGISTER_CALCULATOR(MergeImagesCalculator);
absl::Status MergeImagesCalculator::GetContract(CalculatorContract *cc)
{
CHECK_GE(cc->Inputs().NumEntries(), 1);
CHECK(cc->Outputs().HasTag(kImageFrameTag));
// Data streams to render.
for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId();
++id)
{
auto tag_and_index = cc->Inputs().TagAndIndexFromId(id);
std::string tag = tag_and_index.first;
if (tag == kMatTag)
{
cc->Inputs().Get(id).Set<cv::Mat>();
}
else
{
// Every other tag defaults to accepting a single object of Mat type.
cc->Inputs().Get(id).Set<cv::Mat>();
}
}
if (cc->Outputs().HasTag(kImageFrameTag))
{
cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
}
return absl::OkStatus();
}
absl::Status MergeImagesCalculator::Open(CalculatorContext *cc)
{
cc->SetOffset(TimestampDiff(0));
// Set the output header based on the input header (if present).
const char *tag = kMatTag;
if (!cc->Inputs().Tag(tag).Header().IsEmpty())
{
const auto &input_header =
cc->Inputs().Tag(tag).Header().Get<VideoHeader>();
auto *output_video_header = new VideoHeader(input_header);
cc->Outputs().Tag(tag).SetHeader(Adopt(output_video_header));
}
return absl::OkStatus();
}
absl::Status MergeImagesCalculator::Process(CalculatorContext *cc)
{
if (cc->Inputs().HasTag(kMatTag) &&
cc->Inputs().Tag(kMatTag).IsEmpty())
{
return absl::OkStatus();
}
const auto &input_mat_c = cc->Inputs().Get(kMatTag, 0).Get<cv::Mat>();
cv::Mat input_mat = input_mat_c.clone();
cv::Mat input_mat2 = input_mat.clone();
cv::Mat mask = cc->Inputs().Get(kMaskTag, 0).Get<cv::Mat>();
cv::cvtColor(mask, mask, cv::COLOR_GRAY2RGBA);
input_mat.convertTo(input_mat, CV_32F);
mat_image_ = mask.mul(input_mat);
int image_number = cc->Inputs().NumEntries() / 2;
cv::Mat all_masks = mask;
for (int i = 1; i < image_number; i++)
{
const auto &input_mat_c = cc->Inputs().Get(kMatTag, i).Get<cv::Mat>();
cv::Mat input_mat = input_mat_c.clone();
mask = cc->Inputs().Get(kMaskTag, i).Get<cv::Mat>();
cv::cvtColor(mask, mask, cv::COLOR_GRAY2RGBA);
input_mat.convertTo(input_mat, CV_32F);
mat_image_ += mask.mul(input_mat);
all_masks += mask;
}
input_mat2.convertTo(input_mat2, CV_32F);
cv::threshold(all_masks, all_masks, 1, 255, CV_THRESH_TRUNC);
input_mat2 = input_mat2.mul(1-(all_masks*255));
input_mat2.convertTo(input_mat2, CV_8U);
mat_image_.convertTo(mat_image_, CV_8U);
mat_image_ += input_mat2;
std::unique_ptr<cv::Mat> image_mat = absl::make_unique<cv::Mat>(
mat_image_.rows, mat_image_.cols, ImageFormat::SRGBA);
mat_image_.convertTo(mat_image_, CV_8U);
mat_image_.copyTo(*image_mat);
uchar *image_mat_ptr = image_mat->data;
// Copy the rendered image to output.
auto output_frame = absl::make_unique<ImageFrame>(
ImageFormat::SRGBA, mat_image_.cols, mat_image_.rows);
output_frame->CopyPixelData(ImageFormat::SRGBA, mat_image_.cols, mat_image_.rows, image_mat_ptr,
ImageFrame::kDefaultAlignmentBoundary);
if (cc->Outputs().HasTag(kImageFrameTag))
{
cc->Outputs()
.Tag(kImageFrameTag)
.Add(output_frame.release(), cc->InputTimestamp());
}
return absl::OkStatus();
}
absl::Status MergeImagesCalculator::Close(CalculatorContext *cc)
{
return absl::OkStatus();
}
} // namespace mediapipe

183
mediapipe/calculators/beauty/smooth_face1_calculator.cc
View File

@ -12,15 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <math.h>
#include <algorithm>
#include <cmath>
#include <iostream>
#include <tuple>
#include <memory>
#include "absl/strings/str_cat.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
@ -31,25 +22,20 @@
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/vector.h"
using namespace std;
namespace mediapipe
{
namespace
{
constexpr char kMaskTag[] = "MASK";
constexpr char kFaceBoxTag[] = "FACEBOX";
constexpr char kImageFrameTag[] = "IMAGE";
enum
{
ATTRIB_VERTEX,
ATTRIB_TEXTURE_POSITION,
NUM_ATTRIBUTES
};
constexpr char kMatTag[] = "MAT";
constexpr char kFaceTag[] = "FACE";
inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
} // namespace
@ -68,13 +54,8 @@ namespace mediapipe
absl::Status Close(CalculatorContext *cc) override;
private:
absl::Status CreateRenderTargetCpu(CalculatorContext *cc,
std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format);
absl::Status RenderToCpu(
CalculatorContext *cc, const ImageFormat::Format &target_format,
uchar *data_image);
CalculatorContext *cc);
absl::Status SmoothFace(CalculatorContext *cc,
const std::unordered_map<std::string, cv::Mat> &mask_vec,
@ -90,7 +71,7 @@ namespace mediapipe
cv::Mat mat_image_;
cv::Mat not_full_face;
std::vector<double> face_box;
std::unique_ptr<cv::Mat> image_mat;
std::pair<cv::Mat, std::vector<double>> face;
};
REGISTER_CALCULATOR(SmoothFaceCalculator1);
@ -98,10 +79,10 @@ namespace mediapipe
{
CHECK_GE(cc->Inputs().NumEntries(), 1);
if (cc->Inputs().HasTag(kImageFrameTag))
if (cc->Inputs().HasTag(kMatTag))
{
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
CHECK(cc->Outputs().HasTag(kImageFrameTag));
cc->Inputs().Tag(kMatTag).Set<cv::Mat>();
CHECK(cc->Outputs().HasTag(kMatTag));
CHECK(cc->Outputs().HasTag(kMaskTag));
}
@ -121,23 +102,23 @@ namespace mediapipe
cc->Inputs().Get(id).Set<cv::Mat>();
}
if (tag == kFaceBoxTag)
if (tag == kFaceTag)
{
cc->Inputs().Get(id).Set<std::vector<std::tuple<double, double, double, double>>>();
}
}
if (cc->Outputs().HasTag(kImageFrameTag))
if (cc->Outputs().HasTag(kMatTag))
{
cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
cc->Outputs().Tag(kMatTag).Set<cv::Mat>();
}
if (cc->Outputs().HasTag(kMaskTag))
{
cc->Outputs().Tag(kMaskTag).Set<cv::Mat>();
}
if (cc->Outputs().HasTag(kFaceBoxTag))
if (cc->Outputs().HasTag(kFaceTag))
{
cc->Outputs().Tag(kFaceBoxTag).Set<std::vector<double>>();
cc->Outputs().Tag(kFaceTag).Set<std::pair<cv::Mat, std::vector<double>>>();
}
return absl::OkStatus();
@ -147,13 +128,13 @@ namespace mediapipe
{
cc->SetOffset(TimestampDiff(0));
if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc))
if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc))
{
image_frame_available_ = true;
}
// Set the output header based on the input header (if present).
const char *tag = kImageFrameTag;
const char *tag = kMatTag;
if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty())
{
const auto &input_header =
@ -167,8 +148,8 @@ namespace mediapipe
absl::Status SmoothFaceCalculator1::Process(CalculatorContext *cc)
{
if (cc->Inputs().HasTag(kImageFrameTag) &&
cc->Inputs().Tag(kImageFrameTag).IsEmpty())
if (cc->Inputs().HasTag(kMatTag) &&
cc->Inputs().Tag(kMatTag).IsEmpty())
{
return absl::OkStatus();
}
@ -176,25 +157,24 @@ namespace mediapipe
// Initialize render target, drawn with OpenCV.
ImageFormat::Format target_format;
if (cc->Outputs().HasTag(kImageFrameTag))
{
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
}
const cv::Mat &input_mat =
cc->Inputs().Tag(kMatTag).Get<cv::Mat>();
mat_image_ = *image_mat.get();
image_width_ = image_mat->cols;
image_height_ = image_mat->rows;
mat_image_ = input_mat.clone();
image_width_ = input_mat.cols;
image_height_ = input_mat.rows;
if (cc->Inputs().HasTag(kMaskTag) &&
!cc->Inputs().Tag(kMaskTag).IsEmpty() &&
cc->Inputs().HasTag(kFaceBoxTag) &&
!cc->Inputs().Tag(kFaceBoxTag).IsEmpty())
cc->Inputs().HasTag(kFaceTag) &&
!cc->Inputs().Tag(kFaceTag).IsEmpty())
{
const std::vector<std::unordered_map<std::string, cv::Mat>> &mask_vec =
cc->Inputs().Tag(kMaskTag).Get<std::vector<std::unordered_map<std::string, cv::Mat>>>();
const std::vector<std::tuple<double, double, double, double>> &face_boxes =
cc->Inputs().Tag(kFaceBoxTag).Get<std::vector<std::tuple<double, double, double, double>>>();
cc->Inputs().Tag(kFaceTag).Get<std::vector<std::tuple<double, double, double, double>>>();
if (mask_vec.size() > 0 && face_boxes.size() > 0)
{
@ -202,9 +182,8 @@ namespace mediapipe
MP_RETURN_IF_ERROR(SmoothFace(cc, mask_vec[i], face_boxes[i]));
}
}
// Copy the rendered image to output.
uchar *image_mat_ptr = image_mat->data;
MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr));
MP_RETURN_IF_ERROR(RenderToCpu(cc));
return absl::OkStatus();
}
@ -215,22 +194,18 @@ namespace mediapipe
}
absl::Status SmoothFaceCalculator1::RenderToCpu(
CalculatorContext *cc, const ImageFormat::Format &target_format,
uchar *data_image)
CalculatorContext *cc)
{
auto output_frame1 = absl::make_unique<ImageFrame>(
target_format, image_width_, image_height_);
auto output_frame1 = absl::make_unique<cv::Mat>(mat_image_);
output_frame1->CopyPixelData(target_format, image_width_, image_height_, data_image,
ImageFrame::kDefaultAlignmentBoundary);
if (cc->Outputs().HasTag(kImageFrameTag))
if (cc->Outputs().HasTag(kMatTag))
{
cc->Outputs()
.Tag(kImageFrameTag)
.Tag(kMatTag)
.Add(output_frame1.release(), cc->InputTimestamp());
}
not_full_face.convertTo(not_full_face, CV_32F, 1.0 / 255);
auto output_frame2 = absl::make_unique<cv::Mat>(not_full_face);
if (cc->Outputs().HasTag(kMaskTag))
@ -240,71 +215,15 @@ namespace mediapipe
.Add(output_frame2.release(), cc->InputTimestamp());
}
auto output_frame3 = absl::make_unique<std::vector<double>>(face_box);
auto output_frame3 = absl::make_unique<std::pair<cv::Mat, std::vector<double>>>(
face);
if (cc->Outputs().HasTag(kFaceBoxTag))
if (cc->Outputs().HasTag(kFaceTag))
{
cc->Outputs()
.Tag(kFaceBoxTag)
.Tag(kFaceTag)
.Add(output_frame3.release(), cc->InputTimestamp());
}
return absl::OkStatus();
}
absl::Status SmoothFaceCalculator1::CreateRenderTargetCpu(
CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format)
{
if (image_frame_available_)
{
const auto &input_frame =
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
int target_mat_type;
switch (input_frame.Format())
{
case ImageFormat::SRGBA:
*target_format = ImageFormat::SRGBA;
target_mat_type = CV_8UC4;
break;
case ImageFormat::SRGB:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
case ImageFormat::GRAY8:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
default:
return absl::UnknownError("Unexpected image frame format.");
break;
}
image_mat = absl::make_unique<cv::Mat>(
input_frame.Height(), input_frame.Width(), target_mat_type);
auto input_mat = formats::MatView(&input_frame);
if (input_frame.Format() == ImageFormat::GRAY8)
{
cv::Mat rgb_mat;
cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
rgb_mat.copyTo(*image_mat);
}
else
{
input_mat.copyTo(*image_mat);
}
}
else
{
image_mat = absl::make_unique<cv::Mat>(
150, 150, CV_8UC4,
cv::Scalar::all(255));
*target_format = ImageFormat::SRGBA;
}
return absl::OkStatus();
}
@ -395,26 +314,18 @@ namespace mediapipe
mat_image_.size(), 0, 0,
cv::INTER_LINEAR);
std::vector<int> x, y;
std::vector<cv::Point> location;
cv::Rect rect = cv::boundingRect(not_full_face);
cv::findNonZero(not_full_face, location);
double min_y, min_x, max_x, max_y;
for (auto &i : location)
if (!rect.empty())
{
x.push_back(i.x);
y.push_back(i.y);
double min_y = rect.y, max_y = rect.y + rect.height,
max_x = rect.x + rect.width, min_x = rect.x;
face.second.push_back(min_x);
face.second.push_back(min_y);
face.second.push_back(max_x);
face.second.push_back(max_y);
face.first = mat_image_;
}
cv::minMaxLoc(x, &min_x, &max_x);
cv::minMaxLoc(y, &min_y, &max_y);
face_box.push_back(min_x);
face_box.push_back(min_y);
face_box.push_back(max_x);
face_box.push_back(max_y);
return absl::OkStatus();
}

203
mediapipe/calculators/beauty/smooth_face2_calculator.cc
View File

@ -12,15 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <math.h>
#include <algorithm>
#include <cmath>
#include <iostream>
#include <tuple>
#include <memory>
#include "absl/strings/str_cat.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
@ -31,10 +22,13 @@
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/vector.h"
using namespace std;
namespace mediapipe
{
namespace
@ -42,16 +36,9 @@ namespace mediapipe
constexpr char kMaskTag[] = "MASK";
constexpr char kFaceBoxTag[] = "FACEBOX";
constexpr char kImageFrameTag[] = "IMAGE";
constexpr char kMatTag[] = "MAT";
constexpr char kImageNewTag[] = "IMAGE2";
enum
{
ATTRIB_VERTEX,
ATTRIB_TEXTURE_POSITION,
NUM_ATTRIBUTES
};
inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
} // namespace
@ -69,14 +56,8 @@ namespace mediapipe
absl::Status Close(CalculatorContext *cc) override;
private:
absl::Status CreateRenderTargetCpu(CalculatorContext *cc,
std::unique_ptr<cv::Mat> &image_mat,
std::unique_ptr<cv::Mat> &new_mat,
ImageFormat::Format *target_format);
absl::Status RenderToCpu(
CalculatorContext *cc, const ImageFormat::Format &target_format,
uchar *data_image);
CalculatorContext *cc);
absl::Status SmoothEnd(CalculatorContext *cc,
const std::vector<double> &face_box);
@ -89,9 +70,6 @@ namespace mediapipe
cv::Mat mat_image_;
cv::Mat new_image_;
cv::Mat not_full_face;
std::unique_ptr<cv::Mat> image_mat;
std::unique_ptr<cv::Mat> new_mat;
std::unique_ptr<cv::Mat> nff_mat;
};
REGISTER_CALCULATOR(SmoothFaceCalculator2);
@ -99,40 +77,28 @@ namespace mediapipe
{
CHECK_GE(cc->Inputs().NumEntries(), 1);
if (cc->Inputs().HasTag(kImageFrameTag))
if (cc->Inputs().HasTag(kMatTag))
{
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
CHECK(cc->Outputs().HasTag(kImageFrameTag));
cc->Inputs().Tag(kMatTag).Set<cv::Mat>();
CHECK(cc->Outputs().HasTag(kMatTag));
}
if (cc->Inputs().HasTag(kImageNewTag))
{
cc->Inputs().Tag(kImageNewTag).Set<ImageFrame>();
cc->Inputs().Tag(kImageNewTag).Set<cv::Mat>();
}
if (cc->Inputs().HasTag(kMaskTag))
{
cc->Inputs().Tag(kMaskTag).Set<cv::Mat>();
}
// Data streams to render.
for (CollectionItemId id = cc->Inputs().BeginId(); id < cc->Inputs().EndId();
++id)
if (cc->Inputs().HasTag(kFaceBoxTag))
{
auto tag_and_index = cc->Inputs().TagAndIndexFromId(id);
std::string tag = tag_and_index.first;
if (tag == kFaceBoxTag)
{
cc->Inputs().Get(id).Set<std::vector<double>>();
}
else if (tag.empty())
{
// Empty tag defaults to accepting a single object of Mat type.
cc->Inputs().Get(id).Set<cv::Mat>();
}
cc->Inputs().Tag(kFaceBoxTag).Set<std::pair<cv::Mat, std::vector<double>>>();
}
if (cc->Outputs().HasTag(kImageFrameTag))
if (cc->Outputs().HasTag(kMatTag))
{
cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
cc->Outputs().Tag(kMatTag).Set<cv::Mat>();
}
return absl::OkStatus();
@ -142,13 +108,13 @@ namespace mediapipe
{
cc->SetOffset(TimestampDiff(0));
if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc))
if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc))
{
image_frame_available_ = true;
}
// Set the output header based on the input header (if present).
const char *tag = kImageFrameTag;
const char *tag = kMatTag;
if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty())
{
const auto &input_header =
@ -162,8 +128,9 @@ namespace mediapipe
absl::Status SmoothFaceCalculator2::Process(CalculatorContext *cc)
{
if (cc->Inputs().HasTag(kImageFrameTag) &&
cc->Inputs().Tag(kImageFrameTag).IsEmpty())
if (cc->Inputs().HasTag(kMatTag) &&
cc->Inputs().Tag(kMatTag).IsEmpty())
{
return absl::OkStatus();
}
@ -183,28 +150,33 @@ namespace mediapipe
return absl::OkStatus();
}
// Initialize render target, drawn with OpenCV.
ImageFormat::Format target_format;
not_full_face = cc->Inputs().Tag(kMaskTag).Get<cv::Mat>();
not_full_face.convertTo(not_full_face, CV_8U, 255);
if (cc->Outputs().HasTag(kImageFrameTag))
const cv::Mat &input_mat =
cc->Inputs().Tag(kMatTag).Get<cv::Mat>();
mat_image_ = input_mat.clone();
const cv::Mat &input_new =
cc->Inputs().Tag(kImageNewTag).Get<cv::Mat>();
new_image_ = input_new.clone();
image_width_ = input_mat.cols;
image_height_ = input_mat.rows;
const auto &face_box_pair =
cc->Inputs().Tag(kFaceBoxTag).Get<std::pair<cv::Mat, std::vector<double>>>();
const auto &face_box = face_box_pair.second;
if (!face_box.empty())
{
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, new_mat, &target_format));
MP_RETURN_IF_ERROR(SmoothEnd(cc, face_box));
}
not_full_face = cc->Inputs().Tag(kMaskTag).Get<cv::Mat>();
new_image_ = *new_mat.get();
mat_image_ = *image_mat.get();
image_width_ = image_mat->cols;
image_height_ = image_mat->rows;
const std::vector<double> &face_box =
cc->Inputs().Tag(kFaceBoxTag).Get<std::vector<double>>();
MP_RETURN_IF_ERROR(SmoothEnd(cc, face_box));
// Copy the rendered image to output.
uchar *image_mat_ptr = image_mat->data;
MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr));
MP_RETURN_IF_ERROR(RenderToCpu(cc));
return absl::OkStatus();
}
@ -215,114 +187,31 @@ namespace mediapipe
}
absl::Status SmoothFaceCalculator2::RenderToCpu(
CalculatorContext *cc, const ImageFormat::Format &target_format,
uchar *data_image)
CalculatorContext *cc)
{
auto output_frame = absl::make_unique<ImageFrame>(
target_format, image_width_, image_height_);
auto output_frame = absl::make_unique<cv::Mat>(mat_image_);
output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image,
ImageFrame::kDefaultAlignmentBoundary);
if (cc->Outputs().HasTag(kImageFrameTag))
if (cc->Outputs().HasTag(kMatTag))
{
cc->Outputs()
.Tag(kImageFrameTag)
.Tag(kMatTag)
.Add(output_frame.release(), cc->InputTimestamp());
}
return absl::OkStatus();
}
absl::Status SmoothFaceCalculator2::CreateRenderTargetCpu(
CalculatorContext *cc,
std::unique_ptr<cv::Mat> &image_mat,
std::unique_ptr<cv::Mat> &new_mat,
ImageFormat::Format *target_format)
{
if (image_frame_available_)
{
const auto &input_frame =
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
const auto &new_frame =
cc->Inputs().Tag(kImageNewTag).Get<ImageFrame>();
int target_mat_type;
switch (input_frame.Format())
{
case ImageFormat::SRGBA:
*target_format = ImageFormat::SRGBA;
target_mat_type = CV_8UC4;
break;
case ImageFormat::SRGB:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
case ImageFormat::GRAY8:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
default:
return absl::UnknownError("Unexpected image frame format.");
break;
}
image_mat = absl::make_unique<cv::Mat>(
input_frame.Height(), input_frame.Width(), target_mat_type);
auto input_mat = formats::MatView(&input_frame);
new_mat = absl::make_unique<cv::Mat>(
new_frame.Height(), new_frame.Width(), target_mat_type);
auto new_input_mat = formats::MatView(&new_frame);
if (input_frame.Format() == ImageFormat::GRAY8)
{
cv::Mat rgb_mat;
cv::Mat rgb_mat2;
cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
rgb_mat.copyTo(*image_mat);
cv::cvtColor(new_input_mat, rgb_mat2, CV_GRAY2RGB);
rgb_mat2.copyTo(*new_mat);
}
else
{
input_mat.copyTo(*image_mat);
new_input_mat.copyTo(*new_mat);
}
}
else
{
image_mat = absl::make_unique<cv::Mat>(
150, 150, CV_8UC4,
cv::Scalar::all(255));
nff_mat = absl::make_unique<cv::Mat>(
150, 150, CV_8UC4,
cv::Scalar::all(255));
new_mat = absl::make_unique<cv::Mat>(
150, 150, CV_8UC4,
cv::Scalar::all(255));
*target_format = ImageFormat::SRGBA;
}
return absl::OkStatus();
}
absl::Status SmoothFaceCalculator2::SmoothEnd(CalculatorContext *cc,
const std::vector<double> &face_box)
{
cv::Mat patch_face = mat_image_(cv::Range(face_box[1], face_box[3]),
cv::Range(face_box[0], face_box[2]));
cv::Mat patch_new = new_image_(cv::Range(face_box[1], face_box[3]),
cv::Range(face_box[0], face_box[2]));
cv::Mat patch_nff = not_full_face(cv::Range(face_box[1], face_box[3]),
cv::Range(face_box[0], face_box[2]));
cv::Mat patch_new_nff, patch_new_mask, patch, patch_face_nff;
patch_new.copyTo(patch_new_nff, patch_nff);
new_image_.copyTo(patch_new_nff, patch_nff);
patch_face.copyTo(patch_face_nff, patch_nff);
cv::cvtColor(patch_face_nff, patch_face_nff, cv::COLOR_RGBA2RGB);

173
mediapipe/calculators/beauty/whiten_teeth_calculator.cc
View File

@ -12,16 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <math.h>
#include <algorithm>
#include <cmath>
#include <memory>
#include "absl/strings/str_cat.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
@ -29,17 +21,19 @@
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/vector.h"
using namespace std;
namespace mediapipe
{
namespace
{
constexpr char kMaskTag[] = "MASK";
constexpr char kImageFrameTag[] = "IMAGE";
constexpr char kMatTag[] = "MAT";
inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; }
} // namespace
@ -58,20 +52,15 @@ namespace mediapipe
absl::Status Close(CalculatorContext *cc) override;
private:
absl::Status CreateRenderTargetCpu(CalculatorContext *cc,
std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format);
absl::Status RenderToCpu(
CalculatorContext *cc, const ImageFormat::Format &target_format,
uchar *data_image, std::unique_ptr<cv::Mat> &image_mat);
CalculatorContext *cc);
absl::Status WhitenTeeth(CalculatorContext *cc, ImageFormat::Format *target_format,
absl::Status WhitenTeeth(CalculatorContext *cc,
const std::unordered_map<std::string, cv::Mat> &mask_vec);
// Indicates if image frame is available as input.
bool image_frame_available_ = false;
std::unique_ptr<cv::Mat> image_mat;
cv::Mat mouth;
cv::Mat mat_image_;
int image_width_;
int image_height_;
@ -82,10 +71,10 @@ namespace mediapipe
{
CHECK_GE(cc->Inputs().NumEntries(), 1);
if (cc->Inputs().HasTag(kImageFrameTag))
if (cc->Inputs().HasTag(kMatTag))
{
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
CHECK(cc->Outputs().HasTag(kImageFrameTag));
cc->Inputs().Tag(kMatTag).Set<cv::Mat>();
CHECK(cc->Outputs().HasTag(kMatTag));
}
// Data streams to render.
@ -105,9 +94,13 @@ namespace mediapipe
}
}
if (cc->Outputs().HasTag(kImageFrameTag))
if (cc->Outputs().HasTag(kMatTag))
{
cc->Outputs().Tag(kImageFrameTag).Set<ImageFrame>();
cc->Outputs().Tag(kMatTag).Set<cv::Mat>();
}
if (cc->Outputs().HasTag(kMaskTag))
{
cc->Outputs().Tag(kMaskTag).Set<cv::Mat>();
}
return absl::OkStatus();
@ -117,16 +110,13 @@ namespace mediapipe
{
cc->SetOffset(TimestampDiff(0));
if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc))
if (cc->Inputs().HasTag(kMatTag) || HasImageTag(cc))
{
image_frame_available_ = true;
}
else
{
}
// Set the output header based on the input header (if present).
const char *tag = kImageFrameTag;
const char *tag = kMatTag;
if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty())
{
const auto &input_header =
@ -140,22 +130,21 @@ namespace mediapipe
absl::Status WhitenTeethCalculator::Process(CalculatorContext *cc)
{
if (cc->Inputs().HasTag(kImageFrameTag) &&
cc->Inputs().Tag(kImageFrameTag).IsEmpty())
if (cc->Inputs().HasTag(kMatTag) &&
cc->Inputs().Tag(kMatTag).IsEmpty())
{
return absl::OkStatus();
}
// Initialize render target, drawn with OpenCV.
ImageFormat::Format target_format;
if (cc->Outputs().HasTag(kImageFrameTag))
{
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
}
mat_image_ = *image_mat.get();
image_width_ = image_mat->cols;
image_height_ = image_mat->rows;
const cv::Mat &input_mat =
cc->Inputs().Tag(kMatTag).Get<cv::Mat>();
mat_image_ = input_mat.clone();
image_width_ = input_mat.cols;
image_height_ = input_mat.rows;
if (cc->Inputs().HasTag(kMaskTag) &&
!cc->Inputs().Tag(kMaskTag).IsEmpty())
@ -165,12 +154,11 @@ namespace mediapipe
if (mask_vec.size() > 0)
{
for (auto mask : mask_vec)
MP_RETURN_IF_ERROR(WhitenTeeth(cc, &target_format, mask));
MP_RETURN_IF_ERROR(WhitenTeeth(cc, mask));
}
}
// Copy the rendered image to output.
uchar *image_mat_ptr = image_mat->data;
MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat));
MP_RETURN_IF_ERROR(RenderToCpu(cc));
return absl::OkStatus();
}
@ -181,124 +169,57 @@ namespace mediapipe
}
absl::Status WhitenTeethCalculator::RenderToCpu(
CalculatorContext *cc, const ImageFormat::Format &target_format,
uchar *data_image, std::unique_ptr<cv::Mat> &image_mat)
CalculatorContext *cc)
{
auto output_frame = absl::make_unique<cv::Mat>(mat_image_);
cv::Mat mat_image_ = *image_mat.get();
auto output_frame = absl::make_unique<ImageFrame>(
target_format, image_width_, image_height_);
output_frame->CopyPixelData(target_format, image_width_, image_height_, data_image,
ImageFrame::kDefaultAlignmentBoundary);
if (cc->Outputs().HasTag(kImageFrameTag))
if (cc->Outputs().HasTag(kMatTag))
{
cc->Outputs()
.Tag(kImageFrameTag)
.Tag(kMatTag)
.Add(output_frame.release(), cc->InputTimestamp());
}
return absl::OkStatus();
}
mouth.convertTo(mouth, CV_32F, 255);
auto output_frame2 = absl::make_unique<cv::Mat>(mouth);
absl::Status WhitenTeethCalculator::CreateRenderTargetCpu(
CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format)
if (cc->Outputs().HasTag(kMaskTag))
{
if (image_frame_available_)
{
const auto &input_frame =
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
int target_mat_type;
switch (input_frame.Format())
{
case ImageFormat::SRGBA:
*target_format = ImageFormat::SRGBA;
target_mat_type = CV_8UC4;
break;
case ImageFormat::SRGB:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
case ImageFormat::GRAY8:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
default:
return absl::UnknownError("Unexpected image frame format.");
break;
}
image_mat = absl::make_unique<cv::Mat>(
input_frame.Height(), input_frame.Width(), target_mat_type);
auto input_mat = formats::MatView(&input_frame);
if (input_frame.Format() == ImageFormat::GRAY8)
{
cv::Mat rgb_mat;
cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
rgb_mat.copyTo(*image_mat);
}
else
{
input_mat.copyTo(*image_mat);
}
}
else
{
image_mat = absl::make_unique<cv::Mat>(
150, 150, CV_8UC4,
cv::Scalar(255, 255,
255));
*target_format = ImageFormat::SRGBA;
cc->Outputs()
.Tag(kMaskTag)
.Add(output_frame2.release(), cc->InputTimestamp());
}
return absl::OkStatus();
}
absl::Status WhitenTeethCalculator::WhitenTeeth(CalculatorContext *cc,
ImageFormat::Format *target_format,
const std::unordered_map<std::string, cv::Mat> &mask_vec)
{
cv::Mat mouth_mask, mouth;
mouth_mask = cv::Mat::zeros(mat_image_.size(), CV_32F);
mouth_mask = mask_vec.find("MOUTH_INSIDE")->second.clone();
cv::Mat mouth_mask = mask_vec.find("MOUTH_INSIDE")->second.clone();
cv::resize(mouth_mask, mouth, mat_image_.size(), cv::INTER_LINEAR);
std::vector<int> x, y;
std::vector<cv::Point> location;
cv::Rect rect = cv::boundingRect(mouth);
cv::findNonZero(mouth, location);
for (auto &i : location)
if (!rect.empty())
{
x.push_back(i.x);
y.push_back(i.y);
}
double mouth_min_y = rect.y, mouth_max_y = rect.y + rect.height,
mouth_max_x = rect.x + rect.width, mouth_min_x = rect.x;
if (!(x.empty()) && !(y.empty()))
{
double mouth_min_y, mouth_max_y, mouth_max_x, mouth_min_x;
cv::minMaxLoc(y, &mouth_min_y, &mouth_max_y);
cv::minMaxLoc(x, &mouth_min_x, &mouth_max_x);
double mh = mouth_max_y - mouth_min_y;
double mw = mouth_max_x - mouth_min_x;
cv::Mat mouth_crop_mask;
mouth.convertTo(mouth, CV_32F, 1.0 / 255);
mouth.convertTo(mouth, CV_32F, 1.0 / 255);
if (mh / mw > 0.17)
{
mouth_min_y = static_cast<int>(std::max(mouth_min_y - mh * 0.1, 0.0));
mouth_max_y = static_cast<int>(std::min(mouth_max_y + mh * 0.1, (double)image_height_));
mouth_min_x = static_cast<int>(std::max(mouth_min_x - mw * 0.1, 0.0));
mouth_max_x = static_cast<int>(std::min(mouth_max_x + mw * 0.1, (double)image_width_));
mouth_crop_mask = mouth(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x));
cv::Mat mouth_crop_mask = mouth(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x));
cv::Mat img_hsv, tmp_mask, img_hls;
cv::cvtColor(mat_image_(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)), img_hsv,
cv::COLOR_RGBA2RGB);

1
mediapipe/calculators/core/BUILD
View File

@ -194,6 +194,7 @@ cc_library(
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",

4
mediapipe/calculators/core/begin_loop_calculator.cc
View File

@ -16,6 +16,7 @@
#include <vector>
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
@ -46,6 +47,9 @@ REGISTER_CALCULATOR(BeginLoopDetectionCalculator);
typedef BeginLoopCalculator<std::vector<Matrix>> BeginLoopMatrixCalculator;
REGISTER_CALCULATOR(BeginLoopMatrixCalculator);
typedef BeginLoopCalculator<std::vector<std::vector<cv::Point3d>>> BeginLoopPointsCalculator;
REGISTER_CALCULATOR(BeginLoopPointsCalculator);
// A calculator to process std::vector<std::vector<Matrix>>.
typedef BeginLoopCalculator<std::vector<std::vector<Matrix>>>
BeginLoopMatrixVectorCalculator;

4
mediapipe/calculators/core/end_loop_calculator.cc
View File

@ -57,6 +57,10 @@ typedef EndLoopCalculator<std::vector<std::tuple<double, double, double, double>
EndLoopFaceBoxCalculator;
REGISTER_CALCULATOR(EndLoopFaceBoxCalculator);
typedef EndLoopCalculator<std::vector<std::vector<cv::Point3d>>>
EndLoopPointsCalculator;
REGISTER_CALCULATOR(EndLoopPointsCalculator);
typedef EndLoopCalculator<std::vector<TfLiteTensor>> EndLoopTensorCalculator;
REGISTER_CALCULATOR(EndLoopTensorCalculator);

2
mediapipe/calculators/image_style/fast_utils_calculator.cc
View File

@ -329,7 +329,7 @@ namespace mediapipe
orig_width = size.first;
orig_height = size.second;
CHECK_GT(size.first, 0);
CHECK_GT(orig_height, 0);
CHECK_GT(size.second, 0);
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
mat_image_ = *image_mat.get();

51
mediapipe/calculators/landmarks/BUILD
View File

@ -40,3 +40,54 @@ cc_library(
],
alwayslink = 1,
)
cc_library(
name = "points_to_face_box_calculator",
srcs = ["points_to_face_box_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_highgui",
],
alwayslink = 1,
)
cc_library(
name = "landmarks_to_point_array_calculator",
srcs = ["landmarks_to_point_array_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/formats:landmark_cc_proto",
],
alwayslink = 1,
)
cc_library(
name = "point_vector_to_mask_calculator",
srcs = ["point_vector_to_mask_calculator.cc"],
hdrs = ["point_vector_to_mask_calculator.h"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/formats:landmark_cc_proto",
],
alwayslink = 1,
)

149
mediapipe/calculators/landmarks/forehead_mask_calculator.cc Normal file
View File

@ -0,0 +1,149 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/landmarks/point_vector_to_mask_calculator.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/status.h"
using namespace std;
using namespace cv;
namespace mediapipe
{
namespace api2
{
class PointVectorToMaskCalculator : public Node
{
public:
static constexpr Input<Mat> kImage{"IMAGE"};
static constexpr Input<unordered_map<string, Mat>> kMasks{"MASKS"};
static constexpr Input<tuple<double, double, double, double>> kFaceBox{"FACE_BOX"};
static constexpr Output<Mat> kOut{"FOREHEAD_MASK"};
MEDIAPIPE_NODE_CONTRACT(kImage, kMasks, kFaceBox, kOut);
static absl::Status UpdateContract(CalculatorContract *cc)
{
RET_CHECK(kOut(cc).IsConnected())
<< "At least one output stream is expected.";
return absl::OkStatus();
}
absl::Status Process(CalculatorContext *cc) override
{
if (kImage(cc).IsEmpty() || kMasks(cc).IsEmpty() || kFaceBox(cc).IsEmpty())
{
return absl::OkStatus();
}
const auto &mat_image_ = *kImage(cc);
const auto &masks = *kMasks(cc);
const auto &face_box = *kFaceBox(cc);
MP_RETURN_IF_ERROR(PredictForeheadMask(cc, mat_image_, masks, face_box));
auto output_frame = absl::make_unique<Mat>(new_skin_mask);
kOut(cc).Send(std::move(output_frame));
return absl::OkStatus();
}
absl::Status PredictForeheadMask(CalculatorContext *cc, Mat mat_image_,
unordered_map<string, Mat> mask_vec, tuple<double, double, double, double> face_box)
{
cv::Mat part_forehead_mask = mask_vec.find("PART_FOREHEAD_B")->second.clone();
part_forehead_mask.convertTo(part_forehead_mask, CV_32F, 1.0 / 255);
part_forehead_mask.convertTo(part_forehead_mask, CV_8U);
cv::Mat image_sm, image_sm_hsv, skinMask;
cv::resize(mat_image_, image_sm, cv::Size(mat_image_.cols, mat_image_.rows));
cv::cvtColor(image_sm, image_sm_hsv, cv::COLOR_BGR2HSV);
std::vector<int> x, y;
std::vector<cv::Point> location;
cv::Vec3d hsv_min, hsv_max;
std::vector<cv::Mat> channels(3);
cv::split(image_sm_hsv, channels);
std::vector<std::vector<double>> minx(3), maxx(3);
int c = 0;
for (auto ch : channels)
{
cv::Mat row, mask_row;
double min, max;
for (int i = 0; i < ch.rows; i++)
{
row = ch.row(i);
mask_row = part_forehead_mask.row(i);
cv::minMaxLoc(row, &min, &max, 0, 0, mask_row);
minx[c].push_back(min);
maxx[c].push_back(max);
}
c++;
}
for (int i = 0; i < 3; i++)
{
hsv_min[i] = *std::min_element(minx[i].begin(), minx[i].end());
}
for (int i = 0; i < 3; i++)
{
hsv_max[i] = *std::max_element(maxx[i].begin(), maxx[i].end());
}
cv::Mat _forehead_kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(1, 1));
cv::inRange(image_sm_hsv, hsv_min, hsv_max, skinMask);
cv::erode(skinMask, skinMask, _forehead_kernel, cv::Point(-1, -1), 2);
cv::dilate(skinMask, skinMask, _forehead_kernel, cv::Point(-1, -1), 2);
skinMask.convertTo(skinMask, CV_8U, 1.0 / 255);
cv::findNonZero(skinMask, location);
double max_part_f, x_min_part, x_max_part;
for (auto &i : location)
{
x.push_back(i.x);
y.push_back(i.y);
}
cv::minMaxLoc(y, NULL, &max_part_f);
cv::minMaxLoc(x, &x_min_part, &x_max_part);
cv::Mat new_skin_mask = cv::Mat::zeros(skinMask.size(), CV_8U);
new_skin_mask(cv::Range(get<1>(face_box), max_part_f), cv::Range(x_min_part, x_max_part)) =
skinMask(cv::Range(get<1>(face_box), max_part_f), cv::Range(x_min_part, x_max_part));
return absl::OkStatus();
}
private:
Mat new_skin_mask;
};
MEDIAPIPE_REGISTER_NODE(PointVectorToMaskCalculator);
}
} // namespace mediapipe

246
mediapipe/calculators/landmarks/landmarks_to_mask_calculator.cc
View File

@ -13,17 +13,12 @@
// limitations under the License.
#include "mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h"
#include <math.h>
#include <algorithm>
#include <cmath>
#include <iostream>
#include "absl/memory/memory.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/vector.h"
using namespace std;
namespace mediapipe
{
namespace
@ -33,20 +28,7 @@ namespace mediapipe
constexpr char kVectorTag[] = "VECTOR";
constexpr char kMaskTag[] = "MASK";
constexpr char kFaceBoxTag[] = "FACEBOX";
constexpr char kImageFrameTag[] = "IMAGE";
std::unordered_map<std::string, const std::vector<int>> orderList = {
{"UPPER_LIP", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78}},
{"LOWER_LIP", {61, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
{"FACE_OVAL", {10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356, 454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378, 378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150, 136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162, 21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10}},
{"MOUTH_INSIDE", {78, 191, 80, 81, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95}},
{"LEFT_EYE", {130, 33, 246, 161, 160, 159, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7}},
{"RIGHT_EYE", {362, 398, 384, 385, 386, 387, 388, 466, 263, 249, 390, 373, 374, 380, 381, 382}},
{"LEFT_BROW", {70, 63, 105, 66, 107, 55, 65, 52, 53, 46}},
{"RIGHT_BROW", {336, 296, 334, 293, 301, 300, 283, 282, 295, 285}},
{"LIPS", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
{"PART_FOREHEAD_B", {21, 54, 103, 67, 109, 10, 338, 297, 332, 284, 251, 301, 293, 334, 296, 336, 9, 107, 66, 105, 63, 71}},
};
constexpr char kImageSizeTag[] = "SIZE";
template <class LandmarkType>
bool IsLandmarkVisibleAndPresent(const LandmarkType &landmark,
@ -91,15 +73,10 @@ namespace mediapipe
std::tuple<double, double, double, double> face_box;
std::unique_ptr<cv::Mat> image_mat;
int image_width_;
int image_height_;
float scale_factor_ = 1.0;
bool image_frame_available_ = false;
} // namespace
absl::Status LandmarksToMaskCalculator::GetContract(
@ -113,9 +90,9 @@ namespace mediapipe
<< "Can only one type of landmark can be taken. Either absolute or "
"normalized landmarks.";
if (cc->Inputs().HasTag(kImageFrameTag))
if (cc->Inputs().HasTag(kImageSizeTag))
{
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
}
if (cc->Inputs().HasTag(kLandmarksTag))
@ -142,18 +119,11 @@ namespace mediapipe
{
cc->SetOffset(TimestampDiff(0));
if (cc->Inputs().HasTag(kImageFrameTag))
{
image_frame_available_ = true;
}
return absl::OkStatus();
}
absl::Status LandmarksToMaskCalculator::Process(CalculatorContext *cc)
{
// Check that landmarks are not empty and skip rendering if so.
// Don't emit an empty packet for this timestamp.
if (cc->Inputs().HasTag(kLandmarksTag) &&
cc->Inputs().Tag(kLandmarksTag).IsEmpty())
{
@ -164,27 +134,17 @@ namespace mediapipe
{
return absl::OkStatus();
}
if (cc->Inputs().HasTag(kImageFrameTag) &&
cc->Inputs().Tag(kImageFrameTag).IsEmpty())
{
return absl::OkStatus();
}
// Initialize render target, drawn with OpenCV.
ImageFormat::Format target_format;
std::unordered_map<std::string, cv::Mat> all_masks;
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
cv::Mat mat_image_ = *image_mat.get();
image_width_ = image_mat->cols;
image_height_ = image_mat->rows;
const auto size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
image_width_ = size.first;
image_height_ = size.second;
CHECK_GT(image_width_, 0);
CHECK_GT(image_height_, 0);
MP_RETURN_IF_ERROR(GetMasks(cc, all_masks));
MP_RETURN_IF_ERROR(GetFaceBox(cc));
MP_RETURN_IF_ERROR(RenderToCpu(cc, all_masks));
return absl::OkStatus();
@ -216,116 +176,23 @@ namespace mediapipe
return absl::OkStatus();
}
absl::Status LandmarksToMaskCalculator::CreateRenderTargetCpu(
CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
ImageFormat::Format *target_format)
{
if (image_frame_available_)
{
const auto &input_frame =
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
int target_mat_type;
switch (input_frame.Format())
{
case ImageFormat::SRGBA:
*target_format = ImageFormat::SRGBA;
target_mat_type = CV_8UC4;
break;
case ImageFormat::SRGB:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
case ImageFormat::GRAY8:
*target_format = ImageFormat::SRGB;
target_mat_type = CV_8UC3;
break;
default:
return absl::UnknownError("Unexpected image frame format.");
break;
}
image_mat = absl::make_unique<cv::Mat>(
input_frame.Height(), input_frame.Width(), target_mat_type);
auto input_mat = formats::MatView(&input_frame);
if (input_frame.Format() == ImageFormat::GRAY8)
{
cv::Mat rgb_mat;
cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
rgb_mat.copyTo(*image_mat);
}
else
{
input_mat.copyTo(*image_mat);
}
}
else
{
image_mat = absl::make_unique<cv::Mat>(
150, 150, CV_8UC4,
cv::Scalar(255, 255,
255));
*target_format = ImageFormat::SRGBA;
}
return absl::OkStatus();
}
absl::Status LandmarksToMaskCalculator::GetMasks(CalculatorContext *cc,
std::unordered_map<std::string, cv::Mat> &all_masks)
{
if (cc->Inputs().HasTag(kLandmarksTag))
{
const LandmarkList &landmarks =
cc->Inputs().Tag(kNormLandmarksTag).Get<LandmarkList>();
cv::Mat mask;
std::vector<cv::Point> point_array;
for (const auto &[key, value] : orderList)
{
for (auto order : value)
{
const Landmark &landmark = landmarks.landmark(order);
if (!IsLandmarkVisibleAndPresent<Landmark>(
landmark, false,
0.0, false,
0.0))
{
continue;
}
const auto &point = landmark;
int x = -1;
int y = -1;
CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_,
image_height_, &x, &y));
point_array.push_back(cv::Point(x, y));
}
std::vector<std::vector<cv::Point>> point_vec;
point_vec.push_back(point_array);
mask = cv::Mat::zeros(image_mat->size(), CV_32FC1);
cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA);
mask.convertTo(mask, CV_8U);
all_masks.insert({key, mask});
point_vec.clear();
point_array.clear();
}
}
if (cc->Inputs().HasTag(kNormLandmarksTag))
{
std::vector<int> x_s, y_s;
double box_min_y, box_max_y, box_max_x, box_min_x;
const NormalizedLandmarkList &landmarks =
cc->Inputs().Tag(kNormLandmarksTag).Get<NormalizedLandmarkList>();
cv::Mat mask;
std::vector<cv::Point> point_array;
std::vector<std::vector<cv::Point>> point_vec;
for (const auto &[key, value] : orderList)
{
point_vec = {};
point_array = {};
for (auto order : value)
{
const NormalizedLandmark &landmark = landmarks.landmark(order);
@ -345,88 +212,25 @@ namespace mediapipe
image_height_, &x, &y));
point_array.push_back(cv::Point(x, y));
}
cv::Mat po(point_array);
po.convertTo(po, CV_32F);
cv::Mat min, max;
cv::reduce(po, min, 0, CV_REDUCE_MIN, CV_32F);
cv::reduce(po, max, 0, CV_REDUCE_MAX, CV_32F);
min.at<float>(0,1)*=0.9;
face_box = {min.at<float>(0,0), min.at<float>(0,1), max.at<float>(0,0), max.at<float>(0,1)};
std::vector<std::vector<cv::Point>> point_vec;
point_vec.push_back(point_array);
mask = cv::Mat::zeros(image_mat->size(), CV_32FC1);
mask = cv::Mat::zeros({image_width_, image_height_}, CV_32FC1);
cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA);
mask.convertTo(mask, CV_8U);
all_masks.insert(make_pair(key, mask));
point_vec.clear();
point_array.clear();
all_masks.insert({key, mask});
}
}
return absl::OkStatus();
}
absl::Status LandmarksToMaskCalculator::GetFaceBox(CalculatorContext *cc)
{
std::vector<int> x_s, y_s;
double box_min_y, box_max_y, box_max_x, box_min_x;
if (cc->Inputs().HasTag(kLandmarksTag))
{
const LandmarkList &landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
for (int i = 0; i < landmarks.landmark_size(); ++i)
{
const Landmark &landmark = landmarks.landmark(i);
if (!IsLandmarkVisibleAndPresent<Landmark>(
landmark, false,
0.0, false,
0.0))
{
continue;
}
const auto &point = landmark;
int x = -1;
int y = -1;
CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_,
image_height_, &x, &y));
x_s.push_back(point.x());
x_s.push_back(point.y());
}
cv::minMaxLoc(y_s, &box_min_y, &box_max_y);
cv::minMaxLoc(x_s, &box_min_x, &box_max_x);
box_min_y = box_min_y * 0.9;
face_box = std::make_tuple(box_min_x, box_min_y, box_max_x, box_max_y);
}
if (cc->Inputs().HasTag(kNormLandmarksTag))
{
const NormalizedLandmarkList &landmarks =
cc->Inputs().Tag(kNormLandmarksTag).Get<NormalizedLandmarkList>();
for (int i = 0; i < landmarks.landmark_size(); ++i)
{
const NormalizedLandmark &landmark = landmarks.landmark(i);
if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
landmark, false,
0.0, false,
0.0))
{
continue;
}
const auto &point = landmark;
int x = -1;
int y = -1;
CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_,
image_height_, &x, &y));
x_s.push_back(point.x());
x_s.push_back(point.y());
}
cv::minMaxLoc(y_s, &box_min_y, &box_max_y);
cv::minMaxLoc(x_s, &box_min_x, &box_max_x);
box_min_y = box_min_y * 0.9;
face_box = std::make_tuple(box_min_x, box_min_y, box_max_x, box_max_y);
}
return absl::OkStatus();
}
REGISTER_CALCULATOR(LandmarksToMaskCalculator);
} // namespace mediapipe

13
mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h
View File

@ -41,6 +41,19 @@ namespace mediapipe
// output_stream: "MASK:mask"
// }
std::unordered_map<std::string, const std::vector<int>> orderList = {
{"UPPER_LIP", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78}},
{"LOWER_LIP", {61, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
{"FACE_OVAL", {10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356, 454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378, 378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150, 136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162, 21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10}},
{"MOUTH_INSIDE", {78, 191, 80, 81, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95}},
{"LEFT_EYE", {130, 33, 246, 161, 160, 159, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7}},
{"RIGHT_EYE", {362, 398, 384, 385, 386, 387, 388, 466, 263, 249, 390, 373, 374, 380, 381, 382}},
{"LEFT_BROW", {70, 63, 105, 66, 107, 55, 65, 52, 53, 46}},
{"RIGHT_BROW", {336, 296, 334, 293, 301, 300, 283, 282, 295, 285}},
{"LIPS", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
{"PART_FOREHEAD_B", {21, 54, 103, 67, 109, 10, 338, 297, 332, 284, 251, 301, 293, 334, 296, 336, 9, 107, 66, 105, 63, 71}},
};
class LandmarksToMaskCalculator : public CalculatorBase
{
public:

152
mediapipe/calculators/landmarks/landmarks_to_point_array_calculator.cc Normal file
View File

@ -0,0 +1,152 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/status.h"
using namespace std;
using namespace cv;
namespace mediapipe
{
namespace
{
template <class LandmarkType>
bool IsLandmarkVisibleAndPresent(const LandmarkType &landmark,
bool utilize_visibility,
float visibility_threshold,
bool utilize_presence,
float presence_threshold)
{
if (utilize_visibility && landmark.has_visibility() &&
landmark.visibility() < visibility_threshold)
{
return false;
}
if (utilize_presence && landmark.has_presence() &&
landmark.presence() < presence_threshold)
{
return false;
}
return true;
}
bool NormalizedtoPixelCoordinates(double normalized_x, double normalized_y, double normalized_z,
int image_width, int image_height, double *x_px,
double *y_px, double *z_px)
{
CHECK(x_px != nullptr);
CHECK(y_px != nullptr);
CHECK_GT(image_width, 0);
CHECK_GT(image_height, 0);
if (normalized_x < 0 || normalized_x > 1.0 || normalized_y < 0 ||
normalized_y > 1.0 || normalized_z < 0 ||
normalized_z > 1.0)
{
VLOG(1) << "Normalized coordinates must be between 0.0 and 1.0";
}
*x_px = static_cast<double>(normalized_x) * image_width;
*y_px = static_cast<double>(normalized_y) * image_height;
*z_px = static_cast<double>(normalized_z) * image_width;
return true;
}
}
namespace api2
{
class LandmarksToPointArrayCalculator : public Node
{
public:
static constexpr Input<NormalizedLandmarkList> kNormLandmarks{"NORM_LANDMARKS"};
static constexpr Input<pair<int, int>> kImageSize{"IMAGE_SIZE"};
static constexpr Output<vector<Point3d>> kOut{"POINTS"};
MEDIAPIPE_NODE_CONTRACT(kNormLandmarks, kImageSize, kOut);
static absl::Status UpdateContract(CalculatorContract *cc)
{
RET_CHECK(kOut(cc).IsConnected())
<< "At least one output stream is expected.";
return absl::OkStatus();
}
absl::Status Process(CalculatorContext *cc) override
{
if (kNormLandmarks(cc).IsEmpty())
{
return absl::OkStatus();
}
if (kImageSize(cc).IsEmpty())
{
return absl::OkStatus();
}
const auto &size = *kImageSize(cc);
MP_RETURN_IF_ERROR(GetPoints(cc, size));
auto output_frame = absl::make_unique<vector<Point3d>>(point_array);
kOut(cc).Send(std::move(output_frame));
return absl::OkStatus();
}
absl::Status GetPoints(CalculatorContext *cc, pair<int, int> size)
{
const auto &landmarks = *kNormLandmarks(cc);
point_array = {};
for (int i = 0; i < landmarks.landmark_size(); i++)
{
const NormalizedLandmark &landmark = landmarks.landmark(i);
if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
landmark, false,
0.0, false,
0.0))
{
continue;
}
const auto &point = landmark;
double x = -1;
double y = -1;
double z = -1;
CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), point.z(), size.first,
size.second, &x, &y, &z));
point_array.push_back({x, y, z});
}
return absl::OkStatus();
}
private:
std::vector<cv::Point3d> point_array;
};
MEDIAPIPE_REGISTER_NODE(LandmarksToPointArrayCalculator);
}
} // namespace mediapipe

91
mediapipe/calculators/landmarks/point_vector_to_mask_calculator.cc Normal file
View File

@ -0,0 +1,91 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/landmarks/point_vector_to_mask_calculator.h"
using namespace std;
using namespace cv;
namespace mediapipe
{
namespace api2
{
class PointVectorToMaskCalculator : public Node
{
public:
static constexpr Input<vector<Point3d>> kPoints{"POINTS"};
static constexpr Input<pair<int, int>> kImageSize{"IMAGE_SIZE"};
static constexpr Output<std::unordered_map<std::string, cv::Mat>> kOut{"MASKS"};
MEDIAPIPE_NODE_CONTRACT(kPoints, kImageSize, kOut);
static absl::Status UpdateContract(CalculatorContract *cc)
{
RET_CHECK(kOut(cc).IsConnected())
<< "At least one output stream is expected.";
return absl::OkStatus();
}
absl::Status Process(CalculatorContext *cc) override
{
if (kPoints(cc).IsEmpty())
{
return absl::OkStatus();
}
if (kImageSize(cc).IsEmpty())
{
return absl::OkStatus();
}
const auto &size = *kImageSize(cc);
const auto &points = *kPoints(cc);
MP_RETURN_IF_ERROR(GetMasks(cc, size, points));
auto output_frame = absl::make_unique<std::unordered_map<std::string, cv::Mat>>(all_masks);
kOut(cc).Send(std::move(output_frame), cc->InputTimestamp());
return absl::OkStatus();
}
absl::Status GetMasks(CalculatorContext *cc, pair<int, int> size, vector<Point3d> points)
{
all_masks= {};
for (const auto &[key, value] : orderList)
{
vector<vector<Point>> point_vec = {};
vector<Point> point_array = {};
for (auto order : value)
{
point_array.push_back({points[order].x, points[order].y});
}
point_vec.push_back(point_array);
cv::Mat mask = cv::Mat::zeros({size.first, size.second}, CV_32FC1);
cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA);
mask.convertTo(mask, CV_8U);
all_masks.insert({key, mask});
}
return absl::OkStatus();
}
private:
std::unordered_map<std::string, cv::Mat> all_masks;
};
MEDIAPIPE_REGISTER_NODE(PointVectorToMaskCalculator);
}
} // namespace mediapipe

81
mediapipe/calculators/landmarks/point_vector_to_mask_calculator.h Normal file
View File

@ -0,0 +1,81 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <unordered_map>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe
{
// A calculator that uses face landmarks to create face part masks and facebox for
// visualization. The input should be LandmarkList proto and ImageFrame.
//
// Example config:
// node {
// calculator: "LandmarksToMaskCalculator"
// input_stream: "IMAGE:image"
// input_stream: "NORM_LANDMARKS:face_landmarks"
// output_stream: "FACEBOX:face_box"
// output_stream: "MASK:mask"
// }
std::unordered_map<std::string, const std::vector<int>> orderList = {
{"UPPER_LIP", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78}},
{"LOWER_LIP", {61, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
{"FACE_OVAL", {10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356, 454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378, 378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150, 136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162, 21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10}},
{"MOUTH_INSIDE", {78, 191, 80, 81, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95}},
{"LEFT_EYE", {33, 246, 161, 160, 159, 158, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7}},
{"RIGHT_EYE", {362, 398, 384, 385, 386, 387, 388, 466, 263, 249, 390, 373, 374, 380, 381, 382}},
{"LEFT_IRIS", {468, 469, 470, 471, 472}},
{"RIGHT_IRIS", {473, 474, 475, 476, 477}},
{"LEFT_BROW", {468, 469, 470, 471, 472}},
{"RIGHT_BROW", {336, 296, 334, 293, 301, 300, 283, 282, 295, 285}},
{"LIPS", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
{"PART_FOREHEAD_B", {
21,
54,
103,
67,
109,
10,
338,
297,
332,
284,
251,
301,
293,
334,
296,
336,
9,
107,
66,
105,
63,
71,
}},
};
} // namespace mediapipe

93
mediapipe/calculators/landmarks/points_to_face_box_calculator.cc Normal file
View File

@ -0,0 +1,93 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/status.h"
using namespace std;
using namespace cv;
namespace mediapipe
{
namespace api2
{
class PointsToFaceBoxCalculator : public Node
{
public:
static constexpr Input<vector<Point3d>> kPoints{"POINTS"};
static constexpr Input<pair<int, int>> kImageSize{"IMAGE_SIZE"};
static constexpr Output<tuple<double, double, double, double>> kOut{"FACE_BOX"};
MEDIAPIPE_NODE_CONTRACT(kPoints, kImageSize, kOut);
static absl::Status UpdateContract(CalculatorContract *cc)
{
RET_CHECK(kOut(cc).IsConnected())
<< "At least one output stream is expected.";
return absl::OkStatus();
}
absl::Status Process(CalculatorContext *cc) override
{
if (kPoints(cc).IsEmpty())
{
return absl::OkStatus();
}
if (kImageSize(cc).IsEmpty())
{
return absl::OkStatus();
}
const auto &size = *kImageSize(cc);
const auto &points = *kPoints(cc);
MP_RETURN_IF_ERROR(GetFaceBox(cc, size, points));
auto output_frame = absl::make_unique<tuple<double, double, double, double>>(face_box);
kOut(cc).Send(std::move(output_frame));
return absl::OkStatus();
}
absl::Status GetFaceBox(CalculatorContext *cc, pair<int, int> size, vector<Point3d> points)
{
cv::Mat points_mat(points);
points_mat.convertTo(points_mat, CV_32F);
cv::Mat min, max;
cv::reduce(points_mat, min, 0, CV_REDUCE_MIN, CV_32F);
cv::reduce(points_mat, max, 0, CV_REDUCE_MAX, CV_32F);
min.at<float>(0, 1) *= 0.9;
face_box = {min.at<float>(0, 0), min.at<float>(0, 1), max.at<float>(0, 0), max.at<float>(0, 1)};
return absl::OkStatus();
}
private:
tuple<double, double, double, double> face_box;
};
MEDIAPIPE_REGISTER_NODE(PointsToFaceBoxCalculator);
}
} // namespace mediapipe

1
mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic/AndroidManifest.xml
View File

@ -12,6 +12,7 @@
<!-- For profiling -->
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.MANAGE_EXTERNAL_STORAGE" />
<application
android:allowBackup="true"

1
mediapipe/examples/android/src/java/com/google/mediapipe/apps/deformation/BUILD
View File

@ -55,6 +55,7 @@ android_binary(
ASSETS_DIR+":upperLipCnt.txt",
ASSETS_DIR+":widerLowerLipPts1.txt",
ASSETS_DIR+":widerUpperLipPts1.txt",
"//mediapipe/graphs/deformation/calculators:Tensor.h"
],
assets_dir = "",
manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml",

6
mediapipe/examples/desktop/beauty/BUILD
View File

@ -48,6 +48,12 @@ cc_binary(
],
)
cc_binary(
name = "beauty_gpu_over",
deps = [

34
mediapipe/examples/desktop/makeup/BUILD Normal file
View File

@ -0,0 +1,34 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"])
package(default_visibility = ["//mediapipe/examples:__subpackages__"])
cc_binary(
name = "makeup_cpu",
deps = [
"//mediapipe/examples/desktop:demo_run_graph_main",
"//mediapipe/graphs/makeup:desktop_live_calculators",
],
)
cc_binary(
name = "makeup_mobile",
deps = [
"//mediapipe/examples/desktop:demo_run_graph_main",
"//mediapipe/graphs/makeup:mobile_calculators",
],
)

5
mediapipe/graphs/beauty/README.md
View File

@ -19,10 +19,7 @@ bazel-bin/mediapipe/examples/desktop/beauty/beauty_cpu
```
Run with (using video):
```
bazel-bin/mediapipe/examples/desktop/beauty/beauty_cpu
--calculator_graph_config_file=mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt
--input_video_path=/path/video.mp4
--output_video_path=/path/outvideo.mp4
bazel-bin/mediapipe/examples/desktop/beauty/beauty_cpu --calculator_graph_config_file=mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt --input_video_path=/path/video.mp4 --output_video_path=/path/outvideo.mp4
```
2. Desktop-CPU-Single (Not divided, using render data)

6
mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt
View File

@ -37,6 +37,12 @@ node {
tag_index: "FINISHED"
back_edge: true
}
node_options: {
[type.googleapis.com/mediapipe.FlowLimiterCalculatorOptions]: {
max_in_flight: 3
max_in_queue: 3
}
}
output_stream: "throttled_input_video"
}

2
mediapipe/graphs/beauty/beauty_desktop_cpu_single.pbtxt
View File

@ -14,7 +14,7 @@ profiler_config {
trace_enabled: true
enable_profiler: true
trace_log_interval_count: 200
trace_log_path: "/Users/alena/Workdir/mediapipe/logs/beauty/"
trace_log_path: "/home/mslight/Work/clone/mediapipe/mediapipe/logs/beauty_single/"
}
node {

2
mediapipe/graphs/beauty/beauty_mobile_gpu.pbtxt
View File

@ -15,7 +15,7 @@ output_stream: "multi_face_landmarks"
profiler_config {
trace_enabled: true
enable_profiler: true
trace_log_interval_count: 200
trace_log_path: "/storage/emulated/0/Download/profiles/"
}
# Throttles the images flowing downstream for flow control. It passes through

15
mediapipe/graphs/beauty/subgraphs/BUILD
View File

@ -24,16 +24,22 @@ package(default_visibility = ["//visibility:public"])
cc_library(
name = "renderer_calculators",
deps = [
"//mediapipe/calculators/core:split_proto_list_calculator",
"//mediapipe/util:annotation_renderer",
"//mediapipe/calculators/util:annotation_overlay_calculator",
"//mediapipe/calculators/beauty:smooth_face_calculator1",
"//mediapipe/calculators/beauty:bilateral_filter_calculator",
"//mediapipe/calculators/beauty:merging_images_calculator",
"//mediapipe/calculators/beauty:smooth_face_calculator2",
"//mediapipe/calculators/beauty:draw_lipstick_calculator",
"//mediapipe/calculators/beauty:whiten_teeth_calculator",
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
"//mediapipe/calculators/beauty:imageframe_to_mat_calculator",
"//mediapipe/calculators/landmarks:landmarks_to_mask_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/image:color_convert_calculator",
"//mediapipe/calculators/core:round_robin_demux_calculator",
"//mediapipe/calculators/core:mux_calculator",
"//mediapipe/calculators/util:landmarks_to_render_data_calculator",
"//mediapipe/util:annotation_renderer",
"//mediapipe/calculators/util:annotation_overlay_calculator",
"//mediapipe/calculators/core:split_proto_list_calculator",
"//mediapipe/graphs/face_mesh/calculators:face_landmarks_to_render_data_calculator",
],
)
@ -66,3 +72,4 @@ mediapipe_simple_subgraph(
":renderer_calculators",
],
)

89
mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt
View File

@ -11,6 +11,12 @@ input_stream: "LANDMARKS:multi_face_landmarks"
# CPU image with rendered data. (ImageFrame)
output_stream: "IMAGE:output_image"
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:input_image"
output_stream: "SIZE:size"
}
# Outputs each element of multi_face_landmarks at a fake timestamp for the rest
# of the graph to process. At the end of the loop, outputs the BATCH_END
# timestamp for downstream calculators to inform them that all elements in the
@ -18,16 +24,16 @@ output_stream: "IMAGE:output_image"
node {
calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITERABLE:multi_face_landmarks"
input_stream: "CLONE:input_image"
input_stream: "CLONE:size"
output_stream: "ITEM:face_landmarks"
output_stream: "CLONE:loop_image"
output_stream: "CLONE:cloned_size"
output_stream: "BATCH_END:landmark_timestamp"
}
# Converts landmarks to face part masks.
node {
calculator: "LandmarksToMaskCalculator"
input_stream: "IMAGE:loop_image"
input_stream: "SIZE:cloned_size"
input_stream: "NORM_LANDMARKS:face_landmarks"
output_stream: "FACEBOX:face_box"
output_stream: "MASK:mask"
@ -50,47 +56,88 @@ node {
output_stream: "ITERABLE:multi_face_box"
}
#Applies lipstick to the face on the IMAGE using MASK.
node {
calculator: "ImageFrameToMatCalculator"
input_stream: "IMAGE:input_image"
output_stream: "MAT:input_mat"
}
node {
calculator: "DrawLipstickCalculator"
input_stream: "IMAGE:input_image"
input_stream: "MAT:input_mat"
input_stream: "MASK:0:multi_mask"
output_stream: "IMAGE:input_image_1"
output_stream: "MAT:lipstick_image"
output_stream: "MASK:lipstick_mask"
}
#Whitens teeth of the face on the IMAGE using MASK.
node {
calculator: "WhitenTeethCalculator"
input_stream: "IMAGE:input_image_1"
input_stream: "MAT:input_mat"
input_stream: "MASK:0:multi_mask"
output_stream: "IMAGE:input_image_2"
output_stream: "MAT:teeth_image"
output_stream: "MASK:teeth_mask"
}
#Smoothes face on the IMAGE using MASK.
node {
calculator: "SmoothFaceCalculator1"
input_stream: "IMAGE:input_image_2"
input_stream: "MAT:input_mat"
input_stream: "MASK:0:multi_mask"
input_stream: "FACEBOX:0:multi_face_box"
output_stream: "IMAGE:input_image_3"
output_stream: "MASK:not_full_face"
output_stream: "FACEBOX:box1"
input_stream: "FACE:0:multi_face_box"
output_stream: "MAT:tmp_image_1"
output_stream: "MASK:face_mask"
output_stream: "FACE:box_1"
}
node {
calculator: "RoundRobinDemuxCalculator"
input_stream: "box_1"
output_stream: "OUTPUT:0:box1_0"
output_stream: "OUTPUT:1:box1_1"
output_stream: "SELECT:select"
}
#Smoothes face on the IMAGE using MASK.
node {
calculator: "BilateralCalculator"
input_stream: "IMAGE:input_image_3"
input_stream: "FACEBOX:box1"
output_stream: "IMAGE2:input_image_4"
input_stream: "IMAGE:box1_0"
output_stream: "CVMAT:box14_0"
}
node {
calculator: "BilateralCalculator"
input_stream: "IMAGE:box1_1"
output_stream: "CVMAT:box14_1"
}
node {
calculator: "MuxCalculator"
input_stream: "INPUT:0:box14_0"
input_stream: "INPUT:1:box14_1"
input_stream: "SELECT:select"
output_stream: "OUTPUT:tmp_image_2"
input_stream_handler {
input_stream_handler: "MuxInputStreamHandler"
}
}
#Smoothes face on the IMAGE using MASK.
node {
calculator: "SmoothFaceCalculator2"
input_stream: "IMAGE:input_image_2"
input_stream: "IMAGE2:input_image_4"
input_stream: "MASK:not_full_face"
input_stream: "FACEBOX:box1"
input_stream: "MAT:input_mat"
input_stream: "IMAGE2:tmp_image_2"
input_stream: "MASK:face_mask"
input_stream: "FACEBOX:box_1"
output_stream: "MAT:face_image"
}
#Merges output images of DrawLipstick, WhitenTeeth and SmoothFace calculators
node {
calculator: "MergeImagesCalculator"
input_stream: "MAT:0:lipstick_image"
input_stream: "MASK:0:lipstick_mask"
input_stream: "MAT:1:teeth_image"
input_stream: "MASK:1:teeth_mask"
input_stream: "MAT:2:face_image"
input_stream: "MASK:2:face_mask"
output_stream: "IMAGE:output_image"
}

51
mediapipe/graphs/beauty/subgraphs/face_renderer_gpu.pbtxt
View File

@ -18,7 +18,7 @@ output_stream: "IMAGE:output_image"
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:input_image"
input_stream: "IMAGE:input_image"
output_stream: "SIZE:image_size"
}
# Outputs each element of multi_face_landmarks at a fake timestamp for the rest
@ -28,16 +28,16 @@ node {
node {
calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITERABLE:multi_face_landmarks"
input_stream: "IMAGE_GPU:input_image"
input_stream: "IMAGE:input_image"
output_stream: "ITEM:face_landmarks"
output_stream: "IMAGE_GPU:loop_image"
output_stream: "IMAGE:loop_image"
output_stream: "BATCH_END:landmark_timestamp"
}
# Converts landmarks to face part masks.
node {
calculator: "LandmarksToMaskCalculator"
input_stream: "IMAGE_GPU:loop_image"
input_stream: "IMAGE:loop_image"
input_stream: "NORM_LANDMARKS:face_landmarks"
output_stream: "FACEBOX:face_box"
output_stream: "MASK:mask"
@ -63,17 +63,17 @@ node {
#Applies lipstick to the face on the IMAGE using MASK.
node {
calculator: "DrawLipstickCalculator"
input_stream: "IMAGE_GPU:input_image"
input_stream: "IMAGE:input_image"
input_stream: "MASK:0:multi_mask"
output_stream: "IMAGE_GPU:input_image_1"
output_stream: "IMAGE:input_image_1"
}
#Whitens teeth of the face on the IMAGE using MASK.
node {
calculator: "WhitenTeethCalculator"
input_stream: "IMAGE_GPU:input_image_1"
input_stream: "IMAGE:input_image_1"
input_stream: "MASK:0:multi_mask"
output_stream: "IMAGE_GPU:input_image_2"
output_stream: "IMAGE:input_image_2"
}
#Smoothes face on the IMAGE using MASK.
@ -81,18 +81,40 @@ node {
calculator: "SmoothFaceCalculator1"
input_stream: "IMAGE:input_image_2"
input_stream: "MASK:0:multi_mask"
input_stream: "FACEBOX:0:multi_face_box"
input_stream: "FACE:0:multi_face_box"
output_stream: "IMAGE:input_image_3"
output_stream: "MASK:not_full_face"
output_stream: "FACEBOX:box1"
output_stream: "FACE:box1"
}
node {
calculator: "RoundRobinDemuxCalculator"
input_stream: "box1"
output_stream: "OUTPUT:0:box1_0"
output_stream: "OUTPUT:1:box1_1"
output_stream: "SELECT:select"
}
#Smoothes face on the IMAGE using MASK.
node {
calculator: "BilateralCalculator"
input_stream: "IMAGE:input_image_3"
input_stream: "FACEBOX:box1"
output_stream: "IMAGE2:input_image_4"
input_stream: "IMAGE:box1_0"
output_stream: "CVMAT:box14_0"
}
node {
calculator: "BilateralCalculator"
input_stream: "IMAGE:box1_1"
output_stream: "CVMAT:box14_1"
}
node {
calculator: "MuxCalculator"
input_stream: "INPUT:0:box14_0"
input_stream: "INPUT:1:box14_1"
input_stream: "SELECT:select"
output_stream: "OUTPUT:input_image_4"
input_stream_handler {
input_stream_handler: "MuxInputStreamHandler"
}
}
#Smoothes face on the IMAGE using MASK.
@ -104,3 +126,4 @@ node {
input_stream: "FACEBOX:box1"
output_stream: "IMAGE:output_image"
}

2
mediapipe/graphs/deformation/BUILD
View File

@ -48,7 +48,6 @@ cc_library(
"//mediapipe/calculators/core:constant_side_packet_calculator" ,
"//mediapipe/calculators/image:color_convert_calculator" ,
"//mediapipe/calculators/image:image_properties_calculator" ,
],
)
@ -58,4 +57,3 @@ mediapipe_binary_graph(
output_name = "mobile_gpu.binarypb" ,
deps = [":mobile_calculators"],
)

3
mediapipe/graphs/deformation/README.md
View File

@ -14,8 +14,7 @@ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/d
```
Run with (using your camera):
```
bazel-bin/mediapipe/examples/desktop/deformation/deformation_cpu
--calculator_graph_config_file=mediapipe/graphs/deformation/deformation_cpu.pbtxt
bazel-bin/mediapipe/examples/desktop/deformation/deformation_cpu --calculator_graph_config_file=mediapipe/graphs/deformation/deformation_cpu.pbtxt
```
Run with (using video):
```

27
mediapipe/graphs/deformation/calculators/face_processor_calculator.cc
View File

@ -137,9 +137,9 @@ namespace mediapipe
vector<string> index_names;
map<string, vector<int>> indexes;
map<string, Tensor<double>> masks;
vector<vector<int>> _trianglesIndexes;
map<string, Tensor<double>> masks;
Tensor<double> __facePts;
};
@ -181,14 +181,13 @@ namespace mediapipe
absl::Status FaceProcessorCalculator::Open(CalculatorContext *cc)
{
cc->SetOffset(TimestampDiff(0));
MP_RETURN_IF_ERROR(SetData(cc));
return absl::OkStatus();
}
absl::Status FaceProcessorCalculator::Process(CalculatorContext *cc)
{
MP_RETURN_IF_ERROR(SetData(cc));
if (cc->Inputs().HasTag(kNormLandmarksTag) &&
!cc->Inputs().Tag(kNormLandmarksTag).IsEmpty())
{
@ -206,11 +205,10 @@ namespace mediapipe
absl::Status FaceProcessorCalculator::SetData(CalculatorContext *cc)
{
masks = {};
_trianglesIndexes = {};
string line;
string filename = "mediapipe/graphs/deformation/config/triangles.txt";
string content_blob;
ASSIGN_OR_RETURN(content_blob,
ASSIGN_OR_RETURN(string content_blob,
ReadContentBlobFromFile(filename),
_ << "Failed to read texture blob from file!");
@ -219,7 +217,6 @@ namespace mediapipe
vector<int> tmp;
for (int i = 0; i < 854; ++i)
{
string line;
tmp = {};
for (int j = 0; j < 3; ++j)
{
@ -228,20 +225,20 @@ namespace mediapipe
}
_trianglesIndexes.push_back(tmp);
}
stream.clear();
filename = "./mediapipe/graphs/deformation/config/index_names.txt";
ASSIGN_OR_RETURN(content_blob,
ReadContentBlobFromFile(filename),
_ << "Failed to read texture blob from file!");
istringstream stream2(content_blob);
stream.str(content_blob);
string line;
vector<int> idxs;
while (getline(stream2, line))
while (getline(stream, line))
{
index_names.push_back(line);
}
stream2.clear();
stream.clear();
for (int i = 0; i < index_names.size(); i++)
{
@ -250,16 +247,16 @@ namespace mediapipe
ASSIGN_OR_RETURN(content_blob,
ReadContentBlobFromFile(filename),
_ << "Failed to read texture blob from file!");
stream2.str(content_blob);
stream.str(content_blob);
while (getline(stream2, line))
while (getline(stream, line))
{
idxs.push_back(stoi(line));
}
indexes[index_names[i]] = idxs;
idxs = {};
stream2.clear();
stream.clear();
}
double **zero_arr;

10
mediapipe/graphs/deformation/deformation_mobile.pbtxt
View File

@ -56,6 +56,16 @@ node {
output_stream: "SIZE:size"
}
node {
calculator: "LocalFileContentsCalculator"
input_side_packet: "FILE_PATH:0:file_path1"
input_side_packet: "FILE_PATH:1:file_path2"
input_side_packet: "FILE_PATH:1:file_path2"
output_side_packet: "CONTENTS:0:contents1"
output_side_packet: "CONTENTS:1:contents2"
output_side_packet: "CONTENTS:1:contents2"
}
node {
calculator: "FaceProcessorCalculator"
input_stream: "NORM_LANDMARKS:multi_face_landmarks"

4
mediapipe/graphs/image_style/README.md
View File

@ -18,9 +18,7 @@ bazel-bin/mediapipe/examples/desktop/image_style/image_style_cpu --calculator_gr
```
Run with (using video):
```
bazel-bin/mediapipe/examples/desktop/image_style/image_style_cpu --calculator_graph_config_file=mediapipe/graphs/image_style/image_style_cpu.pbtxt
--input_video_path=/path/video.mp4
--output_video_path=/path/outvideo.mp4
bazel-bin/mediapipe/examples/desktop/image_style/image_style_cpu --calculator_graph_config_file=mediapipe/graphs/image_style/image_style_cpu.pbtxt --input_video_path=/path/video.mp4 --output_video_path=/path/outvideo.mp4
```
2. Mobile (Android)

7
mediapipe/graphs/image_style/image_style_cpu.pbtxt
View File

@ -6,6 +6,13 @@ input_stream: "input_video"
# Output image with rendered results. (ImageFrame)
output_stream: "output_video"
profiler_config {
trace_enabled: true
enable_profiler: true
trace_log_interval_count: 200
trace_log_path: "/home/mslight/Work/clone/mediapipe/mediapipe/logs/imagestyle/"
}
node {
calculator: "FlowLimiterCalculator"
input_stream: "input_video"

63
mediapipe/graphs/makeup/BUILD Normal file
View File

@ -0,0 +1,63 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"mediapipe_binary_graph",
)
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
cc_library(
name = "desktop_live_calculators",
deps = [
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/graphs/makeup/subgraphs:face_renderer_cpu",
"//mediapipe/modules/face_landmark:face_landmark_front_cpu",
],
)
cc_library(
name = "desktop_live_gpu_calculators",
deps = [
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/graphs/makeup/subgraphs:face_renderer_gpu",
"//mediapipe/modules/face_landmark:face_landmark_front_gpu",
],
)
cc_library(
name = "mobile_calculators",
deps = [
"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/graphs/makeup/subgraphs:face_renderer_cpu",
"//mediapipe/graphs/makeup/subgraphs:face_renderer_gpu",
"//mediapipe/modules/face_landmark:face_landmark_front_gpu",
],
)
mediapipe_binary_graph(
name = "beauty_mobile_gpu_binary_graph",
graph = "beauty_mobile_gpu.pbtxt",
output_name = "beauty_mobile_gpu.binarypb",
deps = [":mobile_calculators"],
)

71
mediapipe/graphs/makeup/makeup_desktop_cpu.pbtxt Normal file
View File

@ -0,0 +1,71 @@
# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU.
# Input image. (ImageFrame)
input_stream: "input_video"
# Output image with rendered results. (ImageFrame)
output_stream: "output_video"
# Collection of detected/processed faces, each represented as a list of
# landmarks. (std::vector<NormalizedLandmarkList>)
output_stream: "multi_face_landmarks"
profiler_config {
trace_enabled: true
enable_profiler: true
trace_log_interval_count: 200
trace_log_path: "/home/mslight/Work/clone/mediapipe/mediapipe/logs/makeup/"
}
# Throttles the images flowing downstream for flow control. It passes through
# the very first incoming image unaltered, and waits for downstream nodes
# (calculators and subgraphs) in the graph to finish their tasks before it
# passes through another image. All images that come in while waiting are
# dropped, limiting the number of in-flight images in most part of the graph to
# 1. This prevents the downstream nodes from queuing up incoming images and data
# excessively, which leads to increased latency and memory usage, unwanted in
# real-time mobile applications. It also eliminates unnecessarily computation,
# e.g., the output produced by a node may get dropped downstream if the
# subsequent nodes are still busy processing previous inputs.
node {
calculator: "FlowLimiterCalculator"
input_stream: "input_video"
input_stream: "FINISHED:output_video"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_input_video"
}
# Defines side packets for further use in the graph.
node {
calculator: "ConstantSidePacketCalculator"
output_side_packet: "PACKET:0:num_faces"
output_side_packet: "PACKET:1:with_attention"
node_options: {
[type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
packet { int_value: 1 }
packet { bool_value: true }
}
}
}
# Subgraph that detects faces and corresponding landmarks.
node {
calculator: "FaceLandmarkFrontCpu"
input_stream: "IMAGE:throttled_input_video"
input_side_packet: "NUM_FACES:num_faces"
input_side_packet: "WITH_ATTENTION:with_attention"
output_stream: "LANDMARKS:multi_face_landmarks"
}
# Subgraph that renders face-landmark annotation onto the input image.
node {
calculator: "FaceRendererCpu"
input_stream: "IMAGE:throttled_input_video"
input_stream: "LANDMARKS:multi_face_landmarks"
output_stream: "IMAGE:output_video"
}

84
mediapipe/graphs/makeup/makeup_mobile_gpu.pbtxt Normal file
View File

@ -0,0 +1,84 @@
# MediaPipe graph that performs face mesh with TensorFlow Lite on GPU.
# GPU buffer. (GpuBuffer)
input_stream: "input_video"
# Max number of faces to detect/process. (int)
input_side_packet: "num_faces"
# Output image with rendered results. (GpuBuffer)
output_stream: "output_video"
# Collection of detected/processed faces, each represented as a list of
# landmarks. (std::vector<NormalizedLandmarkList>)
output_stream: "multi_face_landmarks"
profiler_config {
trace_enabled: true
enable_profiler: true
trace_log_interval_count: 200
}
# Throttles the images flowing downstream for flow control. It passes through
# the very first incoming image unaltered, and waits for downstream nodes
# (calculators and subgraphs) in the graph to finish their tasks before it
# passes through another image. All images that come in while waiting are
# dropped, limiting the number of in-flight images in most part of the graph to
# 1. This prevents the downstream nodes from queuing up incoming images and data
# excessively, which leads to increased latency and memory usage, unwanted in
# real-time mobile applications. It also eliminates unnecessarily computation,
# e.g., the output produced by a node may get dropped downstream if the
# subsequent nodes are still busy processing previous inputs.
node {
calculator: "FlowLimiterCalculator"
input_stream: "input_video"
input_stream: "FINISHED:output_video"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_input_video"
}
# Defines side packets for further use in the graph.
node {
calculator: "ConstantSidePacketCalculator"
output_side_packet: "PACKET:with_attention"
node_options: {
[type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
packet { bool_value: true }
}
}
}
# Defines side packets for further use in the graph.
node {
calculator: "GpuBufferToImageFrameCalculator"
input_stream: "throttled_input_video"
output_stream: "throttled_input_video_cpu"
}
# Subgraph that detects faces and corresponding landmarks.
node {
calculator: "FaceLandmarkFrontGpu"
input_stream: "IMAGE:throttled_input_video"
input_side_packet: "NUM_FACES:num_faces"
input_side_packet: "WITH_ATTENTION:with_attention"
output_stream: "LANDMARKS:multi_face_landmarks"
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
output_stream: "DETECTIONS:face_detections"
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
}
# Subgraph that renders face-landmark annotation onto the input image.
node {
calculator: "FaceRendererCpu"
input_stream: "IMAGE:throttled_input_video_cpu"
input_stream: "LANDMARKS:multi_face_landmarks"
output_stream: "IMAGE:output_video_cpu"
}
node {
calculator: "ImageFrameToGpuBufferCalculator"
input_stream: "output_video_cpu"
output_stream: "output_video"
}

60
mediapipe/graphs/makeup/subgraphs/BUILD Normal file
View File

@ -0,0 +1,60 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"mediapipe_simple_subgraph",
)
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
cc_library(
name = "renderer_calculators",
deps = [
"//mediapipe/calculators/beauty:smooth_face_calculator1",
"//mediapipe/calculators/beauty:bilateral_filter_calculator",
"//mediapipe/calculators/beauty:smooth_face_calculator2",
"//mediapipe/calculators/beauty:draw_lipstick_calculator",
"//mediapipe/calculators/beauty:whiten_teeth_calculator",
"//mediapipe/calculators/beauty:merging_images_calculator",
"//mediapipe/calculators/beauty:imageframe_to_mat_calculator",
"//mediapipe/calculators/landmarks:landmarks_to_point_array_calculator",
"//mediapipe/calculators/landmarks:points_to_face_box_calculator",
"//mediapipe/calculators/landmarks:point_vector_to_mask_calculator",
"//mediapipe/calculators/core:round_robin_demux_calculator",
"//mediapipe/calculators/core:mux_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_renderer_gpu",
graph = "face_renderer_gpu.pbtxt",
register_as = "FaceRendererGpu",
deps = [
":renderer_calculators",
],
)
mediapipe_simple_subgraph(
name = "face_renderer_cpu",
graph = "face_renderer_cpu.pbtxt",
register_as = "FaceRendererCpu",
deps = [
":renderer_calculators",
],
)

152
mediapipe/graphs/makeup/subgraphs/face_renderer_cpu.pbtxt Normal file
View File

@ -0,0 +1,152 @@
# MediaPipe face mesh rendering subgraph.
type: "FaceRendererCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:input_image"
# Collection of detected/predicted faces, each represented as a list of
# landmarks. (std::vector<NormalizedLandmarkList>)
input_stream: "LANDMARKS:multi_face_landmarks"
# CPU image with rendered data. (ImageFrame)
output_stream: "IMAGE:output_image"
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:input_image"
output_stream: "SIZE:size"
}
node {
calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITERABLE:multi_face_landmarks"
input_stream: "CLONE:size"
output_stream: "ITEM:face_landmarks"
output_stream: "CLONE:image_size"
output_stream: "BATCH_END:landmark_timestamp"
}
# Converts landmarks to face part masks.
node {
calculator: "LandmarksToPointArrayCalculator"
input_stream: "IMAGE_SIZE:image_size"
input_stream: "NORM_LANDMARKS:face_landmarks"
output_stream: "POINTS:points"
}
# Converts landmarks to face part masks.
node {
calculator: "PointVectorToMaskCalculator"
input_stream: "POINTS:points"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "MASKS:mask"
}
# Converts landmarks to face part masks.
node {
calculator: "PointsToFaceBoxCalculator"
input_stream: "POINTS:points"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "FACE_BOX:face_box"
}
node {
calculator: "EndLoopMapMaskCalculator"
input_stream: "ITEM:mask"
input_stream: "BATCH_END:landmark_timestamp"
output_stream: "ITERABLE:multi_mask"
}
node {
calculator: "EndLoopFaceBoxCalculator"
input_stream: "ITEM:face_box"
input_stream: "BATCH_END:landmark_timestamp"
output_stream: "ITERABLE:multi_face_box"
}
node {
calculator: "ImageFrameToMatCalculator"
input_stream: "IMAGE:input_image"
output_stream: "MAT:input_mat"
}
#Applies lipstick to the face on the IMAGE using MASK.
node {
calculator: "DrawLipstickCalculator"
input_stream: "MAT:input_mat"
input_stream: "MASK:0:multi_mask"
output_stream: "MAT:lipstick_image"
output_stream: "MASK:lipstick_mask"
}
#Whitens teeth of the face on the IMAGE using MASK.
node {
calculator: "WhitenTeethCalculator"
input_stream: "MAT:input_mat"
input_stream: "MASK:0:multi_mask"
output_stream: "MAT:teeth_image"
output_stream: "MASK:teeth_mask"
}
#Smoothes face on the IMAGE using MASK.
node {
calculator: "SmoothFaceCalculator1"
input_stream: "MAT:input_mat"
input_stream: "MASK:0:multi_mask"
input_stream: "FACE:0:multi_face_box"
output_stream: "MAT:tmp_image_1"
output_stream: "MASK:face_mask"
output_stream: "FACE:box_1"
}
node {
calculator: "RoundRobinDemuxCalculator"
input_stream: "box_1"
output_stream: "OUTPUT:0:box1_0"
output_stream: "OUTPUT:1:box1_1"
output_stream: "SELECT:select"
}
node {
calculator: "BilateralCalculator"
input_stream: "IMAGE:box1_0"
output_stream: "CVMAT:box14_0"
}
node {
calculator: "BilateralCalculator"
input_stream: "IMAGE:box1_1"
output_stream: "CVMAT:box14_1"
}
node {
calculator: "MuxCalculator"
input_stream: "INPUT:0:box14_0"
input_stream: "INPUT:1:box14_1"
input_stream: "SELECT:select"
output_stream: "OUTPUT:tmp_image_2"
input_stream_handler {
input_stream_handler: "MuxInputStreamHandler"
}
}
#Smoothes face on the IMAGE using MASK.
node {
calculator: "SmoothFaceCalculator2"
input_stream: "MAT:input_mat"
input_stream: "IMAGE2:tmp_image_2"
input_stream: "MASK:face_mask"
input_stream: "FACEBOX:box_1"
output_stream: "MAT:face_image"
}
#Merges output images of DrawLipstick, WhitenTeeth and SmoothFace calculators
node {
calculator: "MergeImagesCalculator"
input_stream: "MAT:0:lipstick_image"
input_stream: "MASK:0:lipstick_mask"
input_stream: "MAT:1:teeth_image"
input_stream: "MASK:1:teeth_mask"
input_stream: "MAT:2:face_image"
input_stream: "MASK:2:face_mask"
output_stream: "IMAGE:output_image"
}

106
mediapipe/graphs/makeup/subgraphs/face_renderer_gpu.pbtxt Normal file
View File

@ -0,0 +1,106 @@
# MediaPipe face mesh rendering subgraph.
type: "FaceRendererGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:input_image"
# Collection of detected/predicted faces, each represented as a list of
# landmarks. (std::vector<NormalizedLandmarkList>)
input_stream: "LANDMARKS:multi_face_landmarks"
# Regions of interest calculated based on palm detections.
# (std::vector<NormalizedRect>)
input_stream: "NORM_RECTS:rects"
# Detected palms. (std::vector<Detection>)
input_stream: "DETECTIONS:detections"
# GPU image with rendered data. (GpuBuffer)
output_stream: "IMAGE:output_image"
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:input_image"
output_stream: "SIZE:image_size"
}
# Outputs each element of multi_face_landmarks at a fake timestamp for the rest
# of the graph to process. At the end of the loop, outputs the BATCH_END
# timestamp for downstream calculators to inform them that all elements in the
# vector have been processed.
node {
calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITERABLE:multi_face_landmarks"
input_stream: "IMAGE_GPU:input_image"
output_stream: "ITEM:face_landmarks"
output_stream: "IMAGE_GPU:loop_image"
output_stream: "BATCH_END:landmark_timestamp"
}
# Converts landmarks to face part masks.
node {
calculator: "LandmarksToMaskCalculator"
input_stream: "IMAGE_GPU:loop_image"
input_stream: "NORM_LANDMARKS:face_landmarks"
output_stream: "FACEBOX:face_box"
output_stream: "MASK:mask"
}
# Collects a MapMask object for each hand into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END
# timestamp.
node {
calculator: "EndLoopMapMaskCalculator"
input_stream: "ITEM:mask"
input_stream: "BATCH_END:landmark_timestamp"
output_stream: "ITERABLE:multi_mask"
}
node {
calculator: "EndLoopFaceBoxCalculator"
input_stream: "ITEM:face_box"
input_stream: "BATCH_END:landmark_timestamp"
output_stream: "ITERABLE:multi_face_box"
}
#Applies lipstick to the face on the IMAGE using MASK.
node {
calculator: "DrawLipstickCalculator"
input_stream: "IMAGE_GPU:input_image"
input_stream: "MASK:0:multi_mask"
output_stream: "IMAGE_GPU:input_image_1"
}
#Whitens teeth of the face on the IMAGE using MASK.
node {
calculator: "WhitenTeethCalculator"
input_stream: "IMAGE_GPU:input_image_1"
input_stream: "MASK:0:multi_mask"
output_stream: "IMAGE_GPU:input_image_2"
}
#Smoothes face on the IMAGE using MASK.
node {
calculator: "SmoothFaceCalculator1"
input_stream: "IMAGE:input_image_2"
input_stream: "MASK:0:multi_mask"
input_stream: "FACEBOX:0:multi_face_box"
output_stream: "IMAGE:input_image_3"
output_stream: "MASK:not_full_face"
output_stream: "FACEBOX:box1"
}
#Smoothes face on the IMAGE using MASK.
node {
calculator: "BilateralCalculator"
input_stream: "IMAGE:input_image_3"
input_stream: "FACEBOX:box1"
output_stream: "IMAGE2:input_image_4"
}
#Smoothes face on the IMAGE using MASK.
node {
calculator: "SmoothFaceCalculator2"
input_stream: "IMAGE:input_image_2"
input_stream: "IMAGE2:input_image_4"
input_stream: "MASK:not_full_face"
input_stream: "FACEBOX:box1"
output_stream: "IMAGE:output_image"
}