diff --git a/mediapipe/graphs/deformation/calculators/Tensor.h b/mediapipe/graphs/deformation/calculators/Tensor.h index 737200d14..be5da18e3 100644 --- a/mediapipe/graphs/deformation/calculators/Tensor.h +++ b/mediapipe/graphs/deformation/calculators/Tensor.h @@ -64,10 +64,6 @@ public: return vector(M.ptr(0), M.ptr(0) + dims[1]); } - /* vector get_dims() { - return dims; - } */ - T at(vector _indexes) { return M.at(_indexes.data()); } diff --git a/mediapipe/graphs/deformation/calculators/face_processor_calculator.cc b/mediapipe/graphs/deformation/calculators/face_processor_calculator.cc index 3321933c6..10f60a971 100644 --- a/mediapipe/graphs/deformation/calculators/face_processor_calculator.cc +++ b/mediapipe/graphs/deformation/calculators/face_processor_calculator.cc @@ -283,9 +283,9 @@ namespace mediapipe absl::Status FaceProcessorCalculator::ProcessImage(CalculatorContext *cc) { - double alfaNose = 2.7; - double alfaLips = 0.7; - double alfaCheekbones = 0.7; + double alfaNose = 1.2; + double alfaLips = 0.4; + double alfaCheekbones = 0.4; if (cc->Inputs().HasTag(kNormLandmarksTag)) { @@ -381,7 +381,7 @@ namespace mediapipe Tensor _src = __facePts.index(_trianglesIndexes).index(_order); Tensor _dst = ___facePts.index(_trianglesIndexes).index(_order); - // cout << _src.get_dims().size() << endl; + auto srcPtr = absl::make_unique>(_src); cc->Outputs().Tag(kSrcTensorTag).Add(srcPtr.release(), cc->InputTimestamp()); diff --git a/mediapipe/graphs/deformation/calculators/warp_affine_calculator.cc b/mediapipe/graphs/deformation/calculators/warp_affine_calculator.cc index 2762c3461..ca25d5a22 100644 --- a/mediapipe/graphs/deformation/calculators/warp_affine_calculator.cc +++ b/mediapipe/graphs/deformation/calculators/warp_affine_calculator.cc @@ -14,6 +14,7 @@ #include "Tensor.h" #include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/api2/node.h" #include "mediapipe/framework/formats/image_format.pb.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" @@ -23,263 +24,122 @@ #include "mediapipe/framework/port/opencv_highgui_inc.h" #include "mediapipe/framework/port/status.h" -using namespace cv; -using namespace std; - namespace mediapipe { - namespace + namespace api2 { - constexpr char kImageFrameTag[] = "IMAGE"; - constexpr char kSrcTag[] = "SRC_TENSOR"; - constexpr char kDstTag[] = "DST_TENSOR"; - - inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } - } // namespace - - class WarpAffineCalculator : public CalculatorBase - { - public: - WarpAffineCalculator() = default; - ~WarpAffineCalculator() override = default; - - static absl::Status GetContract(CalculatorContract *cc); - - absl::Status Open(CalculatorContext *cc) override; - absl::Status Process(CalculatorContext *cc) override; - absl::Status Close(CalculatorContext *cc) override; - - private: - absl::Status CreateRenderTargetCpu(CalculatorContext *cc, - std::unique_ptr &image_mat, - ImageFormat::Format *target_format); - - absl::Status RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image, std::unique_ptr &image_mat); - - absl::Status AffineTransform(CalculatorContext *cc, std::unique_ptr &image_mat, Tensor _src, Tensor _dst); - - bool image_frame_available_ = false; - std::unique_ptr image_mat; - }; - - absl::Status WarpAffineCalculator::GetContract(CalculatorContract *cc) - { - RET_CHECK(cc->Inputs().HasTag(kImageFrameTag)); - - if (cc->Inputs().HasTag(kImageFrameTag)) + class WarpAffineCalculator : public Node { - cc->Inputs().Tag(kImageFrameTag).Set(); - } - if (cc->Inputs().HasTag(kSrcTag)) - { - cc->Inputs().Tag(kSrcTag).Set>(); - } - if (cc->Inputs().HasTag(kDstTag)) - { - cc->Inputs().Tag(kDstTag).Set>(); - } - if (cc->Outputs().HasTag(kImageFrameTag)) - { - cc->Outputs().Tag(kImageFrameTag).Set(); - } + public: + static constexpr Input kImageFrame{"IMAGE"}; + static constexpr Input>::Optional kSrc{"SRC_TENSOR"}; + static constexpr Input>::Optional kDst{"DST_TENSOR"}; + static constexpr Output kOut{"IMAGE"}; - return absl::OkStatus(); - } + MEDIAPIPE_NODE_CONTRACT(kImageFrame, kSrc, kDst, kOut); - absl::Status WarpAffineCalculator::Open(CalculatorContext *cc) - { - cc->SetOffset(TimestampDiff(0)); - - if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc)) - { - image_frame_available_ = true; - } - - return absl::OkStatus(); - } - - absl::Status WarpAffineCalculator::Process(CalculatorContext *cc) - { - if (cc->Inputs().Tag(kImageFrameTag).IsEmpty()) - { - return absl::OkStatus(); - } - - ImageFormat::Format target_format; - - MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); - - if (!cc->Inputs().Tag(kSrcTag).IsEmpty() && !cc->Inputs().Tag(kDstTag).IsEmpty()) - { - const Tensor _src = cc->Inputs().Tag(kSrcTag).Get>(); - const Tensor _dst = cc->Inputs().Tag(kDstTag).Get>(); - MP_RETURN_IF_ERROR(AffineTransform(cc, image_mat, _src, _dst)); - } - - // Copy the rendered image to output. - uchar *image_mat_ptr = image_mat->data; - MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat)); - - return absl::OkStatus(); - } - - absl::Status WarpAffineCalculator::Close(CalculatorContext *cc) - { - return absl::OkStatus(); - } - - absl::Status WarpAffineCalculator::RenderToCpu( - CalculatorContext *cc, const ImageFormat::Format &target_format, - uchar *data_image, std::unique_ptr &image_mat) - { - auto output_frame = absl::make_unique( - target_format, image_mat->cols, image_mat->rows); - - output_frame->CopyPixelData(target_format, image_mat->cols, image_mat->rows, data_image, - ImageFrame::kDefaultAlignmentBoundary); - - if (cc->Outputs().HasTag(kImageFrameTag)) - { - cc->Outputs() - .Tag(kImageFrameTag) - .Add(output_frame.release(), cc->InputTimestamp()); - } - - return absl::OkStatus(); - } - - absl::Status WarpAffineCalculator::CreateRenderTargetCpu( - CalculatorContext *cc, std::unique_ptr &image_mat, - ImageFormat::Format *target_format) - { - if (image_frame_available_) - { - const auto &input_frame = - cc->Inputs().Tag(kImageFrameTag).Get(); - - int target_mat_type; - switch (input_frame.Format()) + static absl::Status UpdateContract(CalculatorContract *cc) { - case ImageFormat::SRGBA: - *target_format = ImageFormat::SRGBA; - target_mat_type = CV_8UC4; - break; - case ImageFormat::SRGB: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - case ImageFormat::SBGR: - *target_format = ImageFormat::SBGR; - target_mat_type = CV_8UC3; - break; - case ImageFormat::GRAY8: - *target_format = ImageFormat::SRGB; - target_mat_type = CV_8UC3; - break; - default: - return absl::UnknownError("Unexpected image frame format."); - break; + RET_CHECK(kOut(cc).IsConnected()) + << "At least one output stream is expected."; + return absl::OkStatus(); } - image_mat = absl::make_unique( - input_frame.Height(), input_frame.Width(), target_mat_type); - - auto input_mat = formats::MatView(&input_frame); - - if (input_frame.Format() == ImageFormat::GRAY8) + absl::Status Process(CalculatorContext *cc) override { - cv::Mat rgb_mat; - cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB); - rgb_mat.copyTo(*image_mat); - } - else - { - input_mat.copyTo(*image_mat); - } - } - else - { - image_mat = absl::make_unique( - 1920, 1080, CV_8UC4, - cv::Scalar(cv::Scalar::all(255))); - *target_format = ImageFormat::SRGBA; - } - - return absl::OkStatus(); - } - - absl::Status WarpAffineCalculator::AffineTransform(CalculatorContext *cc, std::unique_ptr &image_mat, Tensor _src, Tensor _dst) - { - Mat mat_image_ = *image_mat.get(); - Mat clone_image = mat_image_.clone(); - - Mat outImage = Mat(mat_image_.size(), mat_image_.type()); - Mat out = mat_image_.clone(); - - for (int i = 0; i < 854; ++i) - { - if (i == 246) - { - int pointer = 0; - } - Tensor __t1 = _src.index(vector{i}); - Tensor __t2 = _dst.index(vector{i}); - - vector t1; - vector t2; - - for (int i = 0; i < 3; ++i) - { - t1.push_back(Point( - (int)(__t1.at(vector{0, 3 * i})), - (int)(__t1.at(vector{0, 3 * i + 1})))); - t2.push_back(Point( - (int)(__t2.at(vector{0, 3 * i})), - (int)(__t2.at(vector{0, 3 * i + 1})))); - } - - cv::Rect r1 = cv::boundingRect(t1); - cv::Rect r2 = cv::boundingRect(t2); - cv::Point2f srcTri[3]; - cv::Point2f dstTri[3]; - std::vector t1Rect; - std::vector t2Rect; - - for (int i = 0; i < 3; ++i) - { - srcTri[i] = Point2f(t1[i].x - r1.x, t1[i].y - r1.y); - dstTri[i] = Point2f(t2[i].x - r2.x, t2[i].y - r2.y); - t1Rect.push_back(Point(t1[i].x - r1.x, t1[i].y - r1.y)); - t2Rect.push_back(Point(t2[i].x - r2.x, t2[i].y - r2.y)); - } - - Mat _dst; - Mat mask = Mat::zeros(r2.height, r2.width, CV_8U); - cv::fillConvexPoly(mask, t2Rect, Scalar(1.0, 1.0, 1.0), 16, 0); - - if (r1.x + r1.width < clone_image.cols && r1.x >= 0 && r1.x + r1.width >= 0 && r1.y >= 0 && r1.y < clone_image.rows && r1.y + r1.height < clone_image.rows) - { - Mat imgRect = mat_image_(Range(r1.y, r1.y + r1.height), Range(r1.x, r1.x + r1.width)); - Mat warpMat = getAffineTransform(srcTri, dstTri); - warpAffine(imgRect, _dst, warpMat, mask.size()); - - for (int i = r2.y; i < r2.y + r2.height; ++i) + if (kImageFrame(cc).IsEmpty()) { - for (int j = r2.x; j < r2.x + r2.width; ++j) + return absl::OkStatus(); + } + + const ImageFrame &input = *kImageFrame(cc); + auto image = absl::make_unique( + input.Height(), input.Width(), CV_8UC3); + + auto mat_image_ = formats::MatView(&input); + Mat out = mat_image_.clone(); + + if (!kSrc(cc).IsEmpty() && !kDst(cc).IsEmpty()) + { + Tensor _src = (*kSrc(cc)); + Tensor _dst = (*kDst(cc)); + + Mat clone_image = mat_image_.clone(); + + for (int i = 0; i < 854; ++i) { - if ((int)mask.at(i - r2.y, j - r2.x) > 0) + if (i == 246) { - out.at(i, j) = _dst.at(i - r2.y, j - r2.x); + int pointer = 0; + } + Tensor __t1 = _src.index(vector{i}); + Tensor __t2 = _dst.index(vector{i}); + + vector t1; + vector t2; + + for (int i = 0; i < 3; ++i) + { + t1.push_back(Point( + (int)(__t1.at(vector{0, 3 * i})), + (int)(__t1.at(vector{0, 3 * i + 1})))); + t2.push_back(Point( + (int)(__t2.at(vector{0, 3 * i})), + (int)(__t2.at(vector{0, 3 * i + 1})))); + } + + cv::Rect r1 = cv::boundingRect(t1); + cv::Rect r2 = cv::boundingRect(t2); + cv::Point2f srcTri[3]; + cv::Point2f dstTri[3]; + std::vector t1Rect; + std::vector t2Rect; + + for (int i = 0; i < 3; ++i) + { + srcTri[i] = Point2f(t1[i].x - r1.x, t1[i].y - r1.y); + dstTri[i] = Point2f(t2[i].x - r2.x, t2[i].y - r2.y); + t1Rect.push_back(Point(t1[i].x - r1.x, t1[i].y - r1.y)); + t2Rect.push_back(Point(t2[i].x - r2.x, t2[i].y - r2.y)); + } + + Mat _dst; + Mat mask = Mat::zeros(r2.height, r2.width, CV_8U); + cv::fillConvexPoly(mask, t2Rect, Scalar(1.0, 1.0, 1.0), 16, 0); + + if (r1.x + r1.width < clone_image.cols && r1.x >= 0 && r1.x + r1.width >= 0 && r1.y >= 0 && r1.y < clone_image.rows && r1.y + r1.height < clone_image.rows) + { + Mat imgRect = mat_image_(Range(r1.y, r1.y + r1.height), Range(r1.x, r1.x + r1.width)); + Mat warpMat = getAffineTransform(srcTri, dstTri); + warpAffine(imgRect, _dst, warpMat, mask.size()); + + for (int i = r2.y; i < r2.y + r2.height; ++i) + { + for (int j = r2.x; j < r2.x + r2.width; ++j) + { + if ((int)mask.at(i - r2.y, j - r2.x) > 0) + { + out.at(i, j) = _dst.at(i - r2.y, j - r2.x); + } + } + } } } } - } - } - out.copyTo(*image_mat); - return absl::OkStatus(); + out.copyTo(*image); + + auto output_frame = absl::make_unique( + input.Format(), mat_image_.cols, mat_image_.rows); + uchar *image_mat_ptr = image->data; + output_frame->CopyPixelData(input.Format(), mat_image_.cols, mat_image_.rows, image_mat_ptr, + ImageFrame::kDefaultAlignmentBoundary); + + kOut(cc).Send(std::move(output_frame)); + return absl::OkStatus(); + } + }; + + MEDIAPIPE_REGISTER_NODE(WarpAffineCalculator); } - REGISTER_CALCULATOR(WarpAffineCalculator); } // namespace mediapipe \ No newline at end of file