diff --git a/WORKSPACE b/WORKSPACE index 1e7cf07a4..45a1759ac 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -225,8 +225,8 @@ http_archive( build_file = "@//third_party:opencv_android.BUILD", strip_prefix = "OpenCV-android-sdk", type = "zip", - url = "https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip", -) + url = "https://github.com/opencv/opencv/releases/download/4.0.1/opencv-4.0.1-android-sdk.zip" + ) # After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will # trigger duplicate symbol errors in the linking stage of building a mediapipe ios app. diff --git a/mediapipe/calculators/beauty/draw_lipstick_calculator.cc b/mediapipe/calculators/beauty/draw_lipstick_calculator.cc index 4dcabb00d..f19816e63 100644 --- a/mediapipe/calculators/beauty/draw_lipstick_calculator.cc +++ b/mediapipe/calculators/beauty/draw_lipstick_calculator.cc @@ -42,13 +42,6 @@ namespace mediapipe constexpr char kMaskTag[] = "MASK"; constexpr char kImageFrameTag[] = "IMAGE"; - enum - { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - NUM_ATTRIBUTES - }; - inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } } // namespace @@ -148,11 +141,6 @@ namespace mediapipe { return absl::OkStatus(); } - if (cc->Inputs().HasTag(kMaskTag) && - cc->Inputs().Tag(kMaskTag).IsEmpty()) - { - return absl::OkStatus(); - } // Initialize render target, drawn with OpenCV. std::unique_ptr image_mat; @@ -163,14 +151,17 @@ namespace mediapipe MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); } - const std::vector> &mask_vec = - cc->Inputs().Tag(kMaskTag).Get>>(); - if (mask_vec.size() > 0) + if (cc->Inputs().HasTag(kMaskTag) && + !cc->Inputs().Tag(kMaskTag).IsEmpty()) { - for (auto mask : mask_vec) - MP_RETURN_IF_ERROR(DrawLipstick(cc, image_mat, &target_format, mask)); + const std::vector> &mask_vec = + cc->Inputs().Tag(kMaskTag).Get>>(); + if (mask_vec.size() > 0) + { + for (auto mask : mask_vec) + MP_RETURN_IF_ERROR(DrawLipstick(cc, image_mat, &target_format, mask)); + } } - // Copy the rendered image to output. uchar *image_mat_ptr = image_mat->data; MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat)); @@ -254,8 +245,7 @@ namespace mediapipe { image_mat = absl::make_unique( 150, 150, CV_8UC4, - cv::Scalar(255, 255, - 255)); + cv::Scalar(cv::Scalar::all(255))); *target_format = ImageFormat::SRGBA; } @@ -274,8 +264,6 @@ namespace mediapipe upper_lips_mask = cv::Mat::zeros(mat_image__.size(), CV_32F); lower_lips_mask = cv::Mat::zeros(mat_image__.size(), CV_32F); - //__android_log_print(ANDROID_LOG_ERROR, "OVERSEAS", "%d ", mask_vec[1].size().height); - upper_lips_mask = mask_vec.find("UPPER_LIP")->second; lower_lips_mask = mask_vec.find("LOWER_LIP")->second; diff --git a/mediapipe/calculators/beauty/form_face_mask_calculator.cc b/mediapipe/calculators/beauty/form_face_mask_calculator.cc index 4650df5ab..2ff410a99 100644 --- a/mediapipe/calculators/beauty/form_face_mask_calculator.cc +++ b/mediapipe/calculators/beauty/form_face_mask_calculator.cc @@ -18,7 +18,6 @@ #include #include #include -//#include #include diff --git a/mediapipe/calculators/beauty/smooth_face_calculator.cc b/mediapipe/calculators/beauty/smooth_face_calculator.cc index cdf44f1bb..b2e4c63b9 100644 --- a/mediapipe/calculators/beauty/smooth_face_calculator.cc +++ b/mediapipe/calculators/beauty/smooth_face_calculator.cc @@ -164,16 +164,6 @@ namespace mediapipe { return absl::OkStatus(); } - if (cc->Inputs().HasTag(kMaskTag) && - cc->Inputs().Tag(kMaskTag).IsEmpty()) - { - return absl::OkStatus(); - } - if (cc->Inputs().HasTag(kFaceBoxTag) && - cc->Inputs().Tag(kFaceBoxTag).IsEmpty()) - { - return absl::OkStatus(); - } // Initialize render target, drawn with OpenCV. ImageFormat::Format target_format; @@ -187,18 +177,23 @@ namespace mediapipe image_width_ = image_mat->cols; image_height_ = image_mat->rows; - const std::vector> &mask_vec = - cc->Inputs().Tag(kMaskTag).Get>>(); - - const std::vector> &face_box = - cc->Inputs().Tag(kFaceBoxTag).Get>>(); - - if (mask_vec.size() > 0 && face_box.size() > 0) + if (cc->Inputs().HasTag(kMaskTag) && + !cc->Inputs().Tag(kMaskTag).IsEmpty() && + cc->Inputs().HasTag(kFaceBoxTag) && + !cc->Inputs().Tag(kFaceBoxTag).IsEmpty()) { - for (int i = 0; i < mask_vec.size(); i++) - MP_RETURN_IF_ERROR(SmoothFace(cc, &target_format, mask_vec[i], face_box[i])); - } + const std::vector> &mask_vec = + cc->Inputs().Tag(kMaskTag).Get>>(); + const std::vector> &face_box = + cc->Inputs().Tag(kFaceBoxTag).Get>>(); + + if (mask_vec.size() > 0 && face_box.size() > 0) + { + for (int i = 0; i < mask_vec.size(); i++) + MP_RETURN_IF_ERROR(SmoothFace(cc, &target_format, mask_vec[i], face_box[i])); + } + } // Copy the rendered image to output. uchar *image_mat_ptr = image_mat->data; MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr)); diff --git a/mediapipe/calculators/beauty/whiten_teeth_calculator.cc b/mediapipe/calculators/beauty/whiten_teeth_calculator.cc index 31c67eb69..a96666d9d 100644 --- a/mediapipe/calculators/beauty/whiten_teeth_calculator.cc +++ b/mediapipe/calculators/beauty/whiten_teeth_calculator.cc @@ -16,7 +16,6 @@ #include #include -//#include #include @@ -42,13 +41,6 @@ namespace mediapipe constexpr char kMaskTag[] = "MASK"; constexpr char kImageFrameTag[] = "IMAGE"; - enum - { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - NUM_ATTRIBUTES - }; - inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } } // namespace @@ -153,11 +145,6 @@ namespace mediapipe { return absl::OkStatus(); } - if (cc->Inputs().HasTag(kMaskTag) && - cc->Inputs().Tag(kMaskTag).IsEmpty()) - { - return absl::OkStatus(); - } // Initialize render target, drawn with OpenCV. ImageFormat::Format target_format; @@ -169,15 +156,18 @@ namespace mediapipe mat_image_ = *image_mat.get(); image_width_ = image_mat->cols; image_height_ = image_mat->rows; - - const std::vector> &mask_vec = - cc->Inputs().Tag(kMaskTag).Get>>(); - if (mask_vec.size() > 0) - { - for (auto mask : mask_vec) - MP_RETURN_IF_ERROR(WhitenTeeth(cc, &target_format, mask)); - } + if (cc->Inputs().HasTag(kMaskTag) && + !cc->Inputs().Tag(kMaskTag).IsEmpty()) + { + const std::vector> &mask_vec = + cc->Inputs().Tag(kMaskTag).Get>>(); + if (mask_vec.size() > 0) + { + for (auto mask : mask_vec) + MP_RETURN_IF_ERROR(WhitenTeeth(cc, &target_format, mask)); + } + } // Copy the rendered image to output. uchar *image_mat_ptr = image_mat->data; MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat)); diff --git a/mediapipe/calculators/image_style/fast_utils_calculator.cc b/mediapipe/calculators/image_style/fast_utils_calculator.cc index 257612cbe..5a995d817 100644 --- a/mediapipe/calculators/image_style/fast_utils_calculator.cc +++ b/mediapipe/calculators/image_style/fast_utils_calculator.cc @@ -66,15 +66,6 @@ namespace mediapipe static const std::vector FACEMESH_FACE_OVAL{ {10, 338}, {338, 297}, {297, 332}, {332, 284}, {284, 251}, {251, 389}, {389, 356}, {356, 454}, {454, 323}, {323, 361}, {361, 288}, {288, 397}, {397, 365}, {365, 379}, {379, 378}, {378, 400}, {400, 377}, {377, 152}, {152, 148}, {148, 176}, {176, 149}, {149, 150}, {150, 136}, {136, 172}, {172, 58}, {58, 132}, {132, 93}, {93, 234}, {234, 127}, {127, 162}, {162, 21}, {21, 54}, {54, 103}, {103, 67}, {67, 109}, {109, 10}}; - enum - { - ATTRIB_VERTEX, - ATTRIB_TEXTURE_POSITION, - NUM_ATTRIBUTES - }; - - // Round up n to next multiple of m. - size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } bool NormalizedtoPixelCoordinates(double normalized_x, double normalized_y, @@ -150,8 +141,6 @@ namespace mediapipe source1ch = source.reshape(1, 5); target1ch = target.reshape(1, 5); - //std::cout << "R (numpy) = " << std::endl << cv::format(source, cv::Formatter::FMT_NUMPY) << std::endl << std::endl; - cv::SVD::compute(source1ch.t() * target1ch, w, u, vt); rotation = (u * vt).t(); @@ -252,6 +241,9 @@ namespace mediapipe }; std::unique_ptr image_mat; + cv::Mat mat_image_; + int image_width_; + int image_height_; }; REGISTER_CALCULATOR(FastUtilsCalculator); @@ -298,9 +290,6 @@ namespace mediapipe { image_frame_available_ = true; } - else - { - } // Set the output header based on the input header (if present). const char *tag = kImageFrameTag; @@ -322,32 +311,29 @@ namespace mediapipe { return absl::OkStatus(); } - if (cc->Inputs().HasTag(kLandmarksTag) && - cc->Inputs().Tag(kLandmarksTag).IsEmpty()) - { - return absl::OkStatus(); - } - if (cc->Inputs().HasTag(kNormLandmarksTag) && - cc->Inputs().Tag(kNormLandmarksTag).IsEmpty()) - { - return absl::OkStatus(); - } // Initialize render target, drawn with OpenCV. ImageFormat::Format target_format; std::vector> lms_out; MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); + mat_image_ = *image_mat.get(); + image_width_ = image_mat->cols; + image_height_ = image_mat->rows; - MP_RETURN_IF_ERROR(Call(cc, image_mat, target_format, lms_out)); + if (cc->Inputs().HasTag(kNormLandmarksTag) && + !cc->Inputs().Tag(kNormLandmarksTag).IsEmpty()) + { + MP_RETURN_IF_ERROR(Call(cc, image_mat, target_format, lms_out)); - cv::Mat source_lm = cv::Mat(lms_out[0]); + cv::Mat source_lm = cv::Mat(lms_out[0]); - MP_RETURN_IF_ERROR(Align(image_mat, source_lm)); + MP_RETURN_IF_ERROR(Align(image_mat, source_lm)); + } uchar *image_mat_ptr = image_mat->data; MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat)); - + return absl::OkStatus(); } @@ -360,13 +346,10 @@ namespace mediapipe CalculatorContext *cc, const ImageFormat::Format &target_format, uchar *data_image, std::unique_ptr &image_mat) { - - cv::Mat mat_image_ = *image_mat.get(); - auto output_frame = absl::make_unique( - target_format, mat_image_.cols, mat_image_.rows); + target_format, image_mat->cols, image_mat->rows); - output_frame->CopyPixelData(target_format, mat_image_.cols, mat_image_.rows, data_image, + output_frame->CopyPixelData(target_format, image_mat->cols, image_mat->rows, data_image, ImageFrame::kDefaultAlignmentBoundary); if (cc->Outputs().HasTag(kImageFrameTag)) @@ -441,11 +424,6 @@ namespace mediapipe ImageFormat::Format &target_format, std::vector> &lms_out) { - cv::Mat mat_image_ = *image_mat.get(); - - int image_width_ = image_mat->cols; - int image_height_ = image_mat->rows; - std::vector kps, landmarks; if (cc->Inputs().HasTag(kNormLandmarksTag)) @@ -455,7 +433,7 @@ namespace mediapipe std::vector point_array; for (const auto &face : landmarkslist) - { + { for (const auto &[key, value] : index_dict) { for (auto order : value) @@ -481,13 +459,13 @@ namespace mediapipe cv::Mat mean; cv::reduce(kps, mean, 1, CV_REDUCE_AVG, CV_32F); - + landmarks.push_back({mean.at(0, 0), mean.at(0, 1)}); - + kps.clear(); } lms_out.push_back(landmarks); - + landmarks.clear(); } } @@ -500,8 +478,6 @@ namespace mediapipe cv::Mat target_lm, cv::Size size, float extend, std::tuple roi) { - cv::Mat mat_image_ = *image_mat.get(); - cv::Mat source, target; source_lm.convertTo(source, CV_32F); target_lm.convertTo(target, CV_32F); @@ -528,7 +504,7 @@ namespace mediapipe cv::Mat transform, image; cv::hconcat(vec_mat, transform); - + cv::warpAffine(mat_image_, *image_mat, transform, size, 1, 0, 0.0); return absl::OkStatus(); diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc index 27c18e352..783359faa 100644 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc @@ -13,7 +13,6 @@ // limitations under the License. #include -#include #include #include "absl/strings/str_format.h" @@ -66,9 +65,6 @@ namespace NUM_ATTRIBUTES }; - std::chrono::steady_clock::time_point begin; - -std::chrono::steady_clock::time_point end; // Commonly used to compute the number of blocks to launch in a kernel. int NumGroups(const int size, const int group_size) { // NOLINT @@ -236,7 +232,7 @@ namespace mediapipe { cc->SetOffset(TimestampDiff(0)); bool use_gpu = false; - begin = std::chrono::steady_clock::now(); + if (CanUseGpu()) { #if !MEDIAPIPE_DISABLE_GPU @@ -327,10 +323,6 @@ namespace mediapipe absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext *cc) { - end = std::chrono::steady_clock::now(); - - std::cout << "Time difference = " << std::chrono::duration_cast(end - begin).count() << "[µs]" << std::endl; - std::cout << "Time difference = " << std::chrono::duration_cast(end - begin).count() << "[ns]" << std::endl; #if !MEDIAPIPE_DISABLE_GPU gpu_helper_.RunInGlContext([this] diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/beauty_single/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/beauty_single/BUILD new file mode 100644 index 000000000..e0fffe166 --- /dev/null +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/beauty_single/BUILD @@ -0,0 +1,63 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +package(default_visibility = ["//visibility:private"]) + +cc_binary( + name = "libmediapipe_jni.so", + linkshared = 1, + linkstatic = 1, + deps = [ + "//mediapipe/graphs/beauty:mobile_calculators_single", + "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", + ], +) + +cc_library( + name = "mediapipe_jni_lib", + srcs = [":libmediapipe_jni.so"], + alwayslink = 1, +) + +android_binary( + name = "beautygpusingle", + srcs = glob(["*.java"]), + assets = [ + "//mediapipe/graphs/beauty:beauty_mobile_single.binarypb", + "//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", + ], + assets_dir = "", + manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", + manifest_values = { + "applicationId": "com.google.mediapipe.apps.beautygpusingle", + "appName": "BeautySingle", + "mainActivity": ".MainActivity", + "cameraFacingFront": "True", + "binaryGraphName": "beauty_mobile_single.binarypb", + "inputVideoStreamName": "input_video", + "outputVideoStreamName": "output_video", + "flipFramesVertically": "True", + "converterNumBuffers": "2", + }, + multidex = "native", + deps = [ + ":mediapipe_jni_lib", + "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//mediapipe/framework/formats:landmark_java_proto_lite", + "//mediapipe/java/com/google/mediapipe/framework:android_framework", + ], +) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/beauty_single/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/beauty_single/MainActivity.java new file mode 100644 index 000000000..634ed0a73 --- /dev/null +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/beauty_single/MainActivity.java @@ -0,0 +1,93 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.apps.beautygpusingle; + +import android.os.Bundle; +import android.util.Log; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList; +import com.google.mediapipe.framework.AndroidPacketCreator; +import com.google.mediapipe.framework.Packet; +import com.google.mediapipe.framework.PacketGetter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Main activity of MediaPipe face mesh app. */ +public class MainActivity extends com.google.mediapipe.apps.basic.MainActivity { + private static final String TAG = "MainActivity"; + + private static final String INPUT_NUM_FACES_SIDE_PACKET_NAME = "num_faces"; + private static final String OUTPUT_LANDMARKS_STREAM_NAME = "multi_face_landmarks"; + // Max number of faces to detect/process. + private static final int NUM_FACES = 1; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + AndroidPacketCreator packetCreator = processor.getPacketCreator(); + Map inputSidePackets = new HashMap<>(); + inputSidePackets.put(INPUT_NUM_FACES_SIDE_PACKET_NAME, packetCreator.createInt32(NUM_FACES)); + processor.setInputSidePackets(inputSidePackets); + + // To show verbose logging, run: + // adb shell setprop log.tag.MainActivity VERBOSE + if (Log.isLoggable(TAG, Log.VERBOSE)) { + processor.addPacketCallback( + OUTPUT_LANDMARKS_STREAM_NAME, + (packet) -> { + Log.v(TAG, "Received multi face landmarks packet."); + List multiFaceLandmarks = + PacketGetter.getProtoVector(packet, NormalizedLandmarkList.parser()); + Log.v( + TAG, + "[TS:" + + packet.getTimestamp() + + "] " + + getMultiFaceLandmarksDebugString(multiFaceLandmarks)); + }); + } + } + + private static String getMultiFaceLandmarksDebugString( + List multiFaceLandmarks) { + if (multiFaceLandmarks.isEmpty()) { + return "No face landmarks"; + } + String multiFaceLandmarksStr = "Number of faces detected: " + multiFaceLandmarks.size() + "\n"; + int faceIndex = 0; + for (NormalizedLandmarkList landmarks : multiFaceLandmarks) { + multiFaceLandmarksStr += + "\t#Face landmarks for face[" + faceIndex + "]: " + landmarks.getLandmarkCount() + "\n"; + int landmarkIndex = 0; + for (NormalizedLandmark landmark : landmarks.getLandmarkList()) { + multiFaceLandmarksStr += + "\t\tLandmark [" + + landmarkIndex + + "]: (" + + landmark.getX() + + ", " + + landmark.getY() + + ", " + + landmark.getZ() + + ")\n"; + ++landmarkIndex; + } + ++faceIndex; + } + return multiFaceLandmarksStr; + } +} diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/deformation/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/deformation/BUILD new file mode 100644 index 000000000..ee8d32b08 --- /dev/null +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/deformation/BUILD @@ -0,0 +1,81 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +package(default_visibility = ["//visibility:private"]) + +cc_binary( + name = "libmediapipe_jni.so", + linkshared = 1, + linkstatic = 1, + deps = [ + "//mediapipe/graphs/deformation:mobile_calculators", + "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni", + ], +) + +cc_library( + name = "mediapipe_jni_lib", + srcs = [":libmediapipe_jni.so"], + alwayslink = 1, +) + +ASSETS_DIR = "//mediapipe/graphs/deformation/config" + +android_binary( + name = "deformationgpu", + srcs = glob(["*.java"]), + assets = [ + "//mediapipe/graphs/deformation:mobile_gpu.binarypb", + "//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite", + "//mediapipe/modules/face_detection:face_detection_short_range.tflite", + ASSETS_DIR+":triangles.txt", + ASSETS_DIR+":index_names.txt", + ASSETS_DIR+":mediumFaceIndexes.txt", + ASSETS_DIR+":mediumNoseIndexes.txt", + ASSETS_DIR+":noseAllIndexes.txt", + ASSETS_DIR+":cheekbonesIndexes.txt", + ASSETS_DIR+":additionalNoseIndexes1.txt", + ASSETS_DIR+":additionalNoseIndexes2.txt", + ASSETS_DIR+":additionalNoseIndexes3.txt", + ASSETS_DIR+":faceOvalIndexes.txt", + ASSETS_DIR+":lowerLipCnt.txt", + ASSETS_DIR+":upperLipCnt.txt", + ASSETS_DIR+":widerLowerLipPts1.txt", + ASSETS_DIR+":widerUpperLipPts1.txt", + ], + assets_dir = "", + manifest = "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:AndroidManifest.xml", + manifest_values = { + "applicationId": "com.google.mediapipe.apps.deformationgpu", + "appName": "Deformation", + "mainActivity": "com.google.mediapipe.apps.basic.MainActivity", + "cameraFacingFront": "True", + "binaryGraphName": "mobile_gpu.binarypb", + "inputVideoStreamName": "input_video", + "outputVideoStreamName": "output_video", + "flipFramesVertically": "True", + "converterNumBuffers": "2", + }, + multidex = "native", + deps = [ + ":mediapipe_jni_lib", + "//mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic:basic_lib", + "//mediapipe/framework/formats:landmark_java_proto_lite", + "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//mediapipe/modules/face_geometry/protos:face_geometry_java_proto_lite", + ], +) + diff --git a/mediapipe/examples/desktop/beauty/BUILD b/mediapipe/examples/desktop/beauty/BUILD index 761940617..eea3911d8 100644 --- a/mediapipe/examples/desktop/beauty/BUILD +++ b/mediapipe/examples/desktop/beauty/BUILD @@ -41,10 +41,10 @@ cc_binary( ) cc_binary( - name = "beauty_cpu_over", + name = "beauty_cpu_single", deps = [ "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/beauty:desktop_live_over_calculators", + "//mediapipe/graphs/beauty:desktop_live_single_calculators", ], ) diff --git a/mediapipe/examples/desktop/deformation/BUILD b/mediapipe/examples/desktop/deformation/BUILD new file mode 100644 index 000000000..703a53ce5 --- /dev/null +++ b/mediapipe/examples/desktop/deformation/BUILD @@ -0,0 +1,29 @@ +# Copyright 2021 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +package(default_visibility = ["//mediapipe/examples:__subpackages__"]) + +# Linux only +cc_binary( + name = "deformation_cpu", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main", + "//mediapipe/graphs/deformation:desktop_calculators", + ], +) + + + diff --git a/mediapipe/examples/desktop/demo_run_graph_main.cc b/mediapipe/examples/desktop/demo_run_graph_main.cc index b80237e2f..f08e32d9c 100644 --- a/mediapipe/examples/desktop/demo_run_graph_main.cc +++ b/mediapipe/examples/desktop/demo_run_graph_main.cc @@ -96,7 +96,7 @@ absl::Status RunMPPGraph() { break; } cv::Mat camera_frame; - cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); + cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGBA); if (!load_video) { cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); } diff --git a/mediapipe/graphs/beauty/BUILD b/mediapipe/graphs/beauty/BUILD index 8e8409f59..79e46479e 100644 --- a/mediapipe/graphs/beauty/BUILD +++ b/mediapipe/graphs/beauty/BUILD @@ -21,17 +21,6 @@ licenses(["notice"]) package(default_visibility = ["//visibility:public"]) -cc_library( - name = "desktop_calculators", - deps = [ - "//mediapipe/calculators/core:constant_side_packet_calculator", - "//mediapipe/calculators/video:opencv_video_decoder_calculator", - "//mediapipe/calculators/video:opencv_video_encoder_calculator", - "//mediapipe/graphs/beauty/subgraphs:face_renderer_cpu", - "//mediapipe/modules/face_landmark:face_landmark_front_cpu", - ], -) - cc_library( name = "desktop_live_calculators", deps = [ @@ -43,12 +32,12 @@ cc_library( ) cc_library( - name = "desktop_live_over_calculators", + name = "desktop_live_single_calculators", deps = [ "//mediapipe/calculators/core:constant_side_packet_calculator", "//mediapipe/calculators/core:flow_limiter_calculator", - "//mediapipe/graphs/beauty/subgraphs:face_renderer_cpu_over", - "//mediapipe/modules/face_landmark:face_landmark_front_gpu", + "//mediapipe/graphs/beauty/subgraphs:face_renderer_cpu_single", + "//mediapipe/modules/face_landmark:face_landmark_front_cpu", ], ) @@ -75,12 +64,12 @@ cc_library( ) cc_library( - name = "mobile_calculators_over", + name = "mobile_calculators_single", deps = [ "//mediapipe/gpu:gpu_buffer_to_image_frame_calculator", "//mediapipe/gpu:image_frame_to_gpu_buffer_calculator", "//mediapipe/calculators/core:flow_limiter_calculator", - "//mediapipe/graphs/beauty/subgraphs:face_renderer_gpu_over", + "//mediapipe/graphs/beauty/subgraphs:face_renderer_cpu_single", "//mediapipe/modules/face_landmark:face_landmark_front_gpu", ], ) @@ -94,9 +83,9 @@ mediapipe_binary_graph( ) mediapipe_binary_graph( - name = "beauty_mobile_over_binary_graph", - graph = "beauty_over.pbtxt", - output_name = "beauty_mobile_over.binarypb", - deps = [":mobile_calculators_over"], + name = "beauty_mobile_single_binary_graph", + graph = "beauty_mobile_single.pbtxt", + output_name = "beauty_mobile_single.binarypb", + deps = [":mobile_calculators_single"], ) diff --git a/mediapipe/graphs/beauty/README.md b/mediapipe/graphs/beauty/README.md new file mode 100644 index 000000000..00aefe7af --- /dev/null +++ b/mediapipe/graphs/beauty/README.md @@ -0,0 +1,66 @@ +# Beauty + +this graph performs face processing + +## Getting started + +Clone branch. + +1. Desktop-CPU (Divided calculators) + +Build with: +``` +bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/beauty:beauty_cpu +``` +Run with (using your camera): +``` +bazel-bin/mediapipe/examples/desktop/beauty/beauty_cpu +--calculator_graph_config_file=mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt +``` +Run with (using video): +``` +bazel-bin/mediapipe/examples/desktop/beauty/beauty_cpu +--calculator_graph_config_file=mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt +--input_video_path=/path/video.mp4 +--output_video_path=/path/outvideo.mp4 +``` + +2. Desktop-CPU-Single (Not divided, using render data) + +Build with: +``` +bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/beauty:beauty_cpu_single +``` +Run with (using your camera): +``` +bazel-bin/mediapipe/examples/desktop/beauty/beauty_cpu_single +--calculator_graph_config_file=mediapipe/graphs/beauty/beauty_desktop_cpu_single.pbtxt +``` +Run with (using video): +``` +bazel-bin/mediapipe/examples/desktop/beauty/beauty_cpu_single +--calculator_graph_config_file=mediapipe/graphs/beauty/beauty_desktop_cpu_single.pbtxt +--input_video_path=/path/video.mp4 +--output_video_path=/path/outvideo.mp4 +``` +3. Mobile (Android) + +Build with: +``` +bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/beauty:beautygpu +``` +Install with: +``` +adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/beauty/beautygpu.apk +``` + +4. Mobile-Single (Android) + +Build with: +``` +bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/beauty_single:beautygpusingle +``` +Install with: +``` +adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/beauty_single/beautygpusingle.apk +``` diff --git a/mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt b/mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt index 816f01638..c68bf8df9 100644 --- a/mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt +++ b/mediapipe/graphs/beauty/beauty_desktop_cpu.pbtxt @@ -9,6 +9,7 @@ output_stream: "output_video" # landmarks. (std::vector) output_stream: "multi_face_landmarks" + profiler_config { trace_enabled: true enable_profiler: true @@ -26,6 +27,8 @@ profiler_config { # real-time mobile applications. It also eliminates unnecessarily computation, # e.g., the output produced by a node may get dropped downstream if the # subsequent nodes are still busy processing previous inputs. + + node { calculator: "FlowLimiterCalculator" input_stream: "input_video" diff --git a/mediapipe/graphs/beauty/beauty_desktop.pbtxt b/mediapipe/graphs/beauty/beauty_desktop_cpu_single.pbtxt similarity index 90% rename from mediapipe/graphs/beauty/beauty_desktop.pbtxt rename to mediapipe/graphs/beauty/beauty_desktop_cpu_single.pbtxt index 3a00401a7..b965430a2 100644 --- a/mediapipe/graphs/beauty/beauty_desktop.pbtxt +++ b/mediapipe/graphs/beauty/beauty_desktop_cpu_single.pbtxt @@ -9,6 +9,7 @@ output_stream: "output_video" # landmarks. (std::vector) output_stream: "multi_face_landmarks" + profiler_config { trace_enabled: true enable_profiler: true @@ -16,6 +17,7 @@ profiler_config { trace_log_path: "/Users/alena/Workdir/mediapipe/logs/beauty/" } + node { calculator: "FlowLimiterCalculator" input_stream: "input_video" @@ -27,6 +29,7 @@ node { output_stream: "throttled_input_video" } +# Defines side packets for further use in the graph. node { calculator: "ConstantSidePacketCalculator" output_side_packet: "PACKET:0:num_faces" @@ -48,9 +51,9 @@ node { output_stream: "LANDMARKS:multi_face_landmarks" } -# Subgraph that renders onto the input image. +# Subgraph that renders face-landmark annotation onto the input image. node { - calculator: "FaceRendererCpu" + calculator: "FaceRendererCpuSingle" input_stream: "IMAGE:throttled_input_video" input_stream: "LANDMARKS:multi_face_landmarks" output_stream: "IMAGE:output_video" diff --git a/mediapipe/graphs/beauty/beauty_mobile_single.pbtxt b/mediapipe/graphs/beauty/beauty_mobile_single.pbtxt new file mode 100644 index 000000000..ce0e41012 --- /dev/null +++ b/mediapipe/graphs/beauty/beauty_mobile_single.pbtxt @@ -0,0 +1,76 @@ +# MediaPipe graph that performs face mesh with TensorFlow Lite on GPU. + +# GPU buffer. (GpuBuffer) +input_stream: "input_video" + +# Max number of faces to detect/process. (int) +input_side_packet: "num_faces" + +# Output image with rendered results. (GpuBuffer) +output_stream: "output_video" +# Collection of detected/processed faces, each represented as a list of +# landmarks. (std::vector) +output_stream: "multi_face_landmarks" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for downstream nodes +# (calculators and subgraphs) in the graph to finish their tasks before it +# passes through another image. All images that come in while waiting are +# dropped, limiting the number of in-flight images in most part of the graph to +# 1. This prevents the downstream nodes from queuing up incoming images and data +# excessively, which leads to increased latency and memory usage, unwanted in +# real-time mobile applications. It also eliminates unnecessarily computation, +# e.g., the output produced by a node may get dropped downstream if the +# subsequent nodes are still busy processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:output_video" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Defines side packets for further use in the graph. +node { + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:with_attention" + node_options: { + [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { + packet { bool_value: true } + } + } +} + +# Defines side packets for further use in the graph. +node { + calculator: "GpuBufferToImageFrameCalculator" + input_stream: "throttled_input_video" + output_stream: "throttled_input_video_cpu" +} + +# Subgraph that detects faces and corresponding landmarks. +node { + calculator: "FaceLandmarkFrontGpu" + input_stream: "IMAGE:throttled_input_video" + input_side_packet: "NUM_FACES:num_faces" + input_side_packet: "WITH_ATTENTION:with_attention" + output_stream: "LANDMARKS:multi_face_landmarks" +} + +# Subgraph that renders face-landmark annotation onto the input image. +node { + calculator: "FaceRendererCpuSingle" + input_stream: "IMAGE:throttled_input_video_cpu" + input_stream: "LANDMARKS:multi_face_landmarks" + output_stream: "IMAGE:output_video_cpu" +} + +# Defines side packets for further use in the graph. +node { + calculator: "ImageFrameToGpuBufferCalculator" + input_stream: "output_video_cpu" + output_stream: "output_video" +} diff --git a/mediapipe/graphs/beauty/calculators/annotation_renderer.cc b/mediapipe/graphs/beauty/calculators/annotation_renderer.cc deleted file mode 100644 index fcb5bf109..000000000 --- a/mediapipe/graphs/beauty/calculators/annotation_renderer.cc +++ /dev/null @@ -1,953 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/util/annotation_renderer.h" - -#include - -#include -#include -//#include - -#include "mediapipe/framework/port/logging.h" -#include "mediapipe/framework/port/vector.h" -#include "mediapipe/util/color.pb.h" - -namespace mediapipe { -namespace { - -using Arrow = RenderAnnotation::Arrow; -using FilledOval = RenderAnnotation::FilledOval; -using FilledRectangle = RenderAnnotation::FilledRectangle; -using FilledRoundedRectangle = RenderAnnotation::FilledRoundedRectangle; -using Point = RenderAnnotation::Point; -using Line = RenderAnnotation::Line; -using GradientLine = RenderAnnotation::GradientLine; -using Oval = RenderAnnotation::Oval; -using Rectangle = RenderAnnotation::Rectangle; -using RoundedRectangle = RenderAnnotation::RoundedRectangle; -using Text = RenderAnnotation::Text; - -static const std::vector UPPER_LIP = {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78}; -static const std::vector LOWER_LIP = {61, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}; -static const std::vector FACE_OVAL = {10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356, - 454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378, - 378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150, - 136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162, - 21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10}; -static const std::vector MOUTH_INSIDE = {78, 191, 80, 81, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95}; -static const std::vector PART_FOREHEAD_B = {21, 54, 103, 67, 109, 10, 338, 297, 332, 284, 251, 301, 293, 334, 296, 336, 9, 107, 66, 105, 63, 71}; -static const std::vector LEFT_EYE = {130, 33, 246, 161, 160, 159, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7}; -static const std::vector RIGHT_EYE = {362, 398, 384, 385, 386, 387, 388, 466, 263, 249, 390, 373, 374, 380, 381, 382}; -static const std::vector LIPS = {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}; -static const std::vector LEFT_BROW = {70, 63, 105, 66, 107, 55, 65, 52, 53, 46}; -static const std::vector RIGHT_BROW = {336, 296, 334, 293, 301, 300, 283, 282, 295, 285}; - -int ClampThickness(int thickness) { - constexpr int kMaxThickness = 32767; // OpenCV MAX_THICKNESS - return std::clamp(thickness, 1, kMaxThickness); -} - -bool NormalizedtoPixelCoordinates(double normalized_x, double normalized_y, - int image_width, int image_height, int* x_px, - int* y_px) { - CHECK(x_px != nullptr); - CHECK(y_px != nullptr); - CHECK_GT(image_width, 0); - CHECK_GT(image_height, 0); - - if (normalized_x < 0 || normalized_x > 1.0 || normalized_y < 0 || - normalized_y > 1.0) { - VLOG(1) << "Normalized coordinates must be between 0.0 and 1.0"; - } - - *x_px = static_cast(round(normalized_x * image_width)); - *y_px = static_cast(round(normalized_y * image_height)); - - return true; -} - -cv::Scalar MediapipeColorToOpenCVColor(const Color& color) { - return cv::Scalar(color.r(), color.g(), color.b()); -} - -cv::RotatedRect RectangleToOpenCVRotatedRect(int left, int top, int right, - int bottom, double rotation) { - return cv::RotatedRect( - cv::Point2f((left + right) / 2.f, (top + bottom) / 2.f), - cv::Size2f(right - left, bottom - top), rotation / M_PI * 180.f); -} - -void cv_line2(cv::Mat& img, const cv::Point& start, const cv::Point& end, - const cv::Scalar& color1, const cv::Scalar& color2, - int thickness) { - cv::LineIterator iter(img, start, end, /*cv::LINE_4=*/4); - for (int i = 0; i < iter.count; i++, iter++) { - const double alpha = static_cast(i) / iter.count; - const cv::Scalar new_color(color1 * (1.0 - alpha) + color2 * alpha); - const cv::Rect rect(iter.pos(), cv::Size(thickness, thickness)); - cv::rectangle(img, rect, new_color, /*cv::FILLED=*/-1, /*cv::LINE_4=*/4); - } -} - -} // namespace - -void AnnotationRenderer::RenderDataOnImage(const RenderData &render_data) -{ - if (render_data.render_annotations().size()){ - DrawLipstick(render_data); - WhitenTeeth(render_data); - smooth_face(render_data); - } - else - { - LOG(FATAL) << "Unknown annotation type: "; - } -} - -void AnnotationRenderer::AdoptImage(cv::Mat* input_image) { - image_width_ = input_image->cols; - image_height_ = input_image->rows; - - // No pixel data copy here, only headers are copied. - mat_image_ = *input_image; -} - -int AnnotationRenderer::GetImageWidth() const { return mat_image_.cols; } -int AnnotationRenderer::GetImageHeight() const { return mat_image_.rows; } - -void AnnotationRenderer::SetFlipTextVertically(bool flip) { - flip_text_vertically_ = flip; -} - -void AnnotationRenderer::SetScaleFactor(float scale_factor) { - if (scale_factor > 0.0f) scale_factor_ = std::min(scale_factor, 1.0f); -} - -cv::Mat AnnotationRenderer::FormFacePartMask(std::vector orderList, const RenderData &render_data) -{ - int c = 0; - std::vector point_array; - for (auto order : orderList) - { - c = 0; - for (auto &annotation : render_data.render_annotations()) - { - if (annotation.data_case() == RenderAnnotation::kPoint) - { - if (order == c) - { - const auto &point = annotation.point(); - int x = -1; - int y = -1; - if (point.normalized()) - { - CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_, - image_height_, &x, &y)); - } - else - { - x = static_cast(point.x() * scale_factor_); - y = static_cast(point.y() * scale_factor_); - } - point_array.push_back(cv::Point(x, y)); - } - c += 1; - } - } - } - - cv::Mat mask; - std::vector> point; - point.push_back(point_array); - mask = cv::Mat::zeros(mat_image_.size(), CV_32F); - - cv::fillPoly(mask, point, cv::Scalar::all(255), cv::LINE_AA); - mask.convertTo(mask, CV_8U); - - return mask; -} - -std::tuple AnnotationRenderer::GetFaceBox(const RenderData &render_data) -{ - std::vector x_s, y_s; - double box_min_y, box_max_y, box_max_x, box_min_x; - - for (auto &annotation : render_data.render_annotations()) - { - if (annotation.data_case() == RenderAnnotation::kPoint) - { - const auto &point = annotation.point(); - int x = -1; - int y = -1; - if (point.normalized()) - { - CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_, - image_height_, &x, &y)); - } - else - { - x = static_cast(point.x() * scale_factor_); - y = static_cast(point.y() * scale_factor_); - } - x_s.push_back(point.x()); - x_s.push_back(point.y()); - } - } - cv::minMaxLoc(y_s, &box_min_y, &box_max_y); - cv::minMaxLoc(x_s, &box_min_x, &box_max_x); - box_min_y = box_min_y * 0.9; - - return std::make_tuple(box_min_x, box_min_y, box_max_x, box_max_y); -} - -cv::Mat AnnotationRenderer::predict_forehead_mask(const RenderData &render_data, double face_box_min_y) -{ - - cv::Mat part_forehead_mask = AnnotationRenderer::FormFacePartMask(PART_FOREHEAD_B, render_data); - part_forehead_mask.convertTo(part_forehead_mask, CV_32F, 1.0 / 255); - part_forehead_mask.convertTo(part_forehead_mask, CV_8U); - - cv::Mat image_sm, image_sm_hsv, skinMask; - - cv::resize(mat_image_, image_sm, cv::Size(mat_image_.size().width, mat_image_.size().height)); - cv::cvtColor(image_sm, image_sm_hsv, cv::COLOR_BGR2HSV); - - std::vector x, y; - std::vector location; - // std::cout << "R (numpy) = " << std::endl << cv::format(part_forehead_mask, cv::Formatter::FMT_NUMPY ) << std::endl << std::endl; - - cv::Vec3d hsv_min, hsv_max; - - std::vector channels(3); - cv::split(image_sm_hsv, channels); - std::vector> minx(3), maxx(3); - int c = 0; - for (auto ch : channels) - { - cv::Mat row, mask_row; - double min, max; - for (int i = 0; i < ch.rows; i++) - { - row = ch.row(i); - mask_row = part_forehead_mask.row(i); - cv::minMaxLoc(row, &min, &max, 0, 0, mask_row); - minx[c].push_back(min); - maxx[c].push_back(max); - } - c++; - } - for (int i = 0; i < 3; i++) - { - hsv_min[i] = *std::min_element(minx[i].begin(), minx[i].end()); - } - for (int i = 0; i < 3; i++) - { - hsv_max[i] = *std::max_element(maxx[i].begin(), maxx[i].end()); - } - - cv::Mat _forehead_kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(1, 1)); - cv::inRange(image_sm_hsv, hsv_min, hsv_max, skinMask); - cv::erode(skinMask, skinMask, _forehead_kernel, cv::Point(-1, -1), 2); - cv::dilate(skinMask, skinMask, _forehead_kernel, cv::Point(-1, -1), 2); - skinMask.convertTo(skinMask, CV_8U, 1.0 / 255); - - cv::findNonZero(skinMask, location); - - double max_part_f, x_min_part, x_max_part; - - for (auto &i : location) - { - x.push_back(i.x); - y.push_back(i.y); - } - - cv::minMaxLoc(y, NULL, &max_part_f); - cv::minMaxLoc(x, &x_min_part, &x_max_part); - - cv::Mat new_skin_mask = cv::Mat::zeros(skinMask.size(), CV_8U); - - new_skin_mask(cv::Range(face_box_min_y, max_part_f), cv::Range(x_min_part, x_max_part)) = - skinMask(cv::Range(face_box_min_y, max_part_f), cv::Range(x_min_part, x_max_part)); - - return new_skin_mask; -} - -void AnnotationRenderer::smooth_face(const RenderData &render_data) -{ - - cv::Mat not_full_face = cv::Mat(FormFacePartMask(FACE_OVAL, render_data)) + - cv::Mat(predict_forehead_mask(render_data, std::get<1>(GetFaceBox(render_data)))) - - cv::Mat(FormFacePartMask(LEFT_EYE, render_data)) - - cv::Mat(FormFacePartMask(RIGHT_EYE, render_data)) - - cv::Mat(FormFacePartMask(LEFT_BROW, render_data)) - - cv::Mat(FormFacePartMask(RIGHT_BROW, render_data)) - - cv::Mat(FormFacePartMask(LIPS, render_data)); - - cv::resize(not_full_face, - not_full_face, - mat_image_.size(), 0, 0, - cv::INTER_LINEAR); - - std::vector x, y; - std::vector location; - - cv::findNonZero(not_full_face, location); - - double min_y, min_x, max_x, max_y; - - for (auto &i : location) - { - x.push_back(i.x); - y.push_back(i.y); - } - - cv::minMaxLoc(x, &min_x, &max_x); - cv::minMaxLoc(y, &min_y, &max_y); - - cv::Mat patch_face = mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x)); - cv::Mat patch_nff = not_full_face(cv::Range(min_y, max_y), cv::Range(min_x, max_x)); - cv::Mat patch_new; - cv::bilateralFilter(patch_face, patch_new, 12, 50, 50); - - cv::Mat patch_new_nff, patch_new_mask, patch, patch_face_nff; - - patch_new.copyTo(patch_new_nff, patch_nff); - patch_face.copyTo(patch_face_nff, patch_nff); - - patch_new_mask = 0.85 * patch_new_nff + 0.15 * patch_face_nff; - - patch = cv::min(255, patch_new_mask); - patch.copyTo(patch_face, patch_nff); -} - -cv::Mat matmul32F(cv::Mat& bgr, cv::Mat& mask) -{ - assert(bgr.type() == CV_32FC3 && mask.type() == CV_32FC1 && bgr.size() == mask.size()); - int H = bgr.rows; - int W = bgr.cols; - cv::Mat dst(bgr.size(), bgr.type()); - - if (bgr.isContinuous() && mask.isContinuous()) - { - W *= H; - H = 1; - } - - for( int i = 0; i < H; ++i) - { - float* pdst = ((float*)dst.data)+i*W*3; - float* pbgr = ((float*)bgr.data)+i*W*3; - float* pmask = ((float*)mask.data) + i*W; - for ( int j = 0; j < W; ++j) - { - (*pdst++) = (*pbgr++) *(*pmask); - (*pdst++) = (*pbgr++) *(*pmask); - (*pdst++) = (*pbgr++) *(*pmask); - pmask+=1; - } - } - return dst; -} - -void AnnotationRenderer::DrawLipstick(const RenderData &render_data) -{ - cv::Mat spec_lips_mask, upper_lips_mask, lower_lips_mask; - spec_lips_mask = cv::Mat::zeros(mat_image_.size(), CV_32F); - upper_lips_mask = cv::Mat::zeros(mat_image_.size(), CV_32F); - lower_lips_mask = cv::Mat::zeros(mat_image_.size(), CV_32F); - - upper_lips_mask = AnnotationRenderer::FormFacePartMask(UPPER_LIP, render_data); - lower_lips_mask = AnnotationRenderer::FormFacePartMask(LOWER_LIP, render_data); - - spec_lips_mask = upper_lips_mask + lower_lips_mask; - - spec_lips_mask.convertTo(spec_lips_mask, CV_8U); - - cv::resize(spec_lips_mask, spec_lips_mask, mat_image_.size(), cv::INTER_LINEAR); - - std::vector x, y; - std::vector location; - - cv::findNonZero(spec_lips_mask, location); - - for (auto &i : location) - { - x.push_back(i.x); - y.push_back(i.y); - } - - if (!(x.empty()) && !(y.empty())) - { - double min_y, max_y, max_x, min_x; - cv::minMaxLoc(y, &min_y, &max_y); - cv::minMaxLoc(x, &min_x, &max_x); - - cv::Mat lips_crop_mask = spec_lips_mask(cv::Range(min_y, max_y), cv::Range(min_x, max_x)); - lips_crop_mask.convertTo(lips_crop_mask, CV_32F, 1.0 / 255); - - cv::Mat lips_crop = cv::Mat(mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x)).size(), CV_8UC3); - mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x)).copyTo(lips_crop); - - lips_crop.convertTo(lips_crop, CV_32FC3); - - cv::Mat lips_blend; - lips_blend = cv::Mat(lips_crop.size().height, lips_crop.size().width, CV_32FC3, cv::Scalar(255.0, 0, 0)); - lips_crop_mask *= 50; - lips_crop_mask.convertTo(lips_crop_mask, CV_32F, 1.0 / 255); - - lips_blend = matmul32F(lips_blend, lips_crop_mask); - - cv::Mat tmp_crop_mask = 1.0 - lips_crop_mask; - - cv::Mat slice = mat_image_(cv::Range(min_y, max_y), cv::Range(min_x, max_x)); - - lips_crop = matmul32F(lips_crop, tmp_crop_mask); - - cv::add(lips_blend, lips_crop, slice, cv::noArray(), CV_8U); - } -} - -void AnnotationRenderer::WhitenTeeth(const RenderData &render_data) -{ - cv::Mat mouth_mask, mouth; - - mouth_mask = cv::Mat::zeros(mat_image_.size(), CV_32F); - mouth_mask = AnnotationRenderer::FormFacePartMask(MOUTH_INSIDE, render_data); - - cv::resize(mouth_mask, mouth, mat_image_.size(), cv::INTER_LINEAR); - - std::vector x, y; - std::vector location; - - cv::findNonZero(mouth, location); - - for (auto &i : location) - { - x.push_back(i.x); - y.push_back(i.y); - } - - if (!(x.empty()) && !(y.empty())) - { - double mouth_min_y, mouth_max_y, mouth_max_x, mouth_min_x; - cv::minMaxLoc(y, &mouth_min_y, &mouth_max_y); - cv::minMaxLoc(x, &mouth_min_x, &mouth_max_x); - double mh = mouth_max_y - mouth_min_y; - double mw = mouth_max_x - mouth_min_x; - cv::Mat mouth_crop_mask; - mouth.convertTo(mouth, CV_32F, 1.0 / 255); - mouth.convertTo(mouth, CV_32F, 1.0 / 255); - if (mh / mw > 0.17) - { - mouth_min_y = static_cast(std::max(mouth_min_y - mh * 0.1, 0.0)); - mouth_max_y = static_cast(std::min(mouth_max_y + mh * 0.1, (double)image_height_)); - mouth_min_x = static_cast(std::max(mouth_min_x - mw * 0.1, 0.0)); - mouth_max_x = static_cast(std::min(mouth_max_x + mw * 0.1, (double)image_width_)); - mouth_crop_mask = mouth(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)); - cv::Mat img_hsv, tmp_mask, img_hls; - cv::cvtColor(mat_image_(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)), img_hsv, - cv::COLOR_RGB2HSV); - - cv::Mat _mouth_erode_kernel = cv::getStructuringElement( - cv::MORPH_ELLIPSE, cv::Size(7, 7)); - - cv::erode(mouth_crop_mask * 255, tmp_mask, _mouth_erode_kernel, cv::Point(-1, -1), 3); - cv::GaussianBlur(tmp_mask, tmp_mask, cv::Size(51, 51), 0); - - img_hsv.convertTo(img_hsv, CV_8U); - - std::vector channels(3); - cv::split(img_hsv, channels); - - cv::Mat tmp; - cv::multiply(channels[1], tmp_mask, tmp, 0.3, CV_8U); - cv::subtract(channels[1], tmp, channels[1], cv::noArray(), CV_8U); - channels[1] = cv::min(255, channels[1]); - cv::merge(channels, img_hsv); - - cv::cvtColor(img_hsv, img_hsv, cv::COLOR_HSV2RGB); - cv::cvtColor(img_hsv, img_hls, cv::COLOR_RGB2HLS); - - cv::split(img_hls, channels); - cv::multiply(channels[1], tmp_mask, tmp, 0.3, CV_8U); - cv::add(channels[1], tmp, channels[1], cv::noArray(), CV_8U); - channels[1] = cv::min(255, channels[1]); - cv::merge(channels, img_hls); - - cv::cvtColor(img_hls, img_hls, cv::COLOR_HLS2RGB); - - // std::cout << "R (numpy) = " << std::endl << cv::format(img_hls, cv::Formatter::FMT_NUMPY ) << std::endl << std::endl; - - cv::Mat slice = mat_image_(cv::Range(mouth_min_y, mouth_max_y), cv::Range(mouth_min_x, mouth_max_x)); - img_hls.copyTo(slice); - } - } -} - -void AnnotationRenderer::DrawRectangle(const RenderAnnotation& annotation) { - int left = -1; - int top = -1; - int right = -1; - int bottom = -1; - const auto& rectangle = annotation.rectangle(); - if (rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), - image_width_, image_height_, &left, - &top)); - CHECK(NormalizedtoPixelCoordinates(rectangle.right(), rectangle.bottom(), - image_width_, image_height_, &right, - &bottom)); - } else { - left = static_cast(rectangle.left() * scale_factor_); - top = static_cast(rectangle.top() * scale_factor_); - right = static_cast(rectangle.right() * scale_factor_); - bottom = static_cast(rectangle.bottom() * scale_factor_); - } - - const cv::Scalar color = MediapipeColorToOpenCVColor(annotation.color()); - const int thickness = - ClampThickness(round(annotation.thickness() * scale_factor_)); - if (rectangle.rotation() != 0.0) { - const auto& rect = RectangleToOpenCVRotatedRect(left, top, right, bottom, - rectangle.rotation()); - const int kNumVertices = 4; - cv::Point2f vertices[kNumVertices]; - rect.points(vertices); - for (int i = 0; i < kNumVertices; i++) { - cv::line(mat_image_, vertices[i], vertices[(i + 1) % kNumVertices], color, - thickness); - } - } else { - cv::Rect rect(left, top, right - left, bottom - top); - cv::rectangle(mat_image_, rect, color, thickness); - } - if (rectangle.has_top_left_thickness()) { - const auto& rect = RectangleToOpenCVRotatedRect(left, top, right, bottom, - rectangle.rotation()); - const int kNumVertices = 4; - cv::Point2f vertices[kNumVertices]; - rect.points(vertices); - const int top_left_thickness = - ClampThickness(round(rectangle.top_left_thickness() * scale_factor_)); - cv::ellipse(mat_image_, vertices[1], - cv::Size(top_left_thickness, top_left_thickness), 0.0, 0, 360, - color, -1); - } -} - -void AnnotationRenderer::DrawFilledRectangle( - const RenderAnnotation& annotation) { - int left = -1; - int top = -1; - int right = -1; - int bottom = -1; - const auto& rectangle = annotation.filled_rectangle().rectangle(); - if (rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), - image_width_, image_height_, &left, - &top)); - CHECK(NormalizedtoPixelCoordinates(rectangle.right(), rectangle.bottom(), - image_width_, image_height_, &right, - &bottom)); - } else { - left = static_cast(rectangle.left() * scale_factor_); - top = static_cast(rectangle.top() * scale_factor_); - right = static_cast(rectangle.right() * scale_factor_); - bottom = static_cast(rectangle.bottom() * scale_factor_); - } - - const cv::Scalar color = MediapipeColorToOpenCVColor(annotation.color()); - if (rectangle.rotation() != 0.0) { - const auto& rect = RectangleToOpenCVRotatedRect(left, top, right, bottom, - rectangle.rotation()); - const int kNumVertices = 4; - cv::Point2f vertices2f[kNumVertices]; - rect.points(vertices2f); - // Convert cv::Point2f[] to cv::Point[]. - cv::Point vertices[kNumVertices]; - for (int i = 0; i < kNumVertices; ++i) { - vertices[i] = vertices2f[i]; - } - cv::fillConvexPoly(mat_image_, vertices, kNumVertices, color); - } else { - cv::Rect rect(left, top, right - left, bottom - top); - cv::rectangle(mat_image_, rect, color, -1); - } -} - -void AnnotationRenderer::DrawRoundedRectangle( - const RenderAnnotation& annotation) { - int left = -1; - int top = -1; - int right = -1; - int bottom = -1; - const auto& rectangle = annotation.rounded_rectangle().rectangle(); - if (rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), - image_width_, image_height_, &left, - &top)); - CHECK(NormalizedtoPixelCoordinates(rectangle.right(), rectangle.bottom(), - image_width_, image_height_, &right, - &bottom)); - } else { - left = static_cast(rectangle.left() * scale_factor_); - top = static_cast(rectangle.top() * scale_factor_); - right = static_cast(rectangle.right() * scale_factor_); - bottom = static_cast(rectangle.bottom() * scale_factor_); - } - - const cv::Scalar color = MediapipeColorToOpenCVColor(annotation.color()); - const int thickness = - ClampThickness(round(annotation.thickness() * scale_factor_)); - const int corner_radius = - round(annotation.rounded_rectangle().corner_radius() * scale_factor_); - const int line_type = annotation.rounded_rectangle().line_type(); - DrawRoundedRectangle(mat_image_, cv::Point(left, top), - cv::Point(right, bottom), color, thickness, line_type, - corner_radius); -} - -void AnnotationRenderer::DrawFilledRoundedRectangle( - const RenderAnnotation& annotation) { - int left = -1; - int top = -1; - int right = -1; - int bottom = -1; - const auto& rectangle = - annotation.filled_rounded_rectangle().rounded_rectangle().rectangle(); - if (rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(rectangle.left(), rectangle.top(), - image_width_, image_height_, &left, - &top)); - CHECK(NormalizedtoPixelCoordinates(rectangle.right(), rectangle.bottom(), - image_width_, image_height_, &right, - &bottom)); - } else { - left = static_cast(rectangle.left() * scale_factor_); - top = static_cast(rectangle.top() * scale_factor_); - right = static_cast(rectangle.right() * scale_factor_); - bottom = static_cast(rectangle.bottom() * scale_factor_); - } - - const cv::Scalar color = MediapipeColorToOpenCVColor(annotation.color()); - const int corner_radius = - annotation.rounded_rectangle().corner_radius() * scale_factor_; - const int line_type = annotation.rounded_rectangle().line_type(); - DrawRoundedRectangle(mat_image_, cv::Point(left, top), - cv::Point(right, bottom), color, -1, line_type, - corner_radius); -} - -void AnnotationRenderer::DrawRoundedRectangle(cv::Mat src, cv::Point top_left, - cv::Point bottom_right, - const cv::Scalar& line_color, - int thickness, int line_type, - int corner_radius) { - // Corners: - // p1 - p2 - // | | - // p4 - p3 - cv::Point p1 = top_left; - cv::Point p2 = cv::Point(bottom_right.x, top_left.y); - cv::Point p3 = bottom_right; - cv::Point p4 = cv::Point(top_left.x, bottom_right.y); - - // Draw edges of the rectangle - cv::line(src, cv::Point(p1.x + corner_radius, p1.y), - cv::Point(p2.x - corner_radius, p2.y), line_color, thickness, - line_type); - cv::line(src, cv::Point(p2.x, p2.y + corner_radius), - cv::Point(p3.x, p3.y - corner_radius), line_color, thickness, - line_type); - cv::line(src, cv::Point(p4.x + corner_radius, p4.y), - cv::Point(p3.x - corner_radius, p3.y), line_color, thickness, - line_type); - cv::line(src, cv::Point(p1.x, p1.y + corner_radius), - cv::Point(p4.x, p4.y - corner_radius), line_color, thickness, - line_type); - - // Draw arcs at corners. - cv::ellipse(src, p1 + cv::Point(corner_radius, corner_radius), - cv::Size(corner_radius, corner_radius), 180.0, 0, 90, line_color, - thickness, line_type); - cv::ellipse(src, p2 + cv::Point(-corner_radius, corner_radius), - cv::Size(corner_radius, corner_radius), 270.0, 0, 90, line_color, - thickness, line_type); - cv::ellipse(src, p3 + cv::Point(-corner_radius, -corner_radius), - cv::Size(corner_radius, corner_radius), 0.0, 0, 90, line_color, - thickness, line_type); - cv::ellipse(src, p4 + cv::Point(corner_radius, -corner_radius), - cv::Size(corner_radius, corner_radius), 90.0, 0, 90, line_color, - thickness, line_type); -} - -void AnnotationRenderer::DrawOval(const RenderAnnotation& annotation) { - int left = -1; - int top = -1; - int right = -1; - int bottom = -1; - const auto& enclosing_rectangle = annotation.oval().rectangle(); - if (enclosing_rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(enclosing_rectangle.left(), - enclosing_rectangle.top(), image_width_, - image_height_, &left, &top)); - CHECK(NormalizedtoPixelCoordinates( - enclosing_rectangle.right(), enclosing_rectangle.bottom(), image_width_, - image_height_, &right, &bottom)); - } else { - left = static_cast(enclosing_rectangle.left() * scale_factor_); - top = static_cast(enclosing_rectangle.top() * scale_factor_); - right = static_cast(enclosing_rectangle.right() * scale_factor_); - bottom = static_cast(enclosing_rectangle.bottom() * scale_factor_); - } - - cv::Point center((left + right) / 2, (top + bottom) / 2); - cv::Size size((right - left) / 2, (bottom - top) / 2); - const double rotation = enclosing_rectangle.rotation() / M_PI * 180.f; - const cv::Scalar color = MediapipeColorToOpenCVColor(annotation.color()); - const int thickness = - ClampThickness(round(annotation.thickness() * scale_factor_)); - cv::ellipse(mat_image_, center, size, rotation, 0, 360, color, thickness); -} - -void AnnotationRenderer::DrawFilledOval(const RenderAnnotation& annotation) { - int left = -1; - int top = -1; - int right = -1; - int bottom = -1; - const auto& enclosing_rectangle = annotation.filled_oval().oval().rectangle(); - if (enclosing_rectangle.normalized()) { - CHECK(NormalizedtoPixelCoordinates(enclosing_rectangle.left(), - enclosing_rectangle.top(), image_width_, - image_height_, &left, &top)); - CHECK(NormalizedtoPixelCoordinates( - enclosing_rectangle.right(), enclosing_rectangle.bottom(), image_width_, - image_height_, &right, &bottom)); - } else { - left = static_cast(enclosing_rectangle.left() * scale_factor_); - top = static_cast(enclosing_rectangle.top() * scale_factor_); - right = static_cast(enclosing_rectangle.right() * scale_factor_); - bottom = static_cast(enclosing_rectangle.bottom() * scale_factor_); - } - - cv::Point center((left + right) / 2, (top + bottom) / 2); - cv::Size size(std::max(0, (right - left) / 2), - std::max(0, (bottom - top) / 2)); - const double rotation = enclosing_rectangle.rotation() / M_PI * 180.f; - const cv::Scalar color = MediapipeColorToOpenCVColor(annotation.color()); - cv::ellipse(mat_image_, center, size, rotation, 0, 360, color, -1); -} - -void AnnotationRenderer::DrawArrow(const RenderAnnotation& annotation) { - int x_start = -1; - int y_start = -1; - int x_end = -1; - int y_end = -1; - - const auto& arrow = annotation.arrow(); - if (arrow.normalized()) { - CHECK(NormalizedtoPixelCoordinates(arrow.x_start(), arrow.y_start(), - image_width_, image_height_, &x_start, - &y_start)); - CHECK(NormalizedtoPixelCoordinates(arrow.x_end(), arrow.y_end(), - image_width_, image_height_, &x_end, - &y_end)); - } else { - x_start = static_cast(arrow.x_start() * scale_factor_); - y_start = static_cast(arrow.y_start() * scale_factor_); - x_end = static_cast(arrow.x_end() * scale_factor_); - y_end = static_cast(arrow.y_end() * scale_factor_); - } - - cv::Point arrow_start(x_start, y_start); - cv::Point arrow_end(x_end, y_end); - const cv::Scalar color = MediapipeColorToOpenCVColor(annotation.color()); - const int thickness = - ClampThickness(round(annotation.thickness() * scale_factor_)); - - // Draw the main arrow line. - cv::line(mat_image_, arrow_start, arrow_end, color, thickness); - - // Compute the arrowtip left and right vectors. - Vector2_d L_start(static_cast(x_start), static_cast(y_start)); - Vector2_d L_end(static_cast(x_end), static_cast(y_end)); - Vector2_d U = (L_end - L_start).Normalize(); - Vector2_d V = U.Ortho(); - double line_length = (L_end - L_start).Norm(); - constexpr double kArrowTipLengthProportion = 0.2; - double arrowtip_length = kArrowTipLengthProportion * line_length; - Vector2_d arrowtip_left = L_end - arrowtip_length * U + arrowtip_length * V; - Vector2_d arrowtip_right = L_end - arrowtip_length * U - arrowtip_length * V; - - // Draw the arrowtip left and right lines. - cv::Point arrowtip_left_start(static_cast(round(arrowtip_left[0])), - static_cast(round(arrowtip_left[1]))); - cv::Point arrowtip_right_start(static_cast(round(arrowtip_right[0])), - static_cast(round(arrowtip_right[1]))); - cv::line(mat_image_, arrowtip_left_start, arrow_end, color, thickness); - cv::line(mat_image_, arrowtip_right_start, arrow_end, color, thickness); -} - -void AnnotationRenderer::DrawPoint(const RenderAnnotation& annotation) { - const auto& point = annotation.point(); - int x = -1; - int y = -1; - if (point.normalized()) { - CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_, - image_height_, &x, &y)); - } else { - x = static_cast(point.x() * scale_factor_); - y = static_cast(point.y() * scale_factor_); - } - - cv::Point point_to_draw(x, y); - const cv::Scalar color = MediapipeColorToOpenCVColor(annotation.color()); - const int thickness = - ClampThickness(round(annotation.thickness() * scale_factor_)); - cv::circle(mat_image_, point_to_draw, thickness, color, -1); -} - -void AnnotationRenderer::DrawLine(const RenderAnnotation& annotation) { - int x_start = -1; - int y_start = -1; - int x_end = -1; - int y_end = -1; - - const auto& line = annotation.line(); - if (line.normalized()) { - CHECK(NormalizedtoPixelCoordinates(line.x_start(), line.y_start(), - image_width_, image_height_, &x_start, - &y_start)); - CHECK(NormalizedtoPixelCoordinates(line.x_end(), line.y_end(), image_width_, - image_height_, &x_end, &y_end)); - } else { - x_start = static_cast(line.x_start() * scale_factor_); - y_start = static_cast(line.y_start() * scale_factor_); - x_end = static_cast(line.x_end() * scale_factor_); - y_end = static_cast(line.y_end() * scale_factor_); - } - - cv::Point start(x_start, y_start); - cv::Point end(x_end, y_end); - const cv::Scalar color = MediapipeColorToOpenCVColor(annotation.color()); - const int thickness = - ClampThickness(round(annotation.thickness() * scale_factor_)); - cv::line(mat_image_, start, end, color, thickness); -} - -void AnnotationRenderer::DrawGradientLine(const RenderAnnotation& annotation) { - int x_start = -1; - int y_start = -1; - int x_end = -1; - int y_end = -1; - - const auto& line = annotation.gradient_line(); - if (line.normalized()) { - CHECK(NormalizedtoPixelCoordinates(line.x_start(), line.y_start(), - image_width_, image_height_, &x_start, - &y_start)); - CHECK(NormalizedtoPixelCoordinates(line.x_end(), line.y_end(), image_width_, - image_height_, &x_end, &y_end)); - } else { - x_start = static_cast(line.x_start() * scale_factor_); - y_start = static_cast(line.y_start() * scale_factor_); - x_end = static_cast(line.x_end() * scale_factor_); - y_end = static_cast(line.y_end() * scale_factor_); - } - - const cv::Point start(x_start, y_start); - const cv::Point end(x_end, y_end); - const int thickness = - ClampThickness(round(annotation.thickness() * scale_factor_)); - const cv::Scalar color1 = MediapipeColorToOpenCVColor(line.color1()); - const cv::Scalar color2 = MediapipeColorToOpenCVColor(line.color2()); - cv_line2(mat_image_, start, end, color1, color2, thickness); -} - -void AnnotationRenderer::DrawText(const RenderAnnotation& annotation) { - int left = -1; - int baseline = -1; - int font_size = -1; - - const auto& text = annotation.text(); - if (text.normalized()) { - CHECK(NormalizedtoPixelCoordinates(text.left(), text.baseline(), - image_width_, image_height_, &left, - &baseline)); - font_size = static_cast(round(text.font_height() * image_height_)); - } else { - left = static_cast(text.left() * scale_factor_); - baseline = static_cast(text.baseline() * scale_factor_); - font_size = static_cast(text.font_height() * scale_factor_); - } - - cv::Point origin(left, baseline); - const cv::Scalar color = MediapipeColorToOpenCVColor(annotation.color()); - const int thickness = - ClampThickness(round(annotation.thickness() * scale_factor_)); - const int font_face = text.font_face(); - - const double font_scale = ComputeFontScale(font_face, font_size, thickness); - int text_baseline = 0; - cv::Size text_size = cv::getTextSize(text.display_text(), font_face, - font_scale, thickness, &text_baseline); - - if (text.center_horizontally()) { - origin.x -= text_size.width / 2; - } - if (text.center_vertically()) { - origin.y += text_size.height / 2; - } - - cv::putText(mat_image_, text.display_text(), origin, font_face, font_scale, - color, thickness, /*lineType=*/8, - /*bottomLeftOrigin=*/flip_text_vertically_); -} - -double AnnotationRenderer::ComputeFontScale(int font_face, int font_size, - int thickness) { - double base_line; - double cap_line; - - // The details below of how to compute the font scale from font face, - // thickness, and size were inferred from the OpenCV implementation. - switch (font_face) { - case cv::FONT_HERSHEY_SIMPLEX: - case cv::FONT_HERSHEY_DUPLEX: - case cv::FONT_HERSHEY_COMPLEX: - case cv::FONT_HERSHEY_TRIPLEX: - case cv::FONT_HERSHEY_SCRIPT_SIMPLEX: - case cv::FONT_HERSHEY_SCRIPT_COMPLEX: - base_line = 9; - cap_line = 12; - break; - case cv::FONT_HERSHEY_PLAIN: - base_line = 5; - cap_line = 4; - break; - case cv::FONT_HERSHEY_COMPLEX_SMALL: - base_line = 6; - cap_line = 7; - break; - default: - return -1; - } - - const double thick = static_cast(thickness + 1); - return (static_cast(font_size) - (thick / 2.0F)) / - (cap_line + base_line); -} - -} // namespace mediapipe diff --git a/mediapipe/graphs/beauty/calculators/annotation_renderer.h b/mediapipe/graphs/beauty/calculators/annotation_renderer.h deleted file mode 100644 index b2c4ecbea..000000000 --- a/mediapipe/graphs/beauty/calculators/annotation_renderer.h +++ /dev/null @@ -1,158 +0,0 @@ -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_UTIL_ANNOTATION_RENDERER_H_ -#define MEDIAPIPE_UTIL_ANNOTATION_RENDERER_H_ - -#include - -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/opencv_highgui_inc.h" -#include "mediapipe/util/render_data.pb.h" - -namespace mediapipe { - -// The renderer library for rendering data on images. -// -// Example usage: -// -// AnnotationRenderer renderer; -// -// std::unique_ptr mat_image(new cv::Mat(kImageHeight, kImageWidth, -// CV_8UC3)); -// -// renderer.AdoptImage(mat_image.get()); -// -// RenderData render_data_0; -// -// -// renderer.RenderDataOnImage(render_data_0); -// -// RenderData render_data_1; -// -// -// renderer.RenderDataOnImage(render_data_1); -// -// UseRenderedImage(mat_image.get()); -class AnnotationRenderer { - public: - explicit AnnotationRenderer() {} - - explicit AnnotationRenderer(const cv::Mat& mat_image) - : image_width_(mat_image.cols), - image_height_(mat_image.rows), - mat_image_(mat_image.clone()) {} - - // Renders the image with the input render data. - void RenderDataOnImage(const RenderData& render_data); - - // Resets the renderer with a new image. Does not own input_image. input_image - // must not be modified by caller during rendering. - void AdoptImage(cv::Mat* input_image); - - // Gets image dimensions. - int GetImageWidth() const; - int GetImageHeight() const; - - // Sets whether text should be rendered upside down. This is default to false - // and text is rendered assuming the underlying image has its origin at the - // top-left corner. Set it to true if the image origin is at the bottom-left - // corner. - void SetFlipTextVertically(bool flip); - - // For GPU rendering optimization in AnnotationOverlayCalculator. - // Scale all incoming coordinates,sizes,thickness,etc. by this amount. - // Should be in the range (0-1]. - // See 'gpu_scale_factor' in annotation_overlay_calculator.proto - void SetScaleFactor(float scale_factor); - float GetScaleFactor() { return scale_factor_; } - - private: - // Draws a rectangle on the image as described in the annotation. - void DrawRectangle(const RenderAnnotation& annotation); - - // Draws a filled rectangle on the image as described in the annotation. - void DrawFilledRectangle(const RenderAnnotation& annotation); - - // Draws an oval on the image as described in the annotation. - void DrawOval(const RenderAnnotation& annotation); - - // Draws a filled oval on the image as described in the annotation. - void DrawFilledOval(const RenderAnnotation& annotation); - - // Draws an arrow on the image as described in the annotation. - void DrawArrow(const RenderAnnotation& annotation); - - // Draws a point on the image as described in the annotation. - void DrawPoint(const RenderAnnotation& annotation); - - // Draws lipstick on the face. - void DrawLipstick(const RenderData& render_data); - - // Whitens teeth. - void WhitenTeeth(const RenderData& render_data); - - // Draws a line segment on the image as described in the annotation. - void DrawLine(const RenderAnnotation& annotation); - - // Draws a 2-tone line segment on the image as described in the annotation. - void DrawGradientLine(const RenderAnnotation& annotation); - - // Draws a text on the image as described in the annotation. - void DrawText(const RenderAnnotation& annotation); - - // Draws a rounded rectangle on the image as described in the annotation. - void DrawRoundedRectangle(const RenderAnnotation& annotation); - - // Draws a filled rounded rectangle on the image as described in the - // annotation. - void DrawFilledRoundedRectangle(const RenderAnnotation& annotation); - - // Helper function for drawing a rectangle with rounded corners. The - // parameters are the same as in the OpenCV function rectangle(). - // corner_radius: A positive int value defining the radius of the round - // corners. - void DrawRoundedRectangle(cv::Mat src, cv::Point top_left, - cv::Point bottom_right, - const cv::Scalar& line_color, int thickness = 1, - int line_type = 8, int corner_radius = 0); - - // Computes the font scale from font_face, size and thickness. - double ComputeFontScale(int font_face, int font_size, int thickness); - - cv::Mat FormFacePartMask(std::vector orderList, const RenderData &render_data); - - cv::Mat predict_forehead_mask(const RenderData &render_data, double face_box_min_y); - - void smooth_face(const RenderData &render_data); - - std::tuple GetFaceBox(const RenderData &render_data); - - // Width and Height of the image (in pixels). - int image_width_ = -1; - int image_height_ = -1; - - // The image for rendering. - cv::Mat mat_image_; - - // See SetFlipTextVertically(bool). - bool flip_text_vertically_ = false; - - // See SetScaleFactor(float) - float scale_factor_ = 1.0; -}; -} // namespace mediapipe - -#endif // MEDIAPIPE_UTIL_ANNOTATION_RENDERER_H_ diff --git a/mediapipe/graphs/beauty/face_mesh_desktop.pbtxt b/mediapipe/graphs/beauty/face_mesh_desktop.pbtxt deleted file mode 100644 index 215791a36..000000000 --- a/mediapipe/graphs/beauty/face_mesh_desktop.pbtxt +++ /dev/null @@ -1,70 +0,0 @@ -# MediaPipe graph that performs face mesh on desktop with TensorFlow Lite -# on CPU. - -# Path to the input video file. (string) -input_side_packet: "input_video_path" -# Path to the output video file. (string) -input_side_packet: "output_video_path" - -# max_queue_size limits the number of packets enqueued on any input stream -# by throttling inputs to the graph. This makes the graph only process one -# frame per time. -max_queue_size: 1 - -# Decodes an input video file into images and a video header. -node { - calculator: "OpenCvVideoDecoderCalculator" - input_side_packet: "INPUT_FILE_PATH:input_video_path" - output_stream: "VIDEO:input_video" - output_stream: "VIDEO_PRESTREAM:input_video_header" -} - -# Defines side packets for further use in the graph. -node { - calculator: "ConstantSidePacketCalculator" - output_side_packet: "PACKET:0:num_faces" - output_side_packet: "PACKET:1:with_attention" - node_options: { - [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { - packet { int_value: 1 } - packet { bool_value: true } - } - } -} - -# Subgraph that detects faces and corresponding landmarks. -node { - calculator: "FaceLandmarkFrontCpu" - input_stream: "IMAGE:input_video" - input_side_packet: "NUM_FACES:num_faces" - input_side_packet: "WITH_ATTENTION:with_attention" - output_stream: "LANDMARKS:multi_face_landmarks" - output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" - output_stream: "DETECTIONS:face_detections" - output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" -} - -# Subgraph that renders face-landmark annotation onto the input video. -node { - calculator: "FaceRendererCpu" - input_stream: "IMAGE:input_video" - input_stream: "LANDMARKS:multi_face_landmarks" - input_stream: "NORM_RECTS:face_rects_from_landmarks" - input_stream: "DETECTIONS:face_detections" - output_stream: "IMAGE:output_video" -} - -# Encodes the annotated images into a video file, adopting properties specified -# in the input video header, e.g., video framerate. -node { - calculator: "OpenCvVideoEncoderCalculator" - input_stream: "VIDEO:output_video" - input_stream: "VIDEO_PRESTREAM:input_video_header" - input_side_packet: "OUTPUT_FILE_PATH:output_video_path" - node_options: { - [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: { - codec: "avc1" - video_format: "mp4" - } - } -} diff --git a/mediapipe/graphs/beauty/subgraphs/BUILD b/mediapipe/graphs/beauty/subgraphs/BUILD index 8001e0fc7..8fd1014ae 100644 --- a/mediapipe/graphs/beauty/subgraphs/BUILD +++ b/mediapipe/graphs/beauty/subgraphs/BUILD @@ -30,27 +30,9 @@ cc_library( "//mediapipe/calculators/beauty:smooth_face_calculator", "//mediapipe/calculators/beauty:draw_lipstick_calculator", "//mediapipe/calculators/beauty:whiten_teeth_calculator", - "//mediapipe/calculators/util:detections_to_render_data_calculator", "//mediapipe/calculators/util:landmarks_to_render_data_calculator", "//mediapipe/calculators/landmarks:landmarks_to_mask_calculator", - ], -) - -mediapipe_simple_subgraph( - name = "face_renderer_gpu", - graph = "face_renderer_gpu.pbtxt", - register_as = "FaceRendererGpu", - deps = [ - ":renderer_calculators", - ], -) - -mediapipe_simple_subgraph( - name = "face_renderer_gpu_over", - graph = "face_renderer_gpu_over.pbtxt", - register_as = "FaceRendererGpuOver", - deps = [ - ":renderer_calculators", + "//mediapipe/graphs/face_mesh/calculators:face_landmarks_to_render_data_calculator", ], ) @@ -62,3 +44,12 @@ mediapipe_simple_subgraph( ":renderer_calculators", ], ) + +mediapipe_simple_subgraph( + name = "face_renderer_cpu_single", + graph = "face_renderer_cpu_single.pbtxt", + register_as = "FaceRendererCpuSingle", + deps = [ + ":renderer_calculators", + ], +) diff --git a/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt b/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt index f910775cc..f32f4ac52 100644 --- a/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt +++ b/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu.pbtxt @@ -11,12 +11,6 @@ input_stream: "LANDMARKS:multi_face_landmarks" # CPU image with rendered data. (ImageFrame) output_stream: "IMAGE:output_image" -node { - calculator: "ImagePropertiesCalculator" - input_stream: "IMAGE:input_image" - output_stream: "SIZE:image_size" -} - # Outputs each element of multi_face_landmarks at a fake timestamp for the rest # of the graph to process. At the end of the loop, outputs the BATCH_END # timestamp for downstream calculators to inform them that all elements in the diff --git a/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu_single.pbtxt b/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu_single.pbtxt new file mode 100644 index 000000000..7708365ab --- /dev/null +++ b/mediapipe/graphs/beauty/subgraphs/face_renderer_cpu_single.pbtxt @@ -0,0 +1,59 @@ +# MediaPipe face mesh rendering subgraph. + +type: "FaceRendererCpuSingle" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:input_image" +# Collection of detected/predicted faces, each represented as a list of +# landmarks. (std::vector) +input_stream: "LANDMARKS:multi_face_landmarks" + +# CPU image with rendered data. (ImageFrame) +output_stream: "IMAGE:output_image" + +# Outputs each element of multi_face_landmarks at a fake timestamp for the rest +# of the graph to process. At the end of the loop, outputs the BATCH_END +# timestamp for downstream calculators to inform them that all elements in the +# vector have been processed. +node { + calculator: "BeginLoopNormalizedLandmarkListVectorCalculator" + input_stream: "ITERABLE:multi_face_landmarks" + output_stream: "ITEM:face_landmarks" + output_stream: "BATCH_END:landmark_timestamp" +} + +# Converts landmarks to drawing primitives for annotation overlay. +node { + calculator: "FaceLandmarksToRenderDataCalculator" + input_stream: "NORM_LANDMARKS:face_landmarks" + output_stream: "RENDER_DATA:landmarks_render_data" + node_options: { + [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] { + landmark_color { r: 255 g: 0 b: 0 } + connection_color { r: 0 g: 255 b: 0 } + thickness: 2 + visualize_landmark_depth: false + } + } +} + +# Collects a RenderData object for each hand into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END +# timestamp. +node { + calculator: "EndLoopRenderDataCalculator" + input_stream: "ITEM:landmarks_render_data" + input_stream: "BATCH_END:landmark_timestamp" + output_stream: "ITERABLE:multi_face_landmarks_render_data" +} + +# Draws annotations and overlays them on top of the input images. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "IMAGE:input_image" + input_stream: "VECTOR:0:multi_face_landmarks_render_data" + output_stream: "IMAGE:output_image" +} + + + diff --git a/mediapipe/graphs/deformation/BUILD b/mediapipe/graphs/deformation/BUILD new file mode 100644 index 000000000..65dcf9572 --- /dev/null +++ b/mediapipe/graphs/deformation/BUILD @@ -0,0 +1,54 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load( + "//mediapipe/framework/tool:mediapipe_graph.bzl", + "mediapipe_binary_graph", +) + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "mobile_calculators", + deps = [ + "//mediapipe/graphs/deformation/calculators:face_processor_calculator", + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/calculators/image:image_transformation_calculator", + "//mediapipe/modules/face_landmark:face_landmark_front_gpu", + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/gpu:gpu_buffer_to_image_frame_calculator", + "//mediapipe/gpu:image_frame_to_gpu_buffer_calculator", + ], +) + +cc_library( + name = "desktop_calculators", + deps = [ + "//mediapipe/graphs/deformation/calculators:face_processor_calculator", + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/calculators/image:image_transformation_calculator", + "//mediapipe/modules/face_landmark:face_landmark_front_cpu", + "//mediapipe/calculators/core:constant_side_packet_calculator", + ], +) + +mediapipe_binary_graph( + name = "mobile_gpu_binary_graph", + graph = "deformation_mobile.pbtxt", + output_name = "mobile_gpu.binarypb", + deps = [":mobile_calculators"], +) + diff --git a/mediapipe/graphs/deformation/README.md b/mediapipe/graphs/deformation/README.md new file mode 100644 index 000000000..7a79faddf --- /dev/null +++ b/mediapipe/graphs/deformation/README.md @@ -0,0 +1,37 @@ +# Deformation + +this graph performs face processing + +## Getting started + +Clone branch. + +1. Deformation-CPU + +Build with: +``` +bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/deformation:deformation_cpu +``` +Run with (using your camera): +``` +bazel-bin/mediapipe/examples/desktop/deformation/deformation_cpu +--calculator_graph_config_file=mediapipe/graphs/deformation/deformation_cpu.pbtxt +``` +Run with (using video): +``` +bazel-bin/mediapipe/examples/desktop//deformation/deformation_cpu +--calculator_graph_config_file=mediapipe/graphs/deformation/deformation_cpu.pbtxt +--input_video_path=/path/video.mp4 +--output_video_path=/path/outvideo.mp4 +``` + +2. Mobile (Android) + +Build with: +``` +bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/deformation:deformationgpu +``` +Install with: +``` +adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/deformation/deformationgpu.apk +``` diff --git a/mediapipe/graphs/deformation/calculators/BUILD b/mediapipe/graphs/deformation/calculators/BUILD new file mode 100644 index 000000000..2e8ec4ba0 --- /dev/null +++ b/mediapipe/graphs/deformation/calculators/BUILD @@ -0,0 +1,51 @@ +# Copyright 2020 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "face_processor_calculator", + srcs = ["face_processor_calculator.cc"], + hdrs = ["Tensor.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:video_stream_header", + "//mediapipe/framework/formats:location_data_cc_proto", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + "//mediapipe/framework/formats:image_format_cc_proto", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_highgui", + "//mediapipe/framework/port:vector", + "//mediapipe/framework/deps:file_path", + "//mediapipe/framework/formats/object_detection:anchor_cc_proto", + "//mediapipe/framework/port:file_helpers", + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/util:resource_util", + "@com_google_absl//absl/flags:flag", + ], + alwayslink = 1, +) + + + diff --git a/mediapipe/graphs/deformation/calculators/Tensor.h b/mediapipe/graphs/deformation/calculators/Tensor.h new file mode 100644 index 000000000..be5da18e3 --- /dev/null +++ b/mediapipe/graphs/deformation/calculators/Tensor.h @@ -0,0 +1,224 @@ +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace cv; + +template +class Tensor +{ +private: + vector dims; + Mat M; + int dtype; + + void _set_dtype() { + if (typeid(T).name() == typeid(double).name()) + dtype = CV_64F; + else if (typeid(T).name() == typeid(int).name()) + dtype = CV_32S; + else + dtype = CV_64F; + } + +public: + Tensor() { + _set_dtype(); + dims = vector{ 3, 3 }; + M = Mat(dims, dtype); + } + + Tensor(vector _dims) { + _set_dtype(); + dims = _dims; + M = Mat(dims, dtype); + } + + Tensor(Mat _M) { + dims = vector(_M.size.p, _M.size.p + _M.size.dims()); + M = _M; + dtype = _M.type(); + } + + Tensor(const Tensor& _m) { + dims = _m.dims; + M = _m.M; + dtype = _m.dtype; + } + + Tensor(T** arr, int _n, int _m) { + _set_dtype(); + dims = { _n, _m }; + M = Mat(dims, dtype); + + for (int i = 0; i < _n; i++) + for (int j = 0; j < _m; j++) + M.at(Vec(i, j)) = arr[i][j]; + } + + vector get_1d_data() { + return vector(M.ptr(0), M.ptr(0) + dims[1]); + } + + T at(vector _indexes) { + return M.at(_indexes.data()); + } + + Tensor index(int i1) { + return Tensor(M.row(i1)); + } + + + Tensor index(vector _indexes) { + vector _dims; + _dims = dims; + _dims[0] = 0; + Mat _M = Mat(_dims, dtype); + + for (int i = 0; i < _indexes.size(); ++i) + _M.push_back(M.row(_indexes[i])); + + return Tensor(_M); + } + + Tensor index(vector> _indexes) { + vector _dims; + _dims = dims; + _dims[0] = 0; + Mat _M = Mat(_dims, dtype); + + Mat tmp_M, _tmp_M; + for (int i = 0; i < _indexes.size(); ++i) { + _M.push_back(this->index(_indexes[i]).M); + } + + return Tensor(_M.reshape(1, _indexes.size())); + } + + Tensor index(Range r1, int index2) { + if (r1.end < 0) { + r1.end = M.rows + r1.end + 1; + } + return Tensor(M(r1, Range::all()).col(index2)); + } + + Tensor index(Range r1, Range r2) { + if (r1.end < 0) { + r1.end = M.rows + r1.end + 1; + } + if (r2.end < 0) { + r2.end = M.cols + r2.end + 1; + } + if (r1.start < 0 && r1.start > -2000000000) { + r1.start = M.rows + r1.start; + } + if (r2.start < 0 && r2.start > -2000000000) { + r2.start = M.cols + r2.start; + } + return Tensor(M(r1, r2)); + } + + Tensor concat(Tensor t, int dim) { + Mat dst; + if (dim == 0) { + vconcat(M, t.M, dst); + } + if (dim == 1) { + hconcat(M, t.M, dst); + } + return Tensor(dst); + } + + Tensor matmul(Tensor t) { + return Tensor(M * t.M); + } + + Tensor inverse() { + return Tensor(M.inv()); + } + + Tensor transpose() { + return Tensor(M.t()); + } + + T norm() { + int ndims = dims.size(); + int size = 1; + for (int i = 0; i < ndims; i++) { + size = size * dims[i]; + } + + vector _index(ndims, 0); + T _norm = pow(M.at(_index.data()), 2); + + for (int i = 0; i < size - 1; i++) { + _index[ndims - 1] += 1; + for (int j = ndims-1; j >= 0; j--) { + if (_index[j] >= dims[j]) { + _index[j] = 0; + _index[j - 1] += 1; + } + } + _norm += pow(M.at(_index.data()), 2); + } + + return sqrt(_norm); + } + + + friend ostream& operator<<(ostream& os, const Tensor& _M){ + os << _M.M; + return os; + } + + friend const Tensor operator-(const Tensor& t) { + return Tensor(-t.M); + } + + friend const Tensor operator/(const Tensor& t1, const Tensor& t2) { + return Tensor(t1.M.mul(1 / t2.M)); + } + + friend const Tensor operator*(const Tensor& t1, const Tensor& t2) { + return Tensor(t1.M.mul(t2.M)); + } + + friend const Tensor operator+(const Tensor& t1, const Tensor& t2) { + return Tensor(t1.M + t2.M); + } + + friend const Tensor operator-(const Tensor& t1, const Tensor& t2) { + return Tensor(t1.M - t2.M); + } + + friend const Tensor operator/(const Tensor& t1, const T& val) { + return Tensor(t1.M / val); + } + + friend const Tensor operator*(const Tensor& t1, const T& val) { + return Tensor(t1.M * val); + } + + friend const Tensor operator-(const Tensor& t1, const T& val) { + return Tensor(t1.M - val); + } + + friend const Tensor operator*(const T& val, const Tensor& t1) { + return Tensor(t1.M * val); + } + + static vector sort_indexes(const vector& v) { + vector idx(v.size()); + iota(idx.begin(), idx.end(), 0); + stable_sort(idx.begin(), idx.end(), [&v](int i1, int i2) {return v[i1] < v[i2]; }); + return idx; + } + + ~Tensor() + { + } +}; diff --git a/mediapipe/graphs/deformation/calculators/face_processor_calculator.cc b/mediapipe/graphs/deformation/calculators/face_processor_calculator.cc new file mode 100644 index 000000000..57768dbf1 --- /dev/null +++ b/mediapipe/graphs/deformation/calculators/face_processor_calculator.cc @@ -0,0 +1,593 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include +#include +//#include + +#include +#include "Tensor.h" +#include "absl/strings/str_cat.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_options.pb.h" +#include "mediapipe/framework/formats/image_format.pb.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/video_stream_header.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/status.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/vector.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/util/resource_util.h" + +namespace mediapipe +{ + namespace + { + + constexpr char kImageFrameTag[] = "IMAGE"; + constexpr char kVectorTag[] = "VECTOR"; + constexpr char kLandmarksTag[] = "LANDMARKS"; + constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS"; + + tuple _normalized_to_pixel_coordinates(float normalized_x, + float normalized_y, int image_width, int image_height) + { + // Converts normalized value pair to pixel coordinates + int x_px = min(floor(normalized_x * image_width), image_width - 1); + int y_px = min(floor(normalized_y * image_height), image_height - 1); + + return {x_px, y_px}; + }; + + inline bool HasImageTag(mediapipe::CalculatorContext *cc) { return false; } + + bool NormalizedtoPixelCoordinates(double normalized_x, double normalized_y, double normalized_z, + int image_width, int image_height, double *x_px, + double *y_px, double *z_px) + { + CHECK(x_px != nullptr); + CHECK(y_px != nullptr); + CHECK_GT(image_width, 0); + CHECK_GT(image_height, 0); + + if (normalized_x < 0 || normalized_x > 1.0 || normalized_y < 0 || + normalized_y > 1.0 || normalized_z < 0 || + normalized_z > 1.0) + { + VLOG(1) << "Normalized coordinates must be between 0.0 and 1.0"; + } + + *x_px = static_cast(normalized_x) * image_width; + *y_px = static_cast(normalized_y) * image_height; + *z_px = static_cast(normalized_z) * image_width; + // 2280 + + return true; + } + + template + bool IsLandmarkVisibleAndPresent(const LandmarkType &landmark, + bool utilize_visibility, + float visibility_threshold, + bool utilize_presence, + float presence_threshold) + { + if (utilize_visibility && landmark.has_visibility() && + landmark.visibility() < visibility_threshold) + { + return false; + } + if (utilize_presence && landmark.has_presence() && + landmark.presence() < presence_threshold) + { + return false; + } + return true; + } + + } // namespace + + class FaceProcessorCalculator : public CalculatorBase + { + public: + FaceProcessorCalculator() = default; + ~FaceProcessorCalculator() override = default; + + static absl::Status GetContract(CalculatorContract *cc); + + // From Calculator. + absl::Status Open(CalculatorContext *cc) override; + absl::Status Process(CalculatorContext *cc) override; + absl::Status Close(CalculatorContext *cc) override; + + private: + absl::Status CreateRenderTargetCpu(CalculatorContext *cc, + unique_ptr &image_mat, + ImageFormat::Format *target_format); + + absl::Status RenderToCpu( + CalculatorContext *cc, const ImageFormat::Format &target_format, + uchar *data_image, unique_ptr &image_mat); + + absl::Status SetData(CalculatorContext *cc); + + absl::Status ProcessImage(CalculatorContext *cc, + ImageFormat::Format &target_format); + + static absl::StatusOr ReadContentBlobFromFile( + const string &unresolved_path) + { + ASSIGN_OR_RETURN(string resolved_path, + mediapipe::PathToResourceAsFile(unresolved_path), + _ << "Failed to resolve path! Path = " << unresolved_path); + + string content_blob; + MP_RETURN_IF_ERROR( + mediapipe::GetResourceContents(resolved_path, &content_blob)) + << "Failed to read content blob! Resolved path = " << resolved_path; + + return content_blob; + } + + // Indicates if image frame is available as input. + bool image_frame_available_ = false; + + unique_ptr image_mat; + vector index_names; + map> indexes; + + map> masks; + vector> _trianglesIndexes; + Tensor __facePts; + + int image_width_; + int image_height_; + + Mat mat_image_; + }; + + absl::Status FaceProcessorCalculator::GetContract(CalculatorContract *cc) + { + CHECK_GE(cc->Inputs().NumEntries(), 1); + + if (cc->Inputs().HasTag(kImageFrameTag)) + { + cc->Inputs().Tag(kImageFrameTag).Set(); + CHECK(cc->Outputs().HasTag(kImageFrameTag)); + } + + RET_CHECK(cc->Inputs().HasTag(kLandmarksTag) || + cc->Inputs().HasTag(kNormLandmarksTag)) + << "None of the input streams are provided."; + RET_CHECK(!(cc->Inputs().HasTag(kLandmarksTag) && + cc->Inputs().HasTag(kNormLandmarksTag))) + << "Can only one type of landmark can be taken. Either absolute or " + "normalized landmarks."; + + if (cc->Inputs().HasTag(kLandmarksTag)) + { + cc->Inputs().Tag(kLandmarksTag).Set>(); + } + if (cc->Inputs().HasTag(kNormLandmarksTag)) + { + cc->Inputs().Tag(kNormLandmarksTag).Set>(); + } + + if (cc->Outputs().HasTag(kImageFrameTag)) + { + cc->Outputs().Tag(kImageFrameTag).Set(); + } + + return absl::OkStatus(); + } + + absl::Status FaceProcessorCalculator::Open(CalculatorContext *cc) + { + cc->SetOffset(TimestampDiff(0)); + + if (cc->Inputs().HasTag(kImageFrameTag) || HasImageTag(cc)) + { + image_frame_available_ = true; + } + + // Set the output header based on the input header (if present). + const char *tag = kImageFrameTag; + if (image_frame_available_ && !cc->Inputs().Tag(tag).Header().IsEmpty()) + { + const auto &input_header = + cc->Inputs().Tag(tag).Header().Get(); + auto *output_video_header = new VideoHeader(input_header); + cc->Outputs().Tag(tag).SetHeader(Adopt(output_video_header)); + } + + return absl::OkStatus(); + } + + absl::Status FaceProcessorCalculator::Process(CalculatorContext *cc) + { + if (cc->Inputs().HasTag(kImageFrameTag) && + cc->Inputs().Tag(kImageFrameTag).IsEmpty()) + { + return absl::OkStatus(); + } + + // Initialize render target, drawn with OpenCV. + ImageFormat::Format target_format; + + MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format)); + + mat_image_ = *image_mat.get(); + image_width_ = image_mat->cols; + image_height_ = image_mat->rows; + + MP_RETURN_IF_ERROR(SetData(cc)); + + if (cc->Inputs().HasTag(kNormLandmarksTag) && + !cc->Inputs().Tag(kNormLandmarksTag).IsEmpty()) + { + MP_RETURN_IF_ERROR(ProcessImage(cc, target_format)); + } + + uchar *image_mat_ptr = image_mat->data; + MP_RETURN_IF_ERROR(RenderToCpu(cc, target_format, image_mat_ptr, image_mat)); + + return absl::OkStatus(); + } + + absl::Status FaceProcessorCalculator::Close(CalculatorContext *cc) + { + return absl::OkStatus(); + } + + absl::Status FaceProcessorCalculator::RenderToCpu( + CalculatorContext *cc, const ImageFormat::Format &target_format, + uchar *data_image, unique_ptr &image_mat) + { + + auto output_frame = absl::make_unique( + target_format, mat_image_.cols, mat_image_.rows); + + output_frame->CopyPixelData(target_format, mat_image_.cols, mat_image_.rows, data_image, + ImageFrame::kDefaultAlignmentBoundary); + + if (cc->Outputs().HasTag(kImageFrameTag)) + { + cc->Outputs() + .Tag(kImageFrameTag) + .Add(output_frame.release(), cc->InputTimestamp()); + } + + return absl::OkStatus(); + } + + absl::Status FaceProcessorCalculator::CreateRenderTargetCpu( + CalculatorContext *cc, unique_ptr &image_mat, + ImageFormat::Format *target_format) + { + if (image_frame_available_) + { + const auto &input_frame = + cc->Inputs().Tag(kImageFrameTag).Get(); + + int target_mat_type; + switch (input_frame.Format()) + { + case ImageFormat::SRGBA: + *target_format = ImageFormat::SRGBA; + target_mat_type = CV_8UC4; + break; + case ImageFormat::SRGB: + *target_format = ImageFormat::SRGB; + target_mat_type = CV_8UC3; + break; + case ImageFormat::GRAY8: + *target_format = ImageFormat::SRGB; + target_mat_type = CV_8UC3; + break; + default: + return absl::UnknownError("Unexpected image frame format."); + break; + } + + image_mat = absl::make_unique( + input_frame.Height(), input_frame.Width(), target_mat_type); + + auto input_mat = formats::MatView(&input_frame); + + if (input_frame.Format() == ImageFormat::GRAY8) + { + Mat rgb_mat; + cvtColor(input_mat, rgb_mat, CV_GRAY2RGBA); + rgb_mat.copyTo(*image_mat); + } + else + { + input_mat.copyTo(*image_mat); + } + } + else + { + image_mat = absl::make_unique( + 1920, 1280, CV_8UC4, + Scalar::all(255.0)); + *target_format = ImageFormat::SRGBA; + } + + return absl::OkStatus(); + } + + absl::Status FaceProcessorCalculator::SetData(CalculatorContext *cc) + { + masks.clear(); + _trianglesIndexes.clear(); + + string filename = "mediapipe/graphs/deformation/config/triangles.txt"; + string content_blob; + ASSIGN_OR_RETURN(content_blob, + ReadContentBlobFromFile(filename), + _ << "Failed to read texture blob from file!"); + + istringstream stream(content_blob); + double points[854][3]; + vector tmp; + for (int i = 0; i < 854; ++i) + { + string line; + tmp = {}; + for (int j = 0; j < 3; ++j) + { + stream >> points[i][j]; + tmp.push_back((int)points[i][j]); + } + _trianglesIndexes.push_back(tmp); + } + + filename = "./mediapipe/graphs/deformation/config/index_names.txt"; + ASSIGN_OR_RETURN(content_blob, + ReadContentBlobFromFile(filename), + _ << "Failed to read texture blob from file!"); + istringstream stream2(content_blob); + + string line; + vector idxs; + while (getline(stream2, line)) + { + index_names.push_back(line); + } + stream2.clear(); + + for (int i = 0; i < index_names.size(); i++) + { + filename = "./mediapipe/graphs/deformation/config/" + index_names[i] + ".txt"; + + ASSIGN_OR_RETURN(content_blob, + ReadContentBlobFromFile(filename), + _ << "Failed to read texture blob from file!"); + stream2.str(content_blob); + + while (getline(stream2, line)) + { + idxs.push_back(stoi(line)); + } + indexes[index_names[i]] = idxs; + + idxs = {}; + stream2.clear(); + } + + double **zero_arr; + for (int i = 0; i < index_names.size(); i++) + { + zero_arr = (double **)new double *[478]; + for (int j = 0; j < 478; j++) + { + zero_arr[j] = (double *)new double[1]; + zero_arr[j][0] = 0.0; + } + for (int j = 0; j < indexes[index_names[i]].size(); j++) + { + zero_arr[indexes[index_names[i]][j]][0] = 1; + } + masks[index_names[i]] = Tensor(zero_arr, 478, 1); + } + + return absl::OkStatus(); + } + + absl::Status FaceProcessorCalculator::ProcessImage(CalculatorContext *cc, + ImageFormat::Format &target_format) + { + double alfaNose = 0.7; + double alfaLips = 0.2; + double alfaCheekbones = 0.2; + + if (cc->Inputs().HasTag(kNormLandmarksTag)) + { + const vector &landmarks = + cc->Inputs().Tag(kNormLandmarksTag).Get>(); + + int n = 478; + int m = 3; + + double **_points = (double **)new double *[n]; + for (int i = 0; i < n; i++) + _points[i] = (double *)new double[m]; + + for (int i = 0; i < landmarks[0].landmark_size(); ++i) + { + const NormalizedLandmark &landmark = landmarks[0].landmark(i); + + if (!IsLandmarkVisibleAndPresent( + landmark, false, + 0.0, false, + 0.0)) + { + continue; + } + + const auto &point = landmark; + + double x = -1; + double y = -1; + double z = -1; + + CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), point.z(), image_width_, + image_height_, &x, &y, &z)); + _points[i][0] = x; + _points[i][1] = y; + _points[i][2] = z; + } + __facePts = Tensor(_points, n, m); + } + + cvtColor(mat_image_, mat_image_, COLOR_BGRA2RGB); + Mat clone_image = mat_image_.clone(); + + Tensor ___facePts = __facePts - 0; + + Tensor _X = __facePts.index(indexes["mediumNoseIndexes"]).index(Range::all(), Range(0, 1)); + Tensor __YZ = __facePts.index(indexes["mediumNoseIndexes"]).index(Range::all(), Range(1, -1)); + Tensor ___YZ = __YZ.concat(Tensor(Mat::ones(9, 1, CV_64F)), 1); + Tensor _b = ___YZ.transpose().matmul(___YZ).inverse().matmul(___YZ.transpose()).matmul(_X); + Tensor _ort = Tensor(Mat::ones(1, 1, CV_64F)).concat(-_b.index(Range(0, 2), Range::all()), 0); + double _D = _b.at({2, 0}) / _ort.norm(); + _ort = _ort / _ort.norm(); + + Tensor _mask; + Tensor _dsts; + vector _indexes; + _indexes = {"cheekbonesIndexes", "noseAllIndexes", "additionalNoseIndexes1", "additionalNoseIndexes2", "additionalNoseIndexes3"}; + + vector coeffs; + coeffs = {alfaCheekbones * 0.2, alfaNose * 0.2, alfaNose * 0.1, alfaNose * 0.05, alfaNose * 0.025}; + + _mask = masks["faceOvalIndexes"]; + _dsts = _mask * (___facePts.matmul(_ort) - _D); + ___facePts = ___facePts + _dsts.matmul(_ort.transpose()) * 0.05; + __facePts = __facePts + _dsts.matmul(_ort.transpose()) * 0.05; + + for (int i = 0; i < 5; i++) + { + _mask = masks[_indexes[i]]; + _dsts = _mask * (___facePts.matmul(_ort) - _D); + ___facePts = ___facePts - coeffs[i] * _dsts.matmul(_ort.transpose()); + } + + _D = -1; + Tensor _lipsSupprotPoint = (___facePts.index(11) + ___facePts.index(16)) / 2; + Tensor _ABC = _lipsSupprotPoint.concat(___facePts.index(291), 0).concat(___facePts.index(61), 0).inverse().matmul(Tensor(Mat::ones(3, 1, CV_64F))) * _D; + _D = _D / _ABC.norm(); + _ort = _ABC / _ABC.norm(); + + _indexes = {"upperLipCnt", "lowerLipCnt", "widerUpperLipPts1", "widerLowerLipPts1"}; + coeffs = {alfaLips, alfaLips * 0.5, alfaLips * 0.5, alfaLips * 0.25}; + + for (int i = 0; i < 4; i++) + { + _mask = masks[_indexes[i]]; + _dsts = _mask * (___facePts.matmul(_ort) - _D); + ___facePts = ___facePts + coeffs[i] * _dsts.matmul(_ort.transpose()); + } + + Tensor tmp_order = ___facePts.index(_trianglesIndexes); + tmp_order = -tmp_order.index(Range::all(), 2) - tmp_order.index(Range::all(), 5) - tmp_order.index(Range::all(), 8); + tmp_order = tmp_order.transpose(); + vector __order = tmp_order.get_1d_data(); + vector _order = tmp_order.sort_indexes(__order); + + Tensor _src = __facePts.index(_trianglesIndexes).index(_order); + Tensor _dst = ___facePts.index(_trianglesIndexes).index(_order); + + Mat outImage = mat_image_.clone(); + + for (int i = 0; i < 854; ++i) + { + if (i == 246) + { + int pointer = 0; + } + + Tensor __t1 = _src.index(vector{i}); + Tensor __t2 = _dst.index(vector{i}); + + vector t1; + vector t2; + + for (int i = 0; i < 3; ++i) + { + t1.push_back(Point( + (int)(__t1.at(vector{0, 3 * i})), + (int)(__t1.at(vector{0, 3 * i + 1})))); + t2.push_back(Point( + (int)(__t2.at(vector{0, 3 * i})), + (int)(__t2.at(vector{0, 3 * i + 1})))); + } + + Rect r1 = boundingRect(t1); + Rect r2 = boundingRect(t2); + Point2f srcTri[3]; + Point2f dstTri[3]; + vector t1Rect; + vector t2Rect; + + for (int i = 0; i < 3; ++i) + { + srcTri[i] = Point2f(t1[i].x - r1.x, t1[i].y - r1.y); + dstTri[i] = Point2f(t2[i].x - r2.x, t2[i].y - r2.y); + t1Rect.push_back(Point(t1[i].x - r1.x, t1[i].y - r1.y)); + t2Rect.push_back(Point(t2[i].x - r2.x, t2[i].y - r2.y)); + } + + Mat _dst; + Mat mask = Mat::zeros(r2.height, r2.width, CV_8U); + fillConvexPoly(mask, t2Rect, Scalar(1.0, 1.0, 1.0), 16, 0); + + if (r1.x + r1.width < clone_image.cols && r1.x >= 0 && r1.x + r1.width >= 0 && r1.y >= 0 && r1.y + < clone_image.rows && r1.y + r1.height < clone_image.rows) + { + Mat imgRect = mat_image_(Range(r1.y, r1.y + r1.height), Range(r1.x, r1.x + r1.width)); + Mat warpMat = getAffineTransform(srcTri, dstTri); + warpAffine(imgRect, _dst, warpMat, mask.size()); + + for (int i = r2.y; i < r2.y + r2.height; ++i) + { + for (int j = r2.x; j < r2.x + r2.width; ++j) + { + if ((int)mask.at(i - r2.y, j - r2.x) > 0) + { + outImage.at(i, j) = _dst.at(i - r2.y, j - r2.x); + } + } + } + } + } + cvtColor(outImage, *image_mat, COLOR_RGB2BGRA); + + return absl::OkStatus(); + } + + REGISTER_CALCULATOR(FaceProcessorCalculator); +} // namespace mediapipe diff --git a/mediapipe/graphs/deformation/config/BUILD b/mediapipe/graphs/deformation/config/BUILD new file mode 100644 index 000000000..3ece89a85 --- /dev/null +++ b/mediapipe/graphs/deformation/config/BUILD @@ -0,0 +1,34 @@ +# Copyright 2020 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +package(default_visibility = ["//visibility:public"]) + +load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto") + +encode_binary_proto( + name = "triangles", + input = "triangles.pbtxt", + message_type = "mediapipe.face_geometry.Mesh3d", + output = "triangles.binarypb", + deps = [ + "//mediapipe/modules/face_geometry/protos:mesh_3d_proto", + ], +) + +exports_files( + srcs = glob(["**"]), +) + diff --git a/mediapipe/graphs/deformation/config/additionalNoseIndexes1.txt b/mediapipe/graphs/deformation/config/additionalNoseIndexes1.txt new file mode 100644 index 000000000..1f01cf53f --- /dev/null +++ b/mediapipe/graphs/deformation/config/additionalNoseIndexes1.txt @@ -0,0 +1,24 @@ +128 +2 +8 +266 +417 +36 +423 +193 +453 +326 +327 +328 +329 +203 +464 +350 +97 +98 +99 +100 +357 +233 +244 +121 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/config/additionalNoseIndexes2.txt b/mediapipe/graphs/deformation/config/additionalNoseIndexes2.txt new file mode 100644 index 000000000..c86cfacf1 --- /dev/null +++ b/mediapipe/graphs/deformation/config/additionalNoseIndexes2.txt @@ -0,0 +1,24 @@ +391 +9 +393 +285 +413 +164 +165 +167 +425 +426 +55 +189 +452 +330 +205 +206 +463 +341 +349 +101 +232 +112 +243 +120 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/config/additionalNoseIndexes3.txt b/mediapipe/graphs/deformation/config/additionalNoseIndexes3.txt new file mode 100644 index 000000000..b1c0b8a7b --- /dev/null +++ b/mediapipe/graphs/deformation/config/additionalNoseIndexes3.txt @@ -0,0 +1,40 @@ +0 +256 +133 +267 +269 +151 +280 +26 +411 +155 +414 +37 +39 +295 +427 +50 +436 +441 +187 +190 +65 +322 +450 +451 +207 +336 +337 +216 +347 +92 +221 +348 +230 +231 +362 +107 +108 +118 +119 +382 diff --git a/mediapipe/graphs/deformation/config/cheekbonesIndexes.txt b/mediapipe/graphs/deformation/config/cheekbonesIndexes.txt new file mode 100644 index 000000000..dd1ffd29f --- /dev/null +++ b/mediapipe/graphs/deformation/config/cheekbonesIndexes.txt @@ -0,0 +1,26 @@ +135 +137 +138 +395 +394 +401 +147 +411 +416 +169 +170 +433 +177 +435 +434 +187 +192 +213 +214 +215 +352 +364 +366 +367 +376 +123 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/config/faceOvalIndexes.txt b/mediapipe/graphs/deformation/config/faceOvalIndexes.txt new file mode 100644 index 000000000..7aa9be47e --- /dev/null +++ b/mediapipe/graphs/deformation/config/faceOvalIndexes.txt @@ -0,0 +1,36 @@ +10 +21 +54 +58 +67 +93 +103 +109 +127 +132 +136 +148 +149 +150 +152 +162 +172 +176 +234 +251 +284 +288 +297 +323 +332 +338 +356 +361 +365 +377 +378 +379 +389 +397 +400 +454 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/config/index_names.txt b/mediapipe/graphs/deformation/config/index_names.txt new file mode 100644 index 000000000..a99bc38ff --- /dev/null +++ b/mediapipe/graphs/deformation/config/index_names.txt @@ -0,0 +1,12 @@ +mediumNoseIndexes +mediumFaceIndexes +noseAllIndexes +cheekbonesIndexes +additionalNoseIndexes1 +additionalNoseIndexes2 +additionalNoseIndexes3 +faceOvalIndexes +upperLipCnt +lowerLipCnt +widerUpperLipPts1 +widerLowerLipPts1 diff --git a/mediapipe/graphs/deformation/config/lowerLipCnt.txt b/mediapipe/graphs/deformation/config/lowerLipCnt.txt new file mode 100644 index 000000000..23e0c1923 --- /dev/null +++ b/mediapipe/graphs/deformation/config/lowerLipCnt.txt @@ -0,0 +1,7 @@ +91 +181 +84 +17 +314 +405 +321 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/config/mediumFaceIndexes.txt b/mediapipe/graphs/deformation/config/mediumFaceIndexes.txt new file mode 100644 index 000000000..b7828a623 --- /dev/null +++ b/mediapipe/graphs/deformation/config/mediumFaceIndexes.txt @@ -0,0 +1,28 @@ +0 +1 +2 +4 +5 +6 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +151 +152 +164 +168 +175 +195 +197 +199 +200 +94 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/config/mediumNoseIndexes.txt b/mediapipe/graphs/deformation/config/mediumNoseIndexes.txt new file mode 100644 index 000000000..fd7010507 --- /dev/null +++ b/mediapipe/graphs/deformation/config/mediumNoseIndexes.txt @@ -0,0 +1,9 @@ +1 +195 +4 +5 +6 +197 +168 +19 +94 diff --git a/mediapipe/graphs/deformation/config/noseAllIndexes.txt b/mediapipe/graphs/deformation/config/noseAllIndexes.txt new file mode 100644 index 000000000..01d48aef7 --- /dev/null +++ b/mediapipe/graphs/deformation/config/noseAllIndexes.txt @@ -0,0 +1,95 @@ +1 +3 +4 +5 +6 +19 +20 +44 +45 +47 +48 +49 +51 +59 +60 +64 +75 +79 +94 +102 +114 +115 +122 +125 +126 +129 +131 +134 +141 +142 +166 +168 +174 +188 +195 +196 +197 +198 +209 +217 +218 +219 +220 +235 +236 +237 +238 +239 +240 +241 +242 +245 +248 +250 +274 +275 +277 +278 +279 +281 +289 +290 +294 +305 +309 +331 +343 +344 +351 +354 +355 +358 +360 +363 +370 +371 +392 +399 +412 +419 +420 +429 +437 +438 +439 +440 +455 +456 +457 +458 +459 +460 +461 +462 +465 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/config/tmp.txt b/mediapipe/graphs/deformation/config/tmp.txt new file mode 100644 index 000000000..1c22a47e1 --- /dev/null +++ b/mediapipe/graphs/deformation/config/tmp.txt @@ -0,0 +1,478 @@ +362.31481075286865 600.8182525634766 -49.94013011455536 +359.64590549468994 543.0130004882812 -87.40982294082642 +360.086088180542 561.5730667114258 -46.17353081703186 +347.2103476524353 485.1922607421875 -64.30929780006409 +359.8048210144043 525.0722122192383 -92.92720198631287 +360.4503107070923 501.8289566040039 -86.14297807216644 +362.4685335159302 444.8126220703125 -41.42429620027542 +245.9792947769165 443.63861083984375 28.838303983211517 +363.6971139907837 394.8170471191406 -30.798228979110718 +364.1529178619385 369.5340347290039 -34.45447951555252 +366.7464208602905 289.53128814697266 -16.76752656698227 +362.1213483810425 610.7015991210938 -47.918522357940674 +361.9111490249634 622.2106170654297 -42.000214755535126 +361.7901277542114 630.2610015869141 -33.034483194351196 +361.80956840515137 630.5062103271484 -31.02048486471176 +361.65876388549805 641.8568420410156 -31.974922120571136 +361.32380962371826 655.2098846435547 -34.1738560795784 +361.0052490234375 668.2077789306641 -29.830504059791565 +358.64492654800415 676.3850402832031 -11.20004117488861 +359.86140489578247 552.3215484619141 -79.14359271526337 +346.4217138290405 552.1380233764648 -56.163471937179565 +192.9905390739441 371.3102340698242 112.20251083374023 +292.0316004753113 460.60115814208984 8.2303536683321 +275.04276752471924 462.58548736572266 10.719646140933037 +259.49241399765015 460.65929412841797 16.608951687812805 +240.12107133865356 449.6390914916992 32.28025406599045 +306.7124676704407 454.4320297241211 9.515418112277985 +269.18585300445557 400.675048828125 3.427928164601326 +288.8427543640137 403.7632751464844 2.093831095844507 +250.83645343780518 403.60511779785156 10.545069873332977 +239.18262004852295 411.6160583496094 18.1980961561203 +228.3171844482422 460.1266860961914 41.43935680389404 +302.79680728912354 691.3064575195312 9.149758592247963 +241.1520266532898 437.7595520019531 35.078975558280945 +189.48997735977173 447.4779510498047 118.62715244293213 +212.72387266159058 445.54298400878906 59.737064838409424 +282.91693925857544 530.6702423095703 -5.825772061944008 +342.03819036483765 596.2926864624023 -46.076542139053345 +344.00843381881714 621.6756820678711 -38.56708645820618 +321.01934909820557 603.6526107788086 -35.2379634976387 +307.75672674179077 612.1319580078125 -23.22000414133072 +328.45593452453613 622.2881317138672 -31.554990112781525 +316.4400243759155 624.019889831543 -20.421039909124374 +292.482705116272 644.8335266113281 4.006892032921314 +346.5736126899719 542.3727035522461 -85.37296950817108 +344.0822696685791 525.3844833374023 -90.18237948417664 +217.00727462768555 391.1960983276367 27.05360770225525 +314.4440746307373 486.0729217529297 -9.935649186372757 +311.9938015937805 539.7780990600586 -43.17997097969055 +310.8665657043457 530.599479675293 -38.25033366680145 +237.0926856994629 528.7818908691406 18.765471875667572 +345.63275814056396 503.6951446533203 -80.07521510124207 +253.07697772979736 370.8142852783203 -4.48684886097908 +230.86107730865479 376.7497253417969 10.754622146487236 +205.6266188621521 339.0149688720703 78.1203943490982 +326.48303031921387 390.49964904785156 -23.940061926841736 +305.56124210357666 412.823486328125 5.849309116601944 +282.61058807373047 628.2186889648438 8.089985623955727 +217.4528431892395 606.013298034668 162.1191930770874 +324.4342088699341 549.4257736206055 -37.90909230709076 +336.75106287002563 554.6352386474609 -39.987490475177765 +293.8541293144226 627.0395278930664 4.047446697950363 +297.9695391654968 627.4363708496094 -1.4645572006702423 +223.25343132019043 364.15504455566406 18.954835832118988 +312.2819137573242 548.3728408813477 -34.18475389480591 +284.00705337524414 374.25506591796875 -16.747109591960907 +281.1100745201111 358.78501892089844 -20.09346440434456 +269.8329734802246 297.1759033203125 9.054293408989906 +213.61764907836914 353.2844543457031 44.04454082250595 +276.30271911621094 327.88902282714844 -6.657576039433479 +209.54910278320312 383.55751037597656 41.906957030296326 +200.42315483093262 379.8735046386719 74.44762945175171 +342.41307735443115 609.2689514160156 -44.42476004362106 +324.58361864089966 613.3655548095703 -34.433123767375946 +311.8226981163025 618.1516265869141 -23.389243483543396 +328.36570501327515 552.9110717773438 -34.33730721473694 +295.9403944015503 627.3085021972656 1.058464301750064 +302.7903914451599 633.2058334350586 -8.738560527563095 +300.23465394973755 627.6346206665039 -1.9429570063948631 +330.6955361366272 542.2183990478516 -60.3791481256485 +319.926381111145 627.951774597168 -17.149565666913986 +331.668963432312 628.3108901977539 -24.26801562309265 +345.7926392555237 629.2702865600586 -30.250057876110077 +337.8448462486267 674.9959564208984 -10.129471644759178 +339.70123529434204 665.8506774902344 -28.294233977794647 +341.120080947876 652.9856109619141 -31.669657230377197 +343.2656764984131 639.5659255981445 -29.17739689350128 +345.6427574157715 629.5268249511719 -28.149794340133667 +319.5728015899658 628.4269332885742 -15.51249548792839 +315.7986545562744 633.2476806640625 -17.23422959446907 +311.2205743789673 639.5831680297852 -19.361704140901566 +307.6517343521118 646.3972473144531 -16.121692210435867 +298.32361221313477 595.7241058349609 -16.49742141366005 +197.07026481628418 522.5712585449219 187.61728048324585 +360.03939628601074 556.6411209106445 -57.723058462142944 +310.6411099433899 628.126335144043 -7.598769888281822 +306.76589727401733 630.5442428588867 -8.854755163192749 +340.9685468673706 561.4918518066406 -40.77357888221741 +318.58154296875 559.0150451660156 -22.706395983695984 +339.0328288078308 558.0970001220703 -40.72387754917145 +297.1806049346924 496.8467330932617 -3.014509230852127 +271.3532495498657 508.8191604614258 4.138495959341526 +308.9188528060913 539.0650939941406 -28.855242133140564 +229.7728943824768 313.4063148498535 41.365743577480316 +237.28271484375 334.8050308227539 17.421178221702576 +247.82358169555664 355.85723876953125 -1.6578103601932526 +303.48783016204834 655.7356262207031 -2.8776374459266663 +320.89759826660156 365.4893493652344 -30.40818214416504 +317.74295568466187 328.6730194091797 -21.479719877243042 +314.16688442230225 290.34828186035156 -10.778319463133812 +246.95883750915527 456.8497085571289 24.94966149330139 +213.02721977233887 472.27821350097656 51.60349130630493 +316.6361689567566 448.2948684692383 11.210014969110489 +224.51123714447021 419.96166229248047 34.82187241315842 +325.7254457473755 473.91719818115234 -16.364332884550095 +320.2879858016968 531.8416595458984 -59.67542231082916 +200.17237901687622 482.4532699584961 71.37396275997162 +221.48051261901855 485.61683654785156 35.744640827178955 +240.50400495529175 492.3386001586914 18.90147864818573 +271.28535747528076 489.66487884521484 10.056736171245575 +294.8106050491333 482.0330047607422 4.697423055768013 +312.8950881958008 472.49622344970703 -0.738221537321806 +346.81421756744385 449.1241455078125 -34.28698629140854 +204.62121963500977 519.8909378051758 66.61324560642242 +213.83495092391968 416.9788360595703 43.828730285167694 +352.1424150466919 551.607666015625 -77.710622549057 +316.08299016952515 503.18206787109375 -17.371298521757126 +188.28613758087158 445.55076599121094 175.03106832504272 +326.122305393219 461.4849853515625 -3.944498486816883 +308.3635926246643 540.4094314575195 -12.52964437007904 +233.93686294555664 437.3786163330078 38.97653102874756 +320.04514932632446 520.8884429931641 -53.48777532577515 +204.8868227005005 563.2982635498047 179.70439910888672 +318.0916213989258 441.35467529296875 15.148012787103653 +332.17660903930664 510.15647888183594 -69.7168779373169 +246.40093803405762 649.1670989990234 71.32044732570648 +249.40900325775146 664.6739196777344 101.80629014968872 +194.89213943481445 521.3057327270508 127.27937936782837 +229.2852258682251 625.8639907836914 93.62288117408752 +194.3889570236206 411.54991149902344 101.41774535179138 +300.5897569656372 704.2718505859375 19.820925146341324 +354.11888122558594 555.8930587768555 -56.3017612695694 +301.9624900817871 516.7485809326172 -10.553073585033417 +201.8374729156494 448.94664764404297 77.33492016792297 +260.38565397262573 451.01245880126953 17.36278384923935 +274.47864532470703 452.89093017578125 12.385967820882797 +299.239296913147 636.0906982421875 -5.705782100558281 +211.2467908859253 554.1604995727539 71.35176479816437 +327.4084138870239 725.7848358154297 24.331189692020416 +286.07707500457764 701.6156768798828 57.26816654205322 +269.2962312698364 686.3684844970703 76.31482243537903 +365.2778148651123 330.47584533691406 -27.479083836078644 +358.84770154953003 729.9839019775391 18.132060170173645 +289.1335916519165 451.3136672973633 10.827817618846893 +303.04569482803345 447.21099853515625 12.386578023433685 +312.68999576568604 444.1571044921875 15.405343919992447 +205.10058403015137 413.8731002807617 61.52015447616577 +303.3820652961731 429.1310501098633 9.192320555448532 +288.88489723205566 424.5012664794922 7.256249114871025 +274.1650414466858 423.35071563720703 9.110438749194145 +259.4628667831421 425.9832000732422 14.47862595319748 +250.2515172958374 430.4640579223633 20.563799142837524 +187.92232275009155 404.9408721923828 147.70588159561157 +251.51052474975586 447.45338439941406 23.608345091342926 +359.7864532470703 576.8328094482422 -43.14598202705383 +312.0626163482666 584.9419784545898 -24.61238980293274 +324.0547513961792 545.9969329833984 -43.81172776222229 +339.08645153045654 578.1259536743164 -40.43156236410141 +363.23543071746826 420.7271957397461 -30.086140036582947 +264.0025806427002 671.0563659667969 54.207449555397034 +281.7091941833496 687.3118591308594 37.87458300590515 +325.7036876678467 717.67822265625 3.5573551431298256 +232.91792392730713 640.052490234375 133.60456466674805 +313.4801959991455 436.03153228759766 12.831948101520538 +335.8002305030823 477.6709747314453 -39.5541313290596 +358.56568336486816 721.3011169433594 -2.011377476155758 +304.51385021209717 715.4678344726562 37.6750186085701 +202.95018196105957 558.9566040039062 124.43295478820801 +331.45949363708496 628.7703323364258 -22.34838679432869 +327.7629590034485 636.6561126708984 -23.869176506996155 +324.14485216140747 646.7169189453125 -26.124992072582245 +321.60351276397705 658.1102752685547 -23.21021407842636 +318.0917501449585 666.5240478515625 -8.82074810564518 +306.7166304588318 625.5636215209961 -10.321009531617165 +302.3205542564392 622.8427505493164 -10.094233453273773 +298.712854385376 619.7249603271484 -9.787872210144997 +288.3246374130249 610.8903884887695 -4.832061566412449 +230.22202491760254 564.9159622192383 36.284818947315216 +336.4596891403198 460.1057815551758 -24.287088811397552 +328.8189768791199 425.0093078613281 2.1310108713805676 +319.7708559036255 428.3619689941406 9.346061423420906 +310.8956837654114 627.7709579467773 -8.471513763070107 +232.79664516448975 604.8004150390625 60.95910608768463 +342.1688461303711 422.27134704589844 -18.58388900756836 +309.9311399459839 678.4099578857422 1.6019318997859955 +361.1786699295044 482.5218963623047 -70.79273879528046 +346.76971435546875 468.0949401855469 -51.01327121257782 +361.77459239959717 464.4435501098633 -55.60548663139343 +324.7562026977539 505.9693908691406 -39.207214415073395 +358.36119174957275 707.8496551513672 -10.773193091154099 +358.4371304512024 691.1073303222656 -10.705491453409195 +332.7000045776367 688.8948822021484 -6.7887380719184875 +277.8965950012207 648.0897521972656 15.653365105390549 +295.393545627594 554.5903015136719 -7.615239322185516 +292.2783851623535 665.1010894775391 9.05498743057251 +260.0705909729004 551.2176132202148 2.3820671439170837 +281.56641483306885 572.2418594360352 -4.606827758252621 +250.0050973892212 579.2014312744141 15.266052782535553 +327.3827934265137 704.8444366455078 -5.169866345822811 +316.1485433578491 516.7336654663086 -27.182656824588776 +266.31940841674805 656.5741729736328 33.52453351020813 +284.52027797698975 674.3779754638672 23.33208829164505 +266.80147647857666 627.0775985717773 17.994222790002823 +218.27988624572754 582.8634643554688 76.66635274887085 +249.80828762054443 625.5910110473633 36.55672252178192 +213.114595413208 593.6250305175781 116.87825560569763 +271.6906929016113 596.0990905761719 3.441149778664112 +326.317720413208 490.1215362548828 -27.62029677629471 +325.59906005859375 539.2827987670898 -63.399245738983154 +317.07783222198486 545.1477432250977 -45.42772114276886 +331.3199973106384 527.3456192016602 -75.15479922294617 +314.9270439147949 406.2218475341797 -2.225418258458376 +286.793053150177 392.3470687866211 -2.8813794627785683 +262.07287073135376 389.1081237792969 0.5631764698773623 +242.1646785736084 393.15113067626953 9.229710549116135 +229.1750192642212 402.87853240966797 20.846603214740753 +224.53303813934326 440.9041213989258 46.56286954879761 +189.8299527168274 484.2190933227539 123.90522480010986 +235.57157278060913 468.4526824951172 32.08759367465973 +251.34508609771729 473.97212982177734 21.01878359913826 +272.71915912628174 474.7534942626953 12.821267545223236 +293.9277505874634 470.4969024658203 7.83361941576004 +310.88768005371094 463.1031799316406 6.670016795396805 +322.4747371673584 455.8022689819336 4.7100502252578735 +192.13680267333984 483.88782501220703 187.1008801460266 +317.98115730285645 550.5133056640625 -37.34726071357727 +335.155234336853 493.55438232421875 -52.37326383590698 +335.9710121154785 539.7782897949219 -76.81203961372375 +345.06670475006104 548.7778854370117 -70.86110830307007 +336.8111228942871 542.8650665283203 -68.8918787240982 +322.8489375114441 556.0922622680664 -32.045603692531586 +347.57999897003174 550.3982543945312 -75.349640250206 +349.6057105064392 554.798469543457 -56.22194945812225 +323.45998764038086 441.2953567504883 12.202259302139282 +331.4188313484192 444.6953582763672 2.0043303072452545 +335.8431887626648 446.3604736328125 -9.053357318043709 +245.0877285003662 434.12696838378906 27.005317211151123 +232.52679347991943 422.2683334350586 28.60065758228302 +375.129075050354 485.5686569213867 -66.13412797451019 +489.8169422149658 443.7993621826172 15.649998933076859 +374.1959238052368 552.6567459106445 -57.53134489059448 +549.8661518096924 376.85874938964844 93.96155834197998 +440.2923345565796 458.04664611816406 -0.3014412126503885 +457.0080757141113 461.1259460449219 0.5238388571888208 +473.02605628967285 460.9001159667969 4.890230633318424 +495.628023147583 449.5674133300781 18.310087770223618 +426.0021686553955 452.0965576171875 2.7239740267395973 +466.9395446777344 400.7162857055664 -7.452775910496712 +446.52535915374756 403.75301361083984 -6.610949859023094 +486.21758937835693 403.68438720703125 -1.9183234311640263 +498.62454414367676 411.7655563354492 4.756649918854237 +506.58478260040283 461.15882873535156 26.502676606178284 +416.82146072387695 693.8121032714844 3.1702547147870064 +496.17412090301514 437.4418640136719 21.256814897060394 +549.3464469909668 453.6427307128906 98.6981999874115 +524.5612049102783 445.5344772338867 43.08796048164368 +441.57936573028564 533.2194137573242 -14.471101015806198 +382.65531063079834 597.4103164672852 -48.406153321266174 +379.8743963241577 622.8031539916992 -40.8133989572525 +403.4557342529297 605.9185791015625 -40.07969945669174 +416.3720941543579 615.2373123168945 -29.67095285654068 +395.3726291656494 624.4565582275391 -35.8821764588356 +407.312707901001 626.9761276245117 -25.941161513328552 +429.1133165359497 647.1420288085938 -3.979477509856224 +372.91953563690186 542.8042221069336 -86.50143921375275 +375.26322841644287 525.7580184936523 -91.5913438796997 +522.051043510437 390.28438568115234 11.12120769917965 +411.86036109924316 487.31143951416016 -15.640982687473297 +409.8457860946655 541.0302352905273 -48.444803953170776 +411.71170234680176 531.7612838745117 -43.689598739147186 +489.60219383239746 533.1509399414062 4.472573474049568 +375.36510944366455 504.053955078125 -81.89855933189392 +483.0860137939453 369.4428253173828 -15.890099555253983 +506.9680166244507 375.4949951171875 -3.407749906182289 +535.6603574752808 343.8142395019531 61.62222862243652 +404.35669898986816 390.11817932128906 -27.821945250034332 +429.19429779052734 412.86109924316406 -1.2387271877378225 +439.73095893859863 630.7285308837891 -1.5582244656980038 +517.2312211990356 611.335334777832 145.1186978816986 +397.0336675643921 550.4440307617188 -41.68342173099518 +384.3327856063843 555.4219818115234 -41.896837055683136 +430.18645763397217 631.3722991943359 -4.207942374050617 +426.0752534866333 631.5671157836914 -8.771124556660652 +516.1490678787231 362.3528289794922 4.069492109119892 +409.93663787841797 549.6400833129883 -39.35904085636139 +449.89532947540283 373.4907531738281 -24.650874137878418 +452.7779960632324 357.8565216064453 -28.07433307170868 +467.0009994506836 299.6318054199219 -0.5581757007166743 +523.6217880249023 358.29761505126953 28.0683571100235 +456.60823345184326 330.89515686035156 -15.642706006765366 +531.0435247421265 382.22930908203125 25.11308580636978 +538.4967184066772 385.42991638183594 56.30295217037201 +381.85734272003174 610.4312133789062 -46.80420398712158 +399.549880027771 615.6716156005859 -39.0611869096756 +412.0954942703247 621.1643600463867 -29.384455382823944 +392.91014671325684 553.8689804077148 -37.45108902454376 +428.063006401062 631.5380859375 -6.604541391134262 +420.75469493865967 636.7933654785156 -15.22619515657425 +423.77503395080566 631.6289520263672 -9.005686417222023 +390.20763874053955 543.0792999267578 -63.937007188797 +403.6290693283081 630.7351684570312 -22.330348938703537 +391.8992328643799 630.2763366699219 -28.201560974121094 +377.955265045166 630.2646636962891 -32.497791945934296 +379.99460220336914 675.9234619140625 -12.158462852239609 +382.4532651901245 666.9402313232422 -30.5147784948349 +381.5766763687134 653.9708709716797 -34.037905633449554 +380.03790378570557 640.5079650878906 -31.568213403224945 +378.1312608718872 630.3887939453125 -30.430951416492462 +403.95132064819336 631.0291290283203 -20.634636282920837 +407.68070697784424 635.9734725952148 -22.271784245967865 +412.03386783599854 642.4954986572266 -24.572081565856934 +415.34568786621094 649.3869781494141 -21.70801743865013 +423.0186080932617 597.6608657836914 -23.924936950206757 +542.858247756958 529.2166137695312 169.14464950561523 +412.9946565628052 631.3167572021484 -13.893118500709534 +416.84532165527344 633.946418762207 -15.11006087064743 +379.8309659957886 562.1940994262695 -42.73179531097412 +403.6150360107422 560.328483581543 -26.81964933872223 +381.9331741333008 558.939094543457 -42.57615476846695 +429.0931034088135 498.6862564086914 -10.614701360464096 +454.7354507446289 511.7718505859375 -6.17903508245945 +414.0402889251709 540.428581237793 -34.470741748809814 +509.65288639068604 317.1898078918457 27.857261896133423 +498.0618381500244 339.1490936279297 4.3972546979784966 +489.1952705383301 354.1435241699219 -13.666276037693024 +416.7121124267578 657.8490447998047 -9.33106318116188 +410.217604637146 365.2154541015625 -34.52246278524399 +413.58139514923096 330.2250671386719 -26.18116021156311 +420.7748222351074 291.56354904174805 -15.680942237377167 +487.11220264434814 457.30411529541016 11.90019428730011 +522.3684883117676 474.2524719238281 35.08454114198685 +416.39119148254395 446.71112060546875 5.2111804112792015 +513.421196937561 419.6990966796875 19.548357725143433 +400.43140411376953 474.783935546875 -20.97128838300705 +400.6967496871948 532.8797149658203 -63.90847384929657 +533.5264778137207 488.26610565185547 52.815157771110535 +510.7516050338745 488.4153747558594 20.217141062021255 +488.8346529006958 495.30460357666016 5.44894952327013 +456.92164421081543 490.8591079711914 -0.35857056733220816 +433.9028835296631 481.6496276855469 -3.269107863306999 +416.31767749786377 471.9856643676758 -6.6906289011240005 +378.2948970794678 449.5512390136719 -36.269111931324005 +526.683669090271 525.6262969970703 48.04268717765808 +524.4007015228271 416.3843536376953 27.280404567718506 +367.53138542175293 551.911735534668 -78.59067678451538 +408.7814426422119 504.3170928955078 -22.77878075838089 +555.0616121292114 452.0101547241211 155.97532510757446 +403.1361436843872 461.4551544189453 -8.611655160784721 +415.44490814208984 541.8434143066406 -18.249017894268036 +503.7526273727417 436.89849853515625 24.546589851379395 +402.0468235015869 521.8133926391602 -57.816399335861206 +532.5650024414062 569.5538330078125 162.0654308795929 +416.1742973327637 440.01007080078125 8.98995392024517 +389.1408920288086 510.86524963378906 -72.86863446235657 +480.4836702346802 653.1622314453125 57.52188205718994 +479.2624282836914 668.9244079589844 89.09199714660645 +540.4804801940918 527.3695373535156 107.1337366104126 +499.9533748626709 630.0851440429688 77.29292213916779 +544.6455001831055 417.48207092285156 82.28857934474945 +419.3322229385376 706.9178771972656 13.273288160562515 +366.0675859451294 556.1402893066406 -56.881853342056274 +422.9339790344238 518.4252166748047 -17.242367416620255 +536.9172191619873 449.3646240234375 59.08366799354553 +473.0810308456421 451.0033416748047 5.877308696508408 +458.54156970977783 451.5306091308594 2.163227554410696 +424.2369747161865 639.8194122314453 -12.663331031799316 +519.0340089797974 559.5111465454102 53.35131525993347 +391.2917232513428 727.3930358886719 20.891675055027008 +437.550687789917 704.8687744140625 48.8024765253067 +456.8107080459595 690.1861572265625 65.88669955730438 +444.11866664886475 448.7945556640625 2.128894943743944 +430.5766439437866 444.57725524902344 4.955844841897488 +421.1917018890381 442.2102737426758 8.750274404883385 +534.3507957458496 412.5332260131836 43.76921206712723 +432.30600357055664 429.1683578491211 1.5175767615437508 +447.14394092559814 424.9408721923828 -1.7302481085062027 +462.11225509643555 424.2276382446289 -1.3385665509849787 +477.0910406112671 426.6614532470703 2.5095052644610405 +486.4478302001953 430.6407928466797 7.780308499932289 +555.3960514068604 411.0698699951172 128.79035353660583 +482.9979085922241 447.80887603759766 11.067120283842087 +409.1199588775635 586.3765716552734 -30.128961503505707 +397.227258682251 547.0495223999023 -47.53429055213928 +380.91050148010254 578.7846374511719 -42.74966686964035 +460.7864284515381 674.8594665527344 43.16956669092178 +440.7740592956543 690.6071472167969 29.093618094921112 +392.0944118499756 719.2385864257812 -0.10001176851801574 +498.66119384765625 644.6195983886719 118.40662121772766 +421.3210916519165 435.5072021484375 5.978861823678017 +388.3393621444702 478.3292007446289 -42.62728303670883 +416.32136821746826 718.0264282226562 31.367723643779755 +530.7446193695068 564.459342956543 105.10064363479614 +392.23315715789795 630.512580871582 -26.214196979999542 +395.560941696167 638.4785079956055 -27.860512733459473 +398.8153839111328 648.6321258544922 -30.361192524433136 +400.9631681442261 660.0777435302734 -27.593238651752472 +400.7044744491577 668.1196594238281 -13.446301370859146 +417.14242458343506 629.1056442260742 -16.865828186273575 +421.5265703201294 626.546745300293 -17.268246710300446 +425.33543586730957 623.4793090820312 -17.429639250040054 +433.3791446685791 613.1803512573242 -13.896046131849289 +496.71043395996094 569.5689010620117 21.088573336601257 +389.2374515533447 460.7672119140625 -27.562758028507233 +403.47744941711426 424.4717788696289 -1.895450558513403 +414.1584348678589 427.82482147216797 3.549591489136219 +412.82870292663574 631.1397552490234 -14.735534638166428 +494.7548246383667 609.0705871582031 45.80719470977783 +386.60545349121094 422.6103210449219 -21.048778742551804 +409.3497705459595 680.4081726074219 -3.6002641171216965 +376.8407106399536 468.47896575927734 -52.7133572101593 +398.5814094543457 506.9513702392578 -43.219455778598785 +384.8936891555786 689.9935913085938 -9.327406659722328 +444.5246887207031 651.2082672119141 6.435505226254463 +428.1888771057129 556.5470886230469 -14.717874974012375 +428.813681602478 667.7382659912109 1.8546424247324467 +464.16862964630127 554.7451400756836 -8.631420359015465 +441.45495414733887 574.9321746826172 -13.40932309627533 +474.7306966781616 583.2138061523438 2.6926674507558346 +390.04340171813965 706.3205718994141 -8.578235507011414 +407.80327320098877 517.7619934082031 -32.21008479595184 +457.4419069290161 660.2817535400391 23.080596327781677 +437.2658586502075 677.4289703369141 15.286377221345901 +456.34576320648193 630.5348968505859 6.904166266322136 +511.5388011932373 587.7153015136719 59.68905866146088 +475.2132797241211 629.59228515625 23.859423995018005 +518.8389587402344 598.4183883666992 98.38678479194641 +451.4426851272583 599.2813110351562 -6.91096767783165 +398.29662322998047 491.06029510498047 -31.948606967926025 +395.2072763442993 540.2492141723633 -67.25221753120422 +404.33438301086426 546.3447189331055 -49.92307662963867 +388.8867473602295 528.129768371582 -78.11590433120728 +418.3866262435913 406.26094818115234 -7.83122219145298 +447.628755569458 392.4750518798828 -11.032277047634125 +473.4980821609497 389.22035217285156 -10.253710225224495 +494.4223165512085 393.13838958740234 -3.7459462881088257 +508.447265625 402.79422760009766 6.417246758937836 +512.7858781814575 440.90904235839844 31.261489391326904 +547.2984409332275 490.5126190185547 103.35272312164307 +497.5730323791504 469.93946075439453 17.92179375886917 +479.5932197570801 475.38475036621094 8.673078417778015 +457.4919891357422 474.34764862060547 2.581671606749296 +436.7109203338623 468.65795135498047 -0.3157069184817374 +420.35940170288086 461.30104064941406 0.47410030849277973 +408.8504934310913 454.4709014892578 -0.39229159243404865 +550.1037740707397 490.5290222167969 167.97163367271423 +403.64546298980713 551.6719055175781 -41.71808123588562 +387.63726711273193 494.2497253417969 -55.56053280830383 +384.055552482605 540.538215637207 -79.20849680900574 +375.3244686126709 549.4364547729492 -72.22589671611786 +383.8312339782715 543.7438583374023 -71.36332511901855 +398.81486892700195 557.1538543701172 -35.70137947797775 +372.3822784423828 550.9265899658203 -76.46305739879608 +370.69892406463623 555.1999664306641 -57.17661201953888 +410.05074977874756 440.17982482910156 7.049205377697945 +399.94487285614014 443.82862091064453 -2.0457570254802704 +393.2704210281372 446.66893005371094 -12.368821799755096 +491.9056749343872 434.00867462158203 13.69398057460785 +505.40289402008057 422.0017623901367 14.507773518562317 +278.070273399353 437.74181365966797 16.967550963163376 +296.5090012550354 437.4724578857422 16.967550963163376 +278.02658557891846 420.81432342529297 16.967550963163376 +259.5726013183594 437.9621124267578 16.967550963163376 +278.0579137802124 454.41627502441406 16.967550963163376 +459.17272567749023 438.5255432128906 6.8281786143779755 +478.56732845306396 439.512939453125 6.8281786143779755 +459.9502229690552 422.17994689941406 6.8281786143779755 +439.78996753692627 437.5259017944336 6.8281786143779755 +458.3905506134033 454.89044189453125 6.8281786143779755 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/config/triangles.txt b/mediapipe/graphs/deformation/config/triangles.txt new file mode 100644 index 000000000..f5f0e0428 --- /dev/null +++ b/mediapipe/graphs/deformation/config/triangles.txt @@ -0,0 +1,854 @@ +48 49 64 +259 260 444 +347 348 450 +391 423 426 +88 95 96 +309 457 459 +128 232 233 +28 56 157 +147 177 215 +22 26 154 +308 324 325 +282 283 293 +41 72 73 +202 204 210 +108 109 151 +33 130 247 +174 188 196 +171 175 199 +352 366 401 +7 25 33 +32 140 211 +345 366 447 +262 418 431 +287 410 432 +322 426 436 +271 272 311 +261 340 446 +326 391 393 +355 358 429 +67 69 108 +269 270 322 +55 65 107 +287 291 409 +394 395 431 +265 342 446 +26 112 155 +39 40 92 +97 165 167 +367 433 435 +92 186 216 +4 44 45 +122 188 196 +17 313 314 +279 358 429 +37 39 72 +294 455 460 +15 16 85 +13 38 82 +7 25 110 +250 290 392 +276 283 293 +6 168 351 +138 192 213 +411 425 427 +64 102 129 +285 413 441 +60 75 240 +318 402 403 +44 220 237 +1 19 44 +69 104 105 +63 68 71 +42 74 184 +199 200 208 +272 310 407 +285 295 441 +19 125 141 +345 352 366 +287 422 432 +138 213 215 +326 370 462 +60 99 240 +0 11 267 +50 123 187 +250 290 328 +62 76 77 +125 141 241 +28 157 158 +40 73 74 +270 303 304 +286 384 398 +249 255 339 +273 321 375 +29 223 224 +42 80 183 +252 256 451 +43 61 146 +256 341 382 +266 329 371 +114 174 188 +17 314 315 +282 283 443 +24 110 144 +26 231 232 +83 84 181 +259 443 444 +9 285 336 +250 309 392 +52 65 66 +413 414 463 +255 261 448 +277 355 437 +254 339 373 +260 466 467 +25 31 228 +424 430 431 +32 171 208 +341 362 382 +295 296 336 +40 92 186 +20 99 242 +111 116 123 +253 449 450 +187 192 214 +295 441 442 +46 70 156 +91 106 182 +48 115 219 +198 209 217 +29 30 160 +342 445 467 +9 336 337 +92 165 206 +341 362 463 +420 437 456 +369 395 431 +12 13 268 +46 113 225 +25 130 226 +202 212 214 +114 128 188 +342 446 467 +204 210 211 +85 86 179 +43 57 202 +291 306 375 +260 445 467 +251 284 298 +309 438 457 +3 196 236 +79 237 239 +292 306 307 +101 119 120 +88 178 179 +36 100 142 +28 56 221 +118 229 230 +101 118 119 +3 196 197 +40 74 185 +276 353 383 +363 420 456 +23 229 230 +113 225 247 +307 320 325 +1 4 274 +257 259 387 +20 60 99 +25 31 226 +86 87 178 +111 116 143 +62 183 191 +364 379 394 +41 42 81 +135 150 169 +285 413 417 +93 132 137 +327 358 423 +9 108 151 +50 101 205 +438 440 457 +422 424 430 +89 179 180 +34 127 139 +279 294 358 +346 448 449 +46 63 70 +47 100 126 +135 192 214 +344 360 440 +45 51 134 +195 248 281 +122 193 245 +21 68 71 +97 98 165 +175 377 396 +16 17 315 +369 395 400 +120 230 231 +3 195 197 +186 212 216 +32 194 201 +53 224 225 +200 421 428 +8 168 193 +35 113 226 +340 346 352 +257 386 387 +399 419 456 +277 329 355 +91 181 182 +294 327 460 +411 427 434 +111 117 123 +41 73 74 +58 132 177 +251 298 301 +134 198 236 +262 369 396 +106 182 194 +34 139 156 +254 448 449 +0 37 164 +349 451 452 +458 461 462 +194 204 211 +280 411 425 +266 371 423 +350 357 452 +28 158 159 +50 117 118 +6 197 419 +16 17 85 +64 235 240 +39 165 167 +252 256 381 +318 319 403 +129 142 203 +255 263 359 +270 322 410 +54 103 104 +279 331 358 +30 225 247 +116 143 227 +356 368 389 +271 272 304 +360 363 440 +259 260 387 +175 199 396 +273 287 291 +49 102 129 +252 380 381 +281 363 456 +457 459 461 +20 238 242 +46 113 124 +46 53 63 +420 429 437 +12 38 72 +18 200 201 +193 244 245 +293 298 301 +276 293 300 +21 54 68 +370 461 462 +52 66 105 +122 188 245 +44 125 237 +59 75 166 +38 41 81 +322 410 436 +379 394 395 +313 314 405 +399 437 456 +293 333 334 +261 346 448 +25 110 228 +26 154 155 +257 442 443 +427 432 436 +248 281 456 +292 307 325 +269 270 303 +0 164 267 +292 308 325 +329 330 349 +19 44 125 +35 111 226 +292 407 415 +18 313 421 +340 345 372 +266 330 425 +11 12 72 +8 9 285 +63 68 104 +148 171 175 +17 18 83 +200 201 208 +2 164 167 +314 404 405 +357 452 453 +19 274 354 +30 160 161 +13 268 312 +20 79 166 +39 92 165 +250 328 462 +277 350 357 +323 447 454 +140 148 171 +422 432 434 +359 446 467 +261 340 346 +425 426 436 +273 287 422 +266 423 426 +70 71 139 +37 164 167 +272 310 311 +22 23 145 +38 81 82 +135 169 214 +369 377 400 +15 16 315 +51 134 236 +341 463 464 +260 387 388 +202 210 214 +307 320 321 +53 223 224 +207 212 214 +453 464 465 +20 79 238 +418 424 431 +135 136 150 +169 210 211 +290 305 460 +92 206 216 +151 337 338 +238 241 242 +289 439 455 +198 217 236 +276 342 353 +282 295 296 +22 230 231 +316 402 403 +255 261 446 +320 404 405 +273 321 335 +30 161 247 +22 26 231 +86 178 179 +27 28 222 +60 75 166 +52 65 222 +63 70 71 +112 133 155 +6 196 197 +32 201 208 +123 137 177 +376 401 435 +76 77 146 +43 202 204 +47 100 121 +52 53 63 +329 355 371 +54 68 104 +38 41 72 +11 12 302 +348 349 450 +112 133 243 +291 408 409 +274 354 457 +75 235 240 +344 438 440 +22 23 230 +42 80 81 +322 391 426 +43 57 61 +258 384 385 +253 254 449 +280 346 347 +321 405 406 +34 143 156 +28 221 222 +126 129 209 +341 453 464 +25 33 130 +125 237 241 +116 123 137 +351 412 419 +319 320 404 +318 319 325 +164 267 393 +313 405 406 +14 15 316 +168 351 417 +9 107 108 +55 189 221 +260 444 445 +353 372 383 +34 227 234 +399 412 419 +265 353 372 +110 144 163 +0 11 37 +80 183 191 +64 98 240 +7 110 163 +132 137 177 +57 185 186 +249 255 263 +288 361 401 +36 101 205 +118 119 230 +358 371 423 +67 103 104 +65 66 107 +365 367 397 +189 193 244 +233 244 245 +323 366 447 +339 373 390 +327 328 460 +352 376 411 +34 143 227 +289 392 439 +267 269 302 +292 308 415 +114 174 217 +257 259 443 +271 303 304 +297 332 333 +97 141 242 +26 112 232 +112 232 233 +285 295 336 +12 268 302 +52 63 105 +278 279 360 +5 195 281 +264 372 447 +52 53 223 +34 127 234 +352 376 401 +286 414 441 +20 60 166 +254 339 448 +265 340 372 +284 298 333 +274 275 440 +50 187 205 +346 347 449 +43 91 146 +48 219 235 +77 90 96 +136 138 172 +282 293 334 +115 131 220 +276 342 445 +280 346 352 +327 391 423 +57 61 185 +11 267 302 +128 233 245 +49 131 209 +77 90 91 +133 173 190 +279 360 429 +18 83 201 +292 306 407 +413 414 441 +364 365 367 +2 94 141 +289 305 455 +9 55 107 +112 233 244 +237 239 241 +32 194 211 +36 203 206 +364 416 434 +283 444 445 +42 183 184 +362 414 463 +277 343 437 +264 356 454 +275 281 363 +135 136 138 +83 182 201 +278 279 294 +36 142 203 +9 151 337 +266 425 426 +335 406 418 +23 24 144 +24 110 228 +43 91 106 +278 294 455 +59 166 219 +89 90 96 +262 396 428 +300 368 383 +97 98 99 +44 45 220 +115 218 219 +286 398 414 +174 196 236 +321 335 406 +282 442 443 +310 407 415 +141 241 242 +112 243 244 +2 94 370 +47 114 128 +296 299 334 +330 348 349 +271 302 303 +394 430 434 +425 427 436 +238 239 241 +1 4 44 +130 226 247 +48 115 131 +88 89 96 +256 451 452 +48 64 235 +288 401 435 +17 84 85 +199 200 428 +11 37 72 +199 396 428 +2 97 141 +367 397 435 +378 395 400 +378 379 395 +252 253 374 +280 352 411 +10 109 151 +417 464 465 +46 124 156 +121 128 232 +376 411 433 +40 185 186 +319 320 325 +411 416 433 +340 345 352 +277 329 350 +117 118 229 +120 231 232 +97 99 242 +61 76 146 +269 391 393 +30 224 225 +64 98 129 +8 285 417 +268 271 311 +19 94 141 +56 157 173 +79 218 237 +264 356 368 +4 274 275 +187 192 213 +69 107 108 +367 416 433 +265 340 446 +392 438 439 +264 447 454 +189 190 221 +47 121 128 +354 457 461 +24 228 229 +255 339 448 +250 309 459 +47 126 217 +301 368 389 +297 299 337 +43 106 204 +361 366 401 +320 321 405 +100 101 120 +345 372 447 +39 72 73 +93 137 227 +19 94 370 +272 407 408 +2 97 167 +61 76 184 +148 152 175 +23 24 229 +83 181 182 +354 370 461 +360 363 420 +119 120 230 +343 357 412 +23 144 145 +283 443 444 +5 51 195 +299 333 334 +316 317 402 +127 139 162 +84 180 181 +90 91 181 +297 299 333 +122 168 193 +297 337 338 +62 76 183 +27 222 223 +18 200 421 +344 438 439 +369 377 396 +10 151 338 +410 432 436 +306 307 375 +22 153 154 +278 344 439 +293 300 301 +262 369 431 +52 222 223 +305 455 460 +131 198 209 +360 420 429 +35 111 143 +294 327 358 +48 49 131 +49 129 209 +35 124 143 +58 172 215 +161 246 247 +251 301 389 +256 341 452 +49 64 102 +50 117 123 +115 218 220 +147 213 215 +249 339 390 +252 450 451 +293 298 333 +256 381 382 +187 205 207 +280 330 425 +272 304 408 +306 407 408 +147 187 213 +276 283 445 +6 122 168 +326 327 391 +29 30 224 +71 139 162 +3 51 195 +126 209 217 +124 143 156 +260 388 466 +31 111 117 +289 290 305 +62 78 96 +315 403 404 +318 324 325 +205 206 216 +422 430 434 +8 55 193 +140 170 176 +275 363 440 +66 69 105 +291 306 408 +88 89 179 +323 361 366 +319 403 404 +364 365 379 +300 301 368 +289 290 392 +355 358 371 +98 99 240 +279 294 331 +326 327 328 +189 190 243 +263 466 467 +33 246 247 +315 316 403 +59 219 235 +268 311 312 +376 433 435 +182 194 201 +266 329 330 +268 271 302 +262 418 421 +138 172 215 +195 197 248 +258 286 441 +278 439 455 +278 344 360 +123 147 177 +357 412 465 +131 134 198 +351 412 465 +6 351 419 +189 243 244 +257 258 442 +169 210 214 +270 409 410 +274 440 457 +76 183 184 +37 39 167 +79 166 218 +84 85 180 +55 65 221 +113 226 247 +362 398 414 +257 258 386 +288 397 435 +296 299 336 +17 18 313 +3 51 236 +248 419 456 +343 399 437 +27 29 160 +258 385 386 +59 75 235 +116 137 227 +27 159 160 +133 190 243 +93 227 234 +282 296 334 +47 114 217 +57 202 212 +269 302 303 +250 458 459 +19 354 370 +2 164 393 +55 189 193 +27 28 159 +149 170 176 +4 45 51 +66 69 107 +343 399 412 +264 368 383 +411 416 434 +135 138 192 +252 253 450 +294 331 358 +357 453 465 +31 111 226 +265 342 353 +77 91 146 +263 359 467 +56 173 190 +287 409 410 +458 459 461 +250 458 462 +67 108 109 +62 77 96 +98 129 203 +85 179 180 +330 347 348 +50 101 118 +90 180 181 +171 199 208 +280 330 347 +79 238 239 +314 315 404 +63 104 105 +46 53 225 +123 147 187 +78 95 96 +351 417 465 +126 129 142 +174 217 236 +258 441 442 +341 452 453 +349 450 451 +14 316 317 +56 190 221 +4 275 281 +128 188 245 +41 42 74 +273 291 375 +218 220 237 +276 300 383 +8 168 417 +269 322 391 +1 19 274 +252 374 380 +347 449 450 +264 372 383 +427 432 434 +131 134 220 +152 175 377 +165 203 206 +258 286 384 +326 328 462 +70 139 156 +65 221 222 +329 349 350 +355 429 437 +304 408 409 +21 71 162 +205 207 216 +36 100 101 +273 335 424 +364 367 416 +413 463 464 +98 165 203 +4 5 51 +413 417 464 +61 184 185 +149 150 170 +100 120 121 +120 121 232 +299 336 337 +49 64 129 +364 394 434 +22 145 153 +150 169 170 +35 113 124 +39 40 73 +58 177 215 +166 218 219 +253 373 374 +74 184 185 +267 269 393 +273 422 424 +394 430 431 +17 83 84 +140 170 211 +277 343 357 +4 5 281 +14 15 86 +307 321 375 +36 205 206 +100 126 142 +2 326 393 +89 90 180 +27 29 223 +67 69 104 +62 78 191 +290 328 460 +262 421 428 +2 326 370 +187 207 214 +45 134 220 +253 254 373 +15 315 316 +12 13 38 +207 212 216 +406 418 421 +117 228 229 +106 194 204 +313 406 421 +282 295 442 +349 350 452 +6 122 196 +335 418 424 +140 148 176 +169 170 211 +255 359 446 +197 248 419 +309 392 438 +31 117 228 +284 332 333 +14 86 87 +270 304 409 +8 9 55 +32 140 171 +15 85 86 +57 186 212 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/config/upperLipCnt.txt b/mediapipe/graphs/deformation/config/upperLipCnt.txt new file mode 100644 index 000000000..f464a3929 --- /dev/null +++ b/mediapipe/graphs/deformation/config/upperLipCnt.txt @@ -0,0 +1,7 @@ +40 +39 +37 +0 +267 +269 +270 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/config/widerLowerLipPts1.txt b/mediapipe/graphs/deformation/config/widerLowerLipPts1.txt new file mode 100644 index 000000000..eaf40281f --- /dev/null +++ b/mediapipe/graphs/deformation/config/widerLowerLipPts1.txt @@ -0,0 +1,9 @@ +106 +43 +335 +273 +18 +83 +182 +406 +313 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/config/widerUpperLipPts1.txt b/mediapipe/graphs/deformation/config/widerUpperLipPts1.txt new file mode 100644 index 000000000..50a6c4d2f --- /dev/null +++ b/mediapipe/graphs/deformation/config/widerUpperLipPts1.txt @@ -0,0 +1,9 @@ +322 +164 +165 +391 +167 +393 +186 +410 +92 \ No newline at end of file diff --git a/mediapipe/graphs/deformation/deformation_cpu.pbtxt b/mediapipe/graphs/deformation/deformation_cpu.pbtxt new file mode 100644 index 000000000..c4f786da8 --- /dev/null +++ b/mediapipe/graphs/deformation/deformation_cpu.pbtxt @@ -0,0 +1,48 @@ +# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU. + +# Input image. (ImageFrame) +input_stream: "input_video" + +# Output image with rendered results. (ImageFrame) +output_stream: "output_video" + +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:output_video" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + + +# Defines side packets for further use in the graph. +node { + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:0:num_faces" + output_side_packet: "PACKET:1:with_attention" + node_options: { + [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { + packet { int_value: 1 } + packet { bool_value: true } + } + } +} + +# Subgraph that detects faces and corresponding landmarks. +node { + calculator: "FaceLandmarkFrontCpu" + input_stream: "IMAGE:throttled_input_video" + input_side_packet: "NUM_FACES:num_faces" + input_side_packet: "WITH_ATTENTION:with_attention" + output_stream: "LANDMARKS:multi_face_landmarks" +} + +node { + calculator: "FaceProcessorCalculator" + input_stream: "NORM_LANDMARKS:multi_face_landmarks" + input_stream: "IMAGE:throttled_input_video" + output_stream: "IMAGE:output_video" +} diff --git a/mediapipe/graphs/deformation/deformation_mobile.pbtxt b/mediapipe/graphs/deformation/deformation_mobile.pbtxt new file mode 100644 index 000000000..af3d5bc62 --- /dev/null +++ b/mediapipe/graphs/deformation/deformation_mobile.pbtxt @@ -0,0 +1,59 @@ +# MediaPipe graph that performs hair segmentation with TensorFlow Lite on GPU. +# Used in the example in +# mediapipie/examples/android/src/java/com/mediapipe/apps/hairsegmentationgpu. + +# Images on GPU coming into and out of the graph. +input_stream: "input_video" +output_stream: "output_video" + + +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:output_video" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Defines side packets for further use in the graph. +node { + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:with_attention" + node_options: { + [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { + packet { bool_value: true } + } + } +} + +# Defines side packets for further use in the graph. +node { + calculator: "GpuBufferToImageFrameCalculator" + input_stream: "throttled_input_video" + output_stream: "throttled_input_video_cpu" +} + +# Subgraph that detects faces and corresponding landmarks. +node { + calculator: "FaceLandmarkFrontGpu" + input_stream: "IMAGE:throttled_input_video" + input_side_packet: "WITH_ATTENTION:with_attention" + output_stream: "LANDMARKS:multi_face_landmarks" +} + +node { + calculator: "FaceProcessorCalculator" + input_stream: "NORM_LANDMARKS:multi_face_landmarks" + input_stream: "IMAGE:throttled_input_video_cpu" + output_stream: "IMAGE:out_image_frame" +} + +# Defines side packets for further use in the graph. +node { + calculator: "ImageFrameToGpuBufferCalculator" + input_stream: "out_image_frame" + output_stream: "output_video" +} diff --git a/mediapipe/graphs/image_style/README.md b/mediapipe/graphs/image_style/README.md new file mode 100644 index 000000000..afeb457cd --- /dev/null +++ b/mediapipe/graphs/image_style/README.md @@ -0,0 +1,36 @@ +# Image Style + +This graph performs face aligning. + +## Getting started + +Clone branch. + +1. Desktop-CPU + +Build with: +``` +bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/image_style:image_style_cpu +``` +Run with (using your camera): +``` +bazel-bin/mediapipe/examples/desktop/image_style/image_style_cpu --calculator_graph_config_file=mediapipe/graphs/image_style/image_style_cpu.pbtxt +``` +Run with (using video): +``` +bazel-bin/mediapipe/examples/desktop/image_style/image_style_cpu --calculator_graph_config_file=mediapipe/graphs/image_style/image_style_cpu.pbtxt +--input_video_path=/path/video.mp4 +--output_video_path=/path/outvideo.mp4 +``` + +2. Mobile (Android) + +Build with: +``` +bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/imagestylegpu:imagestylegpu +``` +Install with: +``` +adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/imagestylegpu/imagestylegpu.apk +``` + diff --git a/mediapipe/graphs/image_style/image_style.pbtxt b/mediapipe/graphs/image_style/image_style.pbtxt deleted file mode 100644 index 6ff9318f0..000000000 --- a/mediapipe/graphs/image_style/image_style.pbtxt +++ /dev/null @@ -1,84 +0,0 @@ -# MediaPipe graph that performs hair segmentation with TensorFlow Lite on GPU. -# Used in the example in -# mediapipie/examples/android/src/java/com/mediapipe/apps/hairsegmentationgpu. - -# Images on GPU coming into and out of the graph. -input_stream: "input_video" -output_stream: "output_video" - - -node { - calculator: "FlowLimiterCalculator" - input_stream: "input_video" - input_stream: "FINISHED:output_video" - input_stream_info: { - tag_index: "FINISHED" - back_edge: true - } - output_stream: "throttled_input_video" -} - -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE_GPU:throttled_input_video" - output_stream: "IMAGE_GPU:transformed_input_video" - node_options: { - [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { - output_width: 256 - output_height: 256 - } - } -} - -# Converts the transformed input image on GPU into an image tensor stored in -# tflite::gpu::GlBuffer. The zero_center option is set to false to normalize the -# pixel values to [0.f, 1.f] as opposed to [-1.f, 1.f]. With the -# max_num_channels option set to 4, all 4 RGBA channels are contained in the -# image tensor. -node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE_GPU:transformed_input_video" - output_stream: "TENSORS_GPU:image_tensor" - options { - [mediapipe.TfLiteConverterCalculatorOptions.ext] { - output_tensor_float_range { - min: -1 - max: 1 - } - } - } -} - - -node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS_GPU:image_tensor" - output_stream: "TENSORS:stylized_tensor" - node_options: { - [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] { - model_path: "mediapipe/models/metaf-512-mobile3.tflite" - use_gpu: true - } - } -} - -node { - calculator: "TfLiteTensorsToSegmentationCalculator" - input_stream: "TENSORS:stylized_tensor" - output_stream: "MASK:mask_image" - node_options: { - [type.googleapis.com/mediapipe.TfLiteTensorsToSegmentationCalculatorOptions] { - tensor_width: 256 - tensor_height: 256 - tensor_channels: 3 - } - } -} - -# Transfers the annotated image from CPU back to GPU memory, to be sent out of -# the graph. -node: { - calculator: "ImageFrameToGpuBufferCalculator" - input_stream: "mask_image" - output_stream: "output_video" -} diff --git a/mediapipe/graphs/image_style/image_style_cpu (copy).pbtxt b/mediapipe/graphs/image_style/image_style_cpu (copy).pbtxt deleted file mode 100644 index 5ba1ab286..000000000 --- a/mediapipe/graphs/image_style/image_style_cpu (copy).pbtxt +++ /dev/null @@ -1,94 +0,0 @@ -# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU. - -# Input image. (ImageFrame) -input_stream: "input_video" - -# Output image with rendered results. (ImageFrame) -output_stream: "output_video" - -# Throttles the images flowing downstream for flow control. It passes through -# the very first incoming image unaltered, and waits for downstream nodes -# (calculators and subgraphs) in the graph to finish their tasks before it -# passes through another image. All images that come in while waiting are -# dropped, limiting the number of in-flight images in most part of the graph to -# 1. This prevents the downstream nodes from queuing up incoming images and data -# excessively, which leads to increased latency and memory usage, unwanted in -# real-time mobile applications. It also eliminates unnecessarily computation, -# e.g., the output produced by a node may get dropped downstream if the -# subsequent nodes are still busy processing previous inputs. -node { - calculator: "FlowLimiterCalculator" - input_stream: "input_video" - input_stream: "FINISHED:output_video" - input_stream_info: { - tag_index: "FINISHED" - back_edge: true - } - output_stream: "throttled_input_video" -} - -# Transforms the input image on CPU to a 320x320 image. To scale the image, by -# default it uses the STRETCH scale mode that maps the entire input image to the -# entire transformed image. As a result, image aspect ratio may be changed and -# objects in the image may be deformed (stretched or squeezed), but the object -# detection model used in this graph is agnostic to that deformation. -node: { - calculator: "ImageTransformationCalculator" - input_stream: "IMAGE:throttled_input_video" - output_stream: "IMAGE:transformed_input_video" - node_options: { - [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] { - output_width: 256 - output_height: 256 - } - } -} - -# Converts the transformed input image on CPU into an image tensor as a -# TfLiteTensor. The zero_center option is set to true to normalize the -# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f]. -node { - calculator: "TfLiteConverterCalculator" - input_stream: "IMAGE:transformed_input_video" - output_stream: "TENSORS:input_tensors" - options { - [mediapipe.TfLiteConverterCalculatorOptions.ext] { - zero_center: false - max_num_channels: 3 - output_tensor_float_range { - min: 0.0 - max: 255.0 - } - } - } -} - - -# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a -# vector of tensors representing, for instance, detection boxes/keypoints and -# scores. -node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS:input_tensors" - output_stream: "TENSORS:output_tensors" - node_options: { - [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] { - model_path: "mediapipe/models/model_float32.tflite" - } - } -} - - -node { - calculator: "TfLiteTensorsToSegmentationCalculator" - input_stream: "TENSORS:output_tensors" - output_stream: "MASK:output_video" - node_options: { - [type.googleapis.com/mediapipe.TfLiteTensorsToSegmentationCalculatorOptions] { - tensor_width: 256 - tensor_height: 256 - tensor_channels: 3 - } - } -} -