new landmarks to mask system
This commit is contained in:
mslight
parent
1338a57ba1
commit
6885e6bc75
|
|
@ -17,11 +17,12 @@
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
#include "mediapipe/framework/formats/classification.pb.h"
|
#include "mediapipe/framework/formats/classification.pb.h"
|
||||||
#include "mediapipe/framework/formats/detection.pb.h"
|
|
||||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
#include "mediapipe/framework/formats/rect.pb.h"
|
#include "mediapipe/framework/formats/rect.pb.h"
|
||||||
#include "mediapipe/util/render_data.pb.h"
|
#include "mediapipe/util/render_data.pb.h"
|
||||||
#include "tensorflow/lite/interpreter.h"
|
#include "tensorflow/lite/interpreter.h"
|
||||||
|
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||||
|
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||||
|
|
||||||
namespace mediapipe {
|
namespace mediapipe {
|
||||||
|
|
||||||
|
|
@ -48,11 +49,15 @@ typedef EndLoopCalculator<std::vector<::mediapipe::ClassificationList>>
|
||||||
EndLoopClassificationListCalculator;
|
EndLoopClassificationListCalculator;
|
||||||
REGISTER_CALCULATOR(EndLoopClassificationListCalculator);
|
REGISTER_CALCULATOR(EndLoopClassificationListCalculator);
|
||||||
|
|
||||||
|
typedef EndLoopCalculator<std::vector<std::unordered_map<std::string, cv::Mat>>>
|
||||||
|
EndLoopMapMaskCalculator;
|
||||||
|
REGISTER_CALCULATOR(EndLoopMapMaskCalculator);
|
||||||
|
|
||||||
|
typedef EndLoopCalculator<std::vector<std::tuple<double, double, double, double>>>
|
||||||
|
EndLoopFaceBoxCalculator;
|
||||||
|
REGISTER_CALCULATOR(EndLoopFaceBoxCalculator);
|
||||||
|
|
||||||
typedef EndLoopCalculator<std::vector<TfLiteTensor>> EndLoopTensorCalculator;
|
typedef EndLoopCalculator<std::vector<TfLiteTensor>> EndLoopTensorCalculator;
|
||||||
REGISTER_CALCULATOR(EndLoopTensorCalculator);
|
REGISTER_CALCULATOR(EndLoopTensorCalculator);
|
||||||
|
|
||||||
typedef EndLoopCalculator<std::vector<::mediapipe::Detection>>
|
|
||||||
EndLoopDetectionCalculator;
|
|
||||||
REGISTER_CALCULATOR(EndLoopDetectionCalculator);
|
|
||||||
|
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|
|
||||||
|
|
@ -42,6 +42,16 @@ cc_library(
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
cc_library(
|
||||||
|
name = "desktop_live_over_calculators",
|
||||||
|
deps = [
|
||||||
|
"//mediapipe/calculators/core:constant_side_packet_calculator",
|
||||||
|
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||||
|
"//mediapipe/graphs/beauty/subgraphs:face_renderer_cpu_over",
|
||||||
|
"//mediapipe/modules/face_landmark:face_landmark_front_gpu",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
cc_library(
|
cc_library(
|
||||||
name = "desktop_live_gpu_calculators",
|
name = "desktop_live_gpu_calculators",
|
||||||
deps = [
|
deps = [
|
||||||
|
|
@ -63,9 +73,29 @@ cc_library(
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
cc_library(
|
||||||
|
name = "mobile_calculators_over",
|
||||||
|
deps = [
|
||||||
|
"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
|
||||||
|
"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
|
||||||
|
"//mediapipe/calculators/core:flow_limiter_calculator",
|
||||||
|
"//mediapipe/graphs/beauty/subgraphs:face_renderer_gpu_over",
|
||||||
|
"//mediapipe/modules/face_landmark:face_landmark_front_gpu",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
mediapipe_binary_graph(
|
mediapipe_binary_graph(
|
||||||
name = "beauty_mobile_gpu_binary_graph",
|
name = "beauty_mobile_gpu_binary_graph",
|
||||||
graph = "beauty_mobile.pbtxt",
|
graph = "beauty_mobile.pbtxt",
|
||||||
output_name = "beauty_mobile_gpu.binarypb",
|
output_name = "beauty_mobile_gpu.binarypb",
|
||||||
deps = [":mobile_calculators"],
|
deps = [":mobile_calculators"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
mediapipe_binary_graph(
|
||||||
|
name = "beauty_mobile_over_binary_graph",
|
||||||
|
graph = "beauty_over.pbtxt",
|
||||||
|
output_name = "beauty_mobile_over.binarypb",
|
||||||
|
deps = [":mobile_calculators_over"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
|
||||||
68
mediapipe/graphs/beauty/beauty_over.pbtxt
Normal file
68
mediapipe/graphs/beauty/beauty_over.pbtxt
Normal file
|
|
@ -0,0 +1,68 @@
|
||||||
|
# MediaPipe graph that performs face mesh with TensorFlow Lite on GPU.
|
||||||
|
|
||||||
|
# GPU buffer. (GpuBuffer)
|
||||||
|
input_stream: "input_video"
|
||||||
|
|
||||||
|
# Max number of faces to detect/process. (int)
|
||||||
|
input_side_packet: "num_faces"
|
||||||
|
|
||||||
|
# Output image with rendered results. (GpuBuffer)
|
||||||
|
output_stream: "output_video"
|
||||||
|
# Collection of detected/processed faces, each represented as a list of
|
||||||
|
# landmarks. (std::vector<NormalizedLandmarkList>)
|
||||||
|
output_stream: "multi_face_landmarks"
|
||||||
|
|
||||||
|
# Throttles the images flowing downstream for flow control. It passes through
|
||||||
|
# the very first incoming image unaltered, and waits for downstream nodes
|
||||||
|
# (calculators and subgraphs) in the graph to finish their tasks before it
|
||||||
|
# passes through another image. All images that come in while waiting are
|
||||||
|
# dropped, limiting the number of in-flight images in most part of the graph to
|
||||||
|
# 1. This prevents the downstream nodes from queuing up incoming images and data
|
||||||
|
# excessively, which leads to increased latency and memory usage, unwanted in
|
||||||
|
# real-time mobile applications. It also eliminates unnecessarily computation,
|
||||||
|
# e.g., the output produced by a node may get dropped downstream if the
|
||||||
|
# subsequent nodes are still busy processing previous inputs.
|
||||||
|
node {
|
||||||
|
calculator: "FlowLimiterCalculator"
|
||||||
|
input_stream: "input_video"
|
||||||
|
input_stream: "FINISHED:output_video"
|
||||||
|
input_stream_info: {
|
||||||
|
tag_index: "FINISHED"
|
||||||
|
back_edge: true
|
||||||
|
}
|
||||||
|
output_stream: "throttled_input_video"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Defines side packets for further use in the graph.
|
||||||
|
node {
|
||||||
|
calculator: "ConstantSidePacketCalculator"
|
||||||
|
output_side_packet: "PACKET:with_attention"
|
||||||
|
node_options: {
|
||||||
|
[type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: {
|
||||||
|
packet { bool_value: true }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Subgraph that detects faces and corresponding landmarks.
|
||||||
|
node {
|
||||||
|
calculator: "FaceLandmarkFrontGpu"
|
||||||
|
input_stream: "IMAGE:throttled_input_video"
|
||||||
|
input_side_packet: "NUM_FACES:num_faces"
|
||||||
|
input_side_packet: "WITH_ATTENTION:with_attention"
|
||||||
|
output_stream: "LANDMARKS:multi_face_landmarks"
|
||||||
|
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
|
||||||
|
output_stream: "DETECTIONS:face_detections"
|
||||||
|
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Subgraph that renders face-landmark annotation onto the input image.
|
||||||
|
node {
|
||||||
|
calculator: "FaceRendererGpuOver"
|
||||||
|
input_stream: "IMAGE:throttled_input_video"
|
||||||
|
input_stream: "LANDMARKS:multi_face_landmarks"
|
||||||
|
input_stream: "NORM_RECTS:face_rects_from_landmarks"
|
||||||
|
input_stream: "DETECTIONS:face_detections"
|
||||||
|
output_stream: "IMAGE:output_video"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -48,6 +48,15 @@ mediapipe_simple_subgraph(
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
mediapipe_simple_subgraph(
|
||||||
|
name = "face_renderer_gpu_over",
|
||||||
|
graph = "face_renderer_gpu_over.pbtxt",
|
||||||
|
register_as = "FaceRendererGpuOver",
|
||||||
|
deps = [
|
||||||
|
":renderer_calculators",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
mediapipe_simple_subgraph(
|
mediapipe_simple_subgraph(
|
||||||
name = "face_renderer_cpu",
|
name = "face_renderer_cpu",
|
||||||
graph = "face_renderer_cpu.pbtxt",
|
graph = "face_renderer_cpu.pbtxt",
|
||||||
|
|
|
||||||
|
|
@ -75,7 +75,7 @@ node {
|
||||||
calculator: "SmoothFaceCalculator"
|
calculator: "SmoothFaceCalculator"
|
||||||
input_stream: "IMAGE:input_image_2"
|
input_stream: "IMAGE:input_image_2"
|
||||||
input_stream: "MASK:0:multi_mask"
|
input_stream: "MASK:0:multi_mask"
|
||||||
input_stream: "FACEBOX:multi_face_box"
|
input_stream: "FACEBOX:0:multi_face_box"
|
||||||
output_stream: "IMAGE:output_image"
|
output_stream: "IMAGE:output_image"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
109
mediapipe/graphs/beauty/subgraphs/face_renderer_cpu_old.pbtxt
Normal file
109
mediapipe/graphs/beauty/subgraphs/face_renderer_cpu_old.pbtxt
Normal file
|
|
@ -0,0 +1,109 @@
|
||||||
|
# MediaPipe face mesh rendering subgraph.
|
||||||
|
|
||||||
|
type: "FaceRendererCpu"
|
||||||
|
|
||||||
|
# CPU image. (ImageFrame)
|
||||||
|
input_stream: "IMAGE:input_image"
|
||||||
|
# Collection of detected/predicted faces, each represented as a list of
|
||||||
|
# landmarks. (std::vector<NormalizedLandmarkList>)
|
||||||
|
input_stream: "LANDMARKS:multi_face_landmarks"
|
||||||
|
|
||||||
|
# CPU image with rendered data. (ImageFrame)
|
||||||
|
output_stream: "IMAGE:output_image"
|
||||||
|
|
||||||
|
node {
|
||||||
|
calculator: "ImagePropertiesCalculator"
|
||||||
|
input_stream: "IMAGE:input_image"
|
||||||
|
output_stream: "SIZE:image_size"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Outputs each element of multi_face_landmarks at a fake timestamp for the rest
|
||||||
|
# of the graph to process. At the end of the loop, outputs the BATCH_END
|
||||||
|
# timestamp for downstream calculators to inform them that all elements in the
|
||||||
|
# vector have been processed.
|
||||||
|
node {
|
||||||
|
calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
|
||||||
|
input_stream: "ITERABLE:multi_face_landmarks"
|
||||||
|
output_stream: "ITEM:face_landmarks"
|
||||||
|
output_stream: "BATCH_END:landmark_timestamp"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Converts landmarks to drawing primitives for annotation overlay.
|
||||||
|
node {
|
||||||
|
calculator: "FaceLandmarksToRenderDataCalculator"
|
||||||
|
input_stream: "NORM_LANDMARKS:face_landmarks"
|
||||||
|
output_stream: "RENDER_DATA:landmarks_render_data"
|
||||||
|
node_options: {
|
||||||
|
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
|
||||||
|
landmark_color { r: 255 g: 0 b: 0 }
|
||||||
|
connection_color { r: 0 g: 255 b: 0 }
|
||||||
|
thickness: 2
|
||||||
|
visualize_landmark_depth: false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Collects a RenderData object for each hand into a vector. Upon receiving the
|
||||||
|
# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END
|
||||||
|
# timestamp.
|
||||||
|
node {
|
||||||
|
calculator: "EndLoopRenderDataCalculator"
|
||||||
|
input_stream: "ITEM:landmarks_render_data"
|
||||||
|
input_stream: "BATCH_END:landmark_timestamp"
|
||||||
|
output_stream: "ITERABLE:multi_face_landmarks_render_data"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Converts normalized rects to drawing primitives for annotation overlay.
|
||||||
|
#node {
|
||||||
|
# calculator: "RectToRenderDataCalculator"
|
||||||
|
# input_stream: "NORM_RECTS:rects"
|
||||||
|
# output_stream: "RENDER_DATA:rects_render_data"
|
||||||
|
# node_options: {
|
||||||
|
# [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
|
||||||
|
# filled: false
|
||||||
|
# color { r: 255 g: 0 b: 0 }
|
||||||
|
# thickness: 4.0
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
#}
|
||||||
|
|
||||||
|
node {
|
||||||
|
calculator: "FormFaceMaskCalculator"
|
||||||
|
input_stream: "IMAGE:input_image"
|
||||||
|
input_stream: "VECTOR:0:multi_face_landmarks_render_data"
|
||||||
|
output_stream: "FACEBOX:face_box"
|
||||||
|
output_stream: "MASK:multi_mask"
|
||||||
|
}
|
||||||
|
|
||||||
|
node {
|
||||||
|
calculator: "DrawLipstickCalculator"
|
||||||
|
input_stream: "IMAGE:input_image"
|
||||||
|
input_stream: "MASK:0:multi_mask"
|
||||||
|
output_stream: "IMAGE:input_image_1"
|
||||||
|
}
|
||||||
|
|
||||||
|
node {
|
||||||
|
calculator: "WhitenTeethCalculator"
|
||||||
|
input_stream: "IMAGE:input_image_1"
|
||||||
|
input_stream: "MASK:0:multi_mask"
|
||||||
|
output_stream: "IMAGE:input_image_2"
|
||||||
|
}
|
||||||
|
|
||||||
|
node {
|
||||||
|
calculator: "SmoothFaceCalculator"
|
||||||
|
input_stream: "IMAGE:input_image_2"
|
||||||
|
input_stream: "MASK:0:multi_mask"
|
||||||
|
input_stream: "FACEBOX:face_box"
|
||||||
|
output_stream: "IMAGE:output_image"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Draws annotations and overlays them on top of the input images.
|
||||||
|
#node {
|
||||||
|
# calculator: "AnnotationOverlayCalculator"
|
||||||
|
# input_stream: "IMAGE:input_image"
|
||||||
|
# input_stream: "VECTOR:0:multi_face_landmarks_render_data"
|
||||||
|
# output_stream: "IMAGE:output_image"
|
||||||
|
#}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -0,0 +1,96 @@
|
||||||
|
# MediaPipe face mesh rendering subgraph.
|
||||||
|
|
||||||
|
type: "FaceRendererGpu"
|
||||||
|
|
||||||
|
# GPU image. (GpuBuffer)
|
||||||
|
input_stream: "IMAGE:input_image"
|
||||||
|
# Collection of detected/predicted faces, each represented as a list of
|
||||||
|
# landmarks. (std::vector<NormalizedLandmarkList>)
|
||||||
|
input_stream: "LANDMARKS:multi_face_landmarks"
|
||||||
|
# Regions of interest calculated based on palm detections.
|
||||||
|
# (std::vector<NormalizedRect>)
|
||||||
|
input_stream: "NORM_RECTS:rects"
|
||||||
|
# Detected palms. (std::vector<Detection>)
|
||||||
|
input_stream: "DETECTIONS:detections"
|
||||||
|
|
||||||
|
# GPU image with rendered data. (GpuBuffer)
|
||||||
|
output_stream: "IMAGE:output_image"
|
||||||
|
|
||||||
|
node {
|
||||||
|
calculator: "ImagePropertiesCalculator"
|
||||||
|
input_stream: "IMAGE_GPU:input_image"
|
||||||
|
output_stream: "SIZE:image_size"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Converts detections to drawing primitives for annotation overlay.
|
||||||
|
node {
|
||||||
|
calculator: "DetectionsToRenderDataCalculator"
|
||||||
|
input_stream: "DETECTIONS:detections"
|
||||||
|
output_stream: "RENDER_DATA:detections_render_data"
|
||||||
|
node_options: {
|
||||||
|
[type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
|
||||||
|
thickness: 4.0
|
||||||
|
color { r: 0 g: 255 b: 0 }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Outputs each element of multi_face_landmarks at a fake timestamp for the rest
|
||||||
|
# of the graph to process. At the end of the loop, outputs the BATCH_END
|
||||||
|
# timestamp for downstream calculators to inform them that all elements in the
|
||||||
|
# vector have been processed.
|
||||||
|
node {
|
||||||
|
calculator: "BeginLoopNormalizedLandmarkListVectorCalculator"
|
||||||
|
input_stream: "ITERABLE:multi_face_landmarks"
|
||||||
|
output_stream: "ITEM:face_landmarks"
|
||||||
|
output_stream: "BATCH_END:end_timestamp"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Converts landmarks to drawing primitives for annotation overlay.
|
||||||
|
node {
|
||||||
|
calculator: "FaceLandmarksToRenderDataCalculator"
|
||||||
|
input_stream: "NORM_LANDMARKS:face_landmarks"
|
||||||
|
output_stream: "RENDER_DATA:landmarks_render_data"
|
||||||
|
node_options: {
|
||||||
|
[type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
|
||||||
|
landmark_color { r: 255 g: 0 b: 0 }
|
||||||
|
connection_color { r: 0 g: 255 b: 0 }
|
||||||
|
thickness: 2
|
||||||
|
visualize_landmark_depth: false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
# Collects a RenderData object for each hand into a vector. Upon receiving the
|
||||||
|
# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END
|
||||||
|
# timestamp.
|
||||||
|
node {
|
||||||
|
calculator: "EndLoopRenderDataCalculator"
|
||||||
|
input_stream: "ITEM:landmarks_render_data"
|
||||||
|
input_stream: "BATCH_END:end_timestamp"
|
||||||
|
output_stream: "ITERABLE:multi_face_landmarks_render_data"
|
||||||
|
}
|
||||||
|
|
||||||
|
# Converts normalized rects to drawing primitives for annotation overlay.
|
||||||
|
#node {
|
||||||
|
# calculator: "RectToRenderDataCalculator"
|
||||||
|
# input_stream: "NORM_RECTS:rects"
|
||||||
|
# output_stream: "RENDER_DATA:rects_render_data"
|
||||||
|
# node_options: {
|
||||||
|
# [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
|
||||||
|
# filled: false
|
||||||
|
# color { r: 255 g: 0 b: 0 }
|
||||||
|
# thickness: 4.0
|
||||||
|
# }
|
||||||
|
# }
|
||||||
|
#}
|
||||||
|
|
||||||
|
# Draws annotations and overlays them on top of the input images.
|
||||||
|
node {
|
||||||
|
calculator: "AnnotationOverlayCalculator"
|
||||||
|
input_stream: "IMAGE_GPU:input_image"
|
||||||
|
#input_stream: "detections_render_data"
|
||||||
|
input_stream: "VECTOR:0:multi_face_landmarks_render_data"
|
||||||
|
#input_stream: "rects_render_data"
|
||||||
|
output_stream: "IMAGE_GPU:output_image"
|
||||||
|
}
|
||||||
45
mediapipe/landmarks/BUILD
Normal file
45
mediapipe/landmarks/BUILD
Normal file
|
|
@ -0,0 +1,45 @@
|
||||||
|
# Copyright 2019 The MediaPipe Authors.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||||
|
|
||||||
|
licenses(["notice"])
|
||||||
|
|
||||||
|
package(default_visibility = ["//visibility:public"])
|
||||||
|
|
||||||
|
cc_library(
|
||||||
|
name = "landmarks_to_mask_calculator",
|
||||||
|
srcs = ["landmarks_to_mask_calculator.cc"],
|
||||||
|
hdrs = ["landmarks_to_mask_calculator.h"],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
deps = [
|
||||||
|
"//mediapipe/framework:calculator_framework",
|
||||||
|
"//mediapipe/framework:calculator_options_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:location_data_cc_proto",
|
||||||
|
"//mediapipe/framework/port:ret_check",
|
||||||
|
"//mediapipe/util:color_cc_proto",
|
||||||
|
"//mediapipe/util:render_data_cc_proto",
|
||||||
|
"@com_google_absl//absl/memory",
|
||||||
|
"@com_google_absl//absl/strings",
|
||||||
|
"//mediapipe/framework/formats:image_format_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:image_frame",
|
||||||
|
"//mediapipe/framework/formats:image_frame_opencv",
|
||||||
|
"//mediapipe/framework/port:opencv_core",
|
||||||
|
"//mediapipe/framework/port:opencv_imgproc",
|
||||||
|
"//mediapipe/framework/port:opencv_highgui",
|
||||||
|
"//mediapipe/framework/port:vector",
|
||||||
|
],
|
||||||
|
alwayslink = 1,
|
||||||
|
)
|
||||||
455
mediapipe/landmarks/landmarks_to_mask_calculator.cc
Normal file
455
mediapipe/landmarks/landmarks_to_mask_calculator.cc
Normal file
|
|
@ -0,0 +1,455 @@
|
||||||
|
// Copyright 2019 The MediaPipe Authors.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
#include "mediapipe/calculators/landmarks/landmarks_to_mask_calculator.h"
|
||||||
|
|
||||||
|
#include <math.h>
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cmath>
|
||||||
|
#include <string>
|
||||||
|
#include <map>
|
||||||
|
#include <iostream>
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
|
||||||
|
#include "absl/memory/memory.h"
|
||||||
|
#include "absl/strings/str_cat.h"
|
||||||
|
#include "absl/strings/str_join.h"
|
||||||
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
#include "mediapipe/framework/calculator_options.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
|
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||||
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
|
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||||
|
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||||
|
#include "mediapipe/framework/formats/location_data.pb.h"
|
||||||
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
|
#include "mediapipe/util/color.pb.h"
|
||||||
|
#include "mediapipe/util/render_data.pb.h"
|
||||||
|
#include "absl/strings/str_cat.h"
|
||||||
|
#include "mediapipe/framework/port/logging.h"
|
||||||
|
#include "mediapipe/framework/port/status.h"
|
||||||
|
#include "mediapipe/framework/port/vector.h"
|
||||||
|
|
||||||
|
namespace mediapipe
|
||||||
|
{
|
||||||
|
namespace
|
||||||
|
{
|
||||||
|
constexpr char kLandmarksTag[] = "LANDMARKS";
|
||||||
|
constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS";
|
||||||
|
constexpr char kLandmarkLabel[] = "KEYPOINT";
|
||||||
|
constexpr char kVectorTag[] = "VECTOR";
|
||||||
|
constexpr char kMaskTag[] = "MASK";
|
||||||
|
constexpr char kFaceBoxTag[] = "FACEBOX";
|
||||||
|
constexpr char kImageFrameTag[] = "IMAGE";
|
||||||
|
|
||||||
|
std::unordered_map<std::string, const std::vector<int>> orderList = {
|
||||||
|
{"UPPER_LIP", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 308, 415, 310, 311, 312, 13, 82, 81, 80, 191, 78}},
|
||||||
|
{"LOWER_LIP", {61, 78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
|
||||||
|
{"FACE_OVAL", {10, 338, 338, 297, 297, 332, 332, 284, 284, 251, 251, 389, 389, 356, 356, 454, 454, 323, 323, 361, 361, 288, 288, 397, 397, 365, 365, 379, 379, 378, 378, 400, 400, 377, 377, 152, 152, 148, 148, 176, 176, 149, 149, 150, 150, 136, 136, 172, 172, 58, 58, 132, 132, 93, 93, 234, 234, 127, 127, 162, 162, 21, 21, 54, 54, 103, 103, 67, 67, 109, 109, 10}},
|
||||||
|
{"MOUTH_INSIDE", {78, 191, 80, 81, 13, 312, 311, 310, 415, 308, 324, 318, 402, 317, 14, 87, 178, 88, 95}},
|
||||||
|
{"LEFT_EYE", {130, 33, 246, 161, 160, 159, 157, 173, 133, 155, 154, 153, 145, 144, 163, 7}},
|
||||||
|
{"RIGHT_EYE", {362, 398, 384, 385, 386, 387, 388, 466, 263, 249, 390, 373, 374, 380, 381, 382}},
|
||||||
|
{"LEFT_BROW", {70, 63, 105, 66, 107, 55, 65, 52, 53, 46}},
|
||||||
|
{"RIGHT_BROW", {336, 296, 334, 293, 301, 300, 283, 282, 295, 285}},
|
||||||
|
{"LIPS", {61, 185, 40, 39, 37, 0, 267, 269, 270, 409, 291, 375, 321, 405, 314, 17, 84, 181, 91, 146}},
|
||||||
|
{"PART_FOREHEAD_B", {21, 54, 103, 67, 109, 10, 338, 297, 332, 284, 251, 301, 293, 334, 296, 336, 9, 107, 66, 105, 63, 71}},
|
||||||
|
};
|
||||||
|
|
||||||
|
template <class LandmarkType>
|
||||||
|
bool IsLandmarkVisibleAndPresent(const LandmarkType &landmark,
|
||||||
|
bool utilize_visibility,
|
||||||
|
float visibility_threshold,
|
||||||
|
bool utilize_presence,
|
||||||
|
float presence_threshold)
|
||||||
|
{
|
||||||
|
if (utilize_visibility && landmark.has_visibility() &&
|
||||||
|
landmark.visibility() < visibility_threshold)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (utilize_presence && landmark.has_presence() &&
|
||||||
|
landmark.presence() < presence_threshold)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool NormalizedtoPixelCoordinates(double normalized_x, double normalized_y,
|
||||||
|
int image_width, int image_height, int *x_px,
|
||||||
|
int *y_px)
|
||||||
|
{
|
||||||
|
CHECK(x_px != nullptr);
|
||||||
|
CHECK(y_px != nullptr);
|
||||||
|
CHECK_GT(image_width, 0);
|
||||||
|
CHECK_GT(image_height, 0);
|
||||||
|
|
||||||
|
if (normalized_x < 0 || normalized_x > 1.0 || normalized_y < 0 ||
|
||||||
|
normalized_y > 1.0)
|
||||||
|
{
|
||||||
|
VLOG(1) << "Normalized coordinates must be between 0.0 and 1.0";
|
||||||
|
}
|
||||||
|
|
||||||
|
*x_px = static_cast<int32>(round(normalized_x * image_width));
|
||||||
|
*y_px = static_cast<int32>(round(normalized_y * image_height));
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::tuple<double, double, double, double> face_box;
|
||||||
|
|
||||||
|
float scale_factor_ = 1.0;
|
||||||
|
|
||||||
|
bool image_frame_available_ = false;
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
absl::Status LandmarksToMaskCalculator::GetContract(
|
||||||
|
CalculatorContract *cc)
|
||||||
|
{
|
||||||
|
RET_CHECK(cc->Inputs().HasTag(kLandmarksTag) ||
|
||||||
|
cc->Inputs().HasTag(kNormLandmarksTag))
|
||||||
|
<< "None of the input streams are provided.";
|
||||||
|
RET_CHECK(!(cc->Inputs().HasTag(kLandmarksTag) &&
|
||||||
|
cc->Inputs().HasTag(kNormLandmarksTag)))
|
||||||
|
<< "Can only one type of landmark can be taken. Either absolute or "
|
||||||
|
"normalized landmarks.";
|
||||||
|
|
||||||
|
if (cc->Inputs().HasTag(kImageFrameTag))
|
||||||
|
{
|
||||||
|
cc->Inputs().Tag(kImageFrameTag).Set<ImageFrame>();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cc->Inputs().HasTag(kLandmarksTag))
|
||||||
|
{
|
||||||
|
cc->Inputs().Tag(kLandmarksTag).Set<LandmarkList>();
|
||||||
|
}
|
||||||
|
if (cc->Inputs().HasTag(kNormLandmarksTag))
|
||||||
|
{
|
||||||
|
cc->Inputs().Tag(kNormLandmarksTag).Set<NormalizedLandmarkList>();
|
||||||
|
}
|
||||||
|
if (cc->Outputs().HasTag(kMaskTag))
|
||||||
|
{
|
||||||
|
cc->Outputs().Tag(kMaskTag).Set<std::unordered_map<std::string, cv::Mat>>();
|
||||||
|
}
|
||||||
|
if (cc->Outputs().HasTag(kFaceBoxTag))
|
||||||
|
{
|
||||||
|
cc->Outputs().Tag(kFaceBoxTag).Set<std::tuple<double, double, double, double>>();
|
||||||
|
}
|
||||||
|
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::Status LandmarksToMaskCalculator::Open(CalculatorContext *cc)
|
||||||
|
{
|
||||||
|
cc->SetOffset(TimestampDiff(0));
|
||||||
|
|
||||||
|
if (cc->Inputs().HasTag(kImageFrameTag))
|
||||||
|
{
|
||||||
|
image_frame_available_ = true;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::Status LandmarksToMaskCalculator::Process(CalculatorContext *cc)
|
||||||
|
{
|
||||||
|
// Check that landmarks are not empty and skip rendering if so.
|
||||||
|
// Don't emit an empty packet for this timestamp.
|
||||||
|
if (cc->Inputs().HasTag(kLandmarksTag) &&
|
||||||
|
cc->Inputs().Tag(kLandmarksTag).IsEmpty())
|
||||||
|
{
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
if (cc->Inputs().HasTag(kNormLandmarksTag) &&
|
||||||
|
cc->Inputs().Tag(kNormLandmarksTag).IsEmpty())
|
||||||
|
{
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
if (cc->Inputs().HasTag(kImageFrameTag) &&
|
||||||
|
cc->Inputs().Tag(kImageFrameTag).IsEmpty())
|
||||||
|
{
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize render target, drawn with OpenCV.
|
||||||
|
|
||||||
|
std::unique_ptr<cv::Mat> image_mat;
|
||||||
|
ImageFormat::Format target_format;
|
||||||
|
std::unordered_map<std::string, cv::Mat> all_masks;
|
||||||
|
|
||||||
|
MP_RETURN_IF_ERROR(CreateRenderTargetCpu(cc, image_mat, &target_format));
|
||||||
|
|
||||||
|
MP_RETURN_IF_ERROR(GetMasks(cc, all_masks, image_mat));
|
||||||
|
|
||||||
|
MP_RETURN_IF_ERROR(GetFaceBox(cc, image_mat));
|
||||||
|
|
||||||
|
MP_RETURN_IF_ERROR(RenderToCpu(cc, all_masks));
|
||||||
|
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::Status LandmarksToMaskCalculator::RenderToCpu(CalculatorContext *cc,
|
||||||
|
std::unordered_map<std::string, cv::Mat> &all_masks)
|
||||||
|
{
|
||||||
|
|
||||||
|
auto output_frame = absl::make_unique<std::unordered_map<std::string, cv::Mat>>(all_masks, all_masks.get_allocator());
|
||||||
|
|
||||||
|
if (cc->Outputs().HasTag(kMaskTag))
|
||||||
|
{
|
||||||
|
cc->Outputs()
|
||||||
|
.Tag(kMaskTag)
|
||||||
|
.Add(output_frame.release(), cc->InputTimestamp());
|
||||||
|
}
|
||||||
|
|
||||||
|
auto output_frame2 = absl::make_unique<std::tuple<double, double, double, double>>(face_box);
|
||||||
|
|
||||||
|
if (cc->Outputs().HasTag(kFaceBoxTag))
|
||||||
|
{
|
||||||
|
cc->Outputs()
|
||||||
|
.Tag(kFaceBoxTag)
|
||||||
|
.Add(output_frame2.release(), cc->InputTimestamp());
|
||||||
|
}
|
||||||
|
|
||||||
|
all_masks.clear();
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::Status LandmarksToMaskCalculator::CreateRenderTargetCpu(
|
||||||
|
CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
|
||||||
|
ImageFormat::Format *target_format)
|
||||||
|
{
|
||||||
|
if (image_frame_available_)
|
||||||
|
{
|
||||||
|
const auto &input_frame =
|
||||||
|
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
|
||||||
|
|
||||||
|
int target_mat_type;
|
||||||
|
switch (input_frame.Format())
|
||||||
|
{
|
||||||
|
case ImageFormat::SRGBA:
|
||||||
|
*target_format = ImageFormat::SRGBA;
|
||||||
|
target_mat_type = CV_8UC4;
|
||||||
|
break;
|
||||||
|
case ImageFormat::SRGB:
|
||||||
|
*target_format = ImageFormat::SRGB;
|
||||||
|
target_mat_type = CV_8UC3;
|
||||||
|
break;
|
||||||
|
case ImageFormat::GRAY8:
|
||||||
|
*target_format = ImageFormat::SRGB;
|
||||||
|
target_mat_type = CV_8UC3;
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
return absl::UnknownError("Unexpected image frame format.");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
image_mat = absl::make_unique<cv::Mat>(
|
||||||
|
input_frame.Height(), input_frame.Width(), target_mat_type);
|
||||||
|
|
||||||
|
auto input_mat = formats::MatView(&input_frame);
|
||||||
|
|
||||||
|
if (input_frame.Format() == ImageFormat::GRAY8)
|
||||||
|
{
|
||||||
|
cv::Mat rgb_mat;
|
||||||
|
cv::cvtColor(input_mat, rgb_mat, CV_GRAY2RGB);
|
||||||
|
rgb_mat.copyTo(*image_mat);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
input_mat.copyTo(*image_mat);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
image_mat = absl::make_unique<cv::Mat>(
|
||||||
|
150, 150, CV_8UC4,
|
||||||
|
cv::Scalar(255, 255,
|
||||||
|
255));
|
||||||
|
*target_format = ImageFormat::SRGBA;
|
||||||
|
}
|
||||||
|
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::Status LandmarksToMaskCalculator::GetMasks(CalculatorContext *cc,
|
||||||
|
std::unordered_map<std::string, cv::Mat> &all_masks, std::unique_ptr<cv::Mat> &image_mat)
|
||||||
|
{
|
||||||
|
|
||||||
|
int image_width_ = image_mat->cols;
|
||||||
|
int image_height_ = image_mat->rows;
|
||||||
|
|
||||||
|
if (cc->Inputs().HasTag(kLandmarksTag))
|
||||||
|
{
|
||||||
|
const LandmarkList &landmarks =
|
||||||
|
cc->Inputs().Tag(kNormLandmarksTag).Get<LandmarkList>();
|
||||||
|
|
||||||
|
cv::Mat mask;
|
||||||
|
std::vector<cv::Point> point_array;
|
||||||
|
for (const auto &[key, value] : orderList)
|
||||||
|
{
|
||||||
|
for (auto order : value)
|
||||||
|
{
|
||||||
|
const Landmark &landmark = landmarks.landmark(order);
|
||||||
|
|
||||||
|
if (!IsLandmarkVisibleAndPresent<Landmark>(
|
||||||
|
landmark, false,
|
||||||
|
0.0, false,
|
||||||
|
0.0))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto &point = landmark;
|
||||||
|
int x = -1;
|
||||||
|
int y = -1;
|
||||||
|
CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_,
|
||||||
|
image_height_, &x, &y));
|
||||||
|
point_array.push_back(cv::Point(x, y));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<std::vector<cv::Point>> point_vec;
|
||||||
|
point_vec.push_back(point_array);
|
||||||
|
mask = cv::Mat::zeros(image_mat->size(), CV_32FC1);
|
||||||
|
cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA);
|
||||||
|
mask.convertTo(mask, CV_8U);
|
||||||
|
all_masks.insert(make_pair(key, mask));
|
||||||
|
point_vec.clear();
|
||||||
|
point_array.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cc->Inputs().HasTag(kNormLandmarksTag))
|
||||||
|
{
|
||||||
|
const NormalizedLandmarkList &landmarks =
|
||||||
|
cc->Inputs().Tag(kNormLandmarksTag).Get<NormalizedLandmarkList>();
|
||||||
|
|
||||||
|
cv::Mat mask;
|
||||||
|
std::vector<cv::Point> point_array;
|
||||||
|
for (const auto &[key, value] : orderList)
|
||||||
|
{
|
||||||
|
for (auto order : value)
|
||||||
|
{
|
||||||
|
const NormalizedLandmark &landmark = landmarks.landmark(order);
|
||||||
|
|
||||||
|
if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
|
||||||
|
landmark, false,
|
||||||
|
0.0, false,
|
||||||
|
0.0))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto &point = landmark;
|
||||||
|
int x = -1;
|
||||||
|
int y = -1;
|
||||||
|
CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_,
|
||||||
|
image_height_, &x, &y));
|
||||||
|
point_array.push_back(cv::Point(x, y));
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<std::vector<cv::Point>> point_vec;
|
||||||
|
point_vec.push_back(point_array);
|
||||||
|
mask = cv::Mat::zeros(image_mat->size(), CV_32FC1);
|
||||||
|
cv::fillPoly(mask, point_vec, cv::Scalar::all(255), cv::LINE_AA);
|
||||||
|
mask.convertTo(mask, CV_8U);
|
||||||
|
all_masks.insert(make_pair(key, mask));
|
||||||
|
point_vec.clear();
|
||||||
|
point_array.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
absl::Status LandmarksToMaskCalculator::GetFaceBox(CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat)
|
||||||
|
{
|
||||||
|
cv::Mat mat_image_ = *image_mat.get();
|
||||||
|
|
||||||
|
int image_width_ = image_mat->cols;
|
||||||
|
int image_height_ = image_mat->rows;
|
||||||
|
|
||||||
|
std::vector<int> x_s, y_s;
|
||||||
|
double box_min_y, box_max_y, box_max_x, box_min_x;
|
||||||
|
if (cc->Inputs().HasTag(kLandmarksTag))
|
||||||
|
{
|
||||||
|
const LandmarkList &landmarks =
|
||||||
|
cc->Inputs().Tag(kLandmarksTag).Get<LandmarkList>();
|
||||||
|
|
||||||
|
for (int i = 0; i < landmarks.landmark_size(); ++i)
|
||||||
|
{
|
||||||
|
const Landmark &landmark = landmarks.landmark(i);
|
||||||
|
|
||||||
|
if (!IsLandmarkVisibleAndPresent<Landmark>(
|
||||||
|
landmark, false,
|
||||||
|
0.0, false,
|
||||||
|
0.0))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto &point = landmark;
|
||||||
|
int x = -1;
|
||||||
|
int y = -1;
|
||||||
|
CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_,
|
||||||
|
image_height_, &x, &y));
|
||||||
|
x_s.push_back(point.x());
|
||||||
|
x_s.push_back(point.y());
|
||||||
|
}
|
||||||
|
cv::minMaxLoc(y_s, &box_min_y, &box_max_y);
|
||||||
|
cv::minMaxLoc(x_s, &box_min_x, &box_max_x);
|
||||||
|
box_min_y = box_min_y * 0.9;
|
||||||
|
face_box = std::make_tuple(box_min_x, box_min_y, box_max_x, box_max_y);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (cc->Inputs().HasTag(kNormLandmarksTag))
|
||||||
|
{
|
||||||
|
const NormalizedLandmarkList &landmarks =
|
||||||
|
cc->Inputs().Tag(kNormLandmarksTag).Get<NormalizedLandmarkList>();
|
||||||
|
|
||||||
|
for (int i = 0; i < landmarks.landmark_size(); ++i)
|
||||||
|
{
|
||||||
|
const NormalizedLandmark &landmark = landmarks.landmark(i);
|
||||||
|
|
||||||
|
if (!IsLandmarkVisibleAndPresent<NormalizedLandmark>(
|
||||||
|
landmark, false,
|
||||||
|
0.0, false,
|
||||||
|
0.0))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto &point = landmark;
|
||||||
|
int x = -1;
|
||||||
|
int y = -1;
|
||||||
|
CHECK(NormalizedtoPixelCoordinates(point.x(), point.y(), image_width_,
|
||||||
|
image_height_, &x, &y));
|
||||||
|
x_s.push_back(point.x());
|
||||||
|
x_s.push_back(point.y());
|
||||||
|
}
|
||||||
|
cv::minMaxLoc(y_s, &box_min_y, &box_max_y);
|
||||||
|
cv::minMaxLoc(x_s, &box_min_x, &box_max_x);
|
||||||
|
box_min_y = box_min_y * 0.9;
|
||||||
|
face_box = std::make_tuple(box_min_x, box_min_y, box_max_x, box_max_y);
|
||||||
|
}
|
||||||
|
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
REGISTER_CALCULATOR(LandmarksToMaskCalculator);
|
||||||
|
} // namespace mediapipe
|
||||||
87
mediapipe/landmarks/landmarks_to_mask_calculator.h
Normal file
87
mediapipe/landmarks/landmarks_to_mask_calculator.h
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
// Copyright 2020 The MediaPipe Authors.
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
#ifndef MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_TO_MASK_CALCULATOR_H_
|
||||||
|
#define MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_TO_MASK_CALCULATOR_H_
|
||||||
|
|
||||||
|
#include "absl/memory/memory.h"
|
||||||
|
#include "absl/strings/str_cat.h"
|
||||||
|
#include "absl/strings/str_join.h"
|
||||||
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
#include "mediapipe/framework/calculator_options.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/location_data.pb.h"
|
||||||
|
#include "mediapipe/framework/port/ret_check.h"
|
||||||
|
#include "mediapipe/util/color.pb.h"
|
||||||
|
#include "mediapipe/util/render_data.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/image_frame.h"
|
||||||
|
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||||
|
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||||
|
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||||
|
#include "mediapipe/framework/port/logging.h"
|
||||||
|
#include "mediapipe/framework/port/status.h"
|
||||||
|
#include "mediapipe/framework/port/vector.h"
|
||||||
|
|
||||||
|
namespace mediapipe
|
||||||
|
{
|
||||||
|
|
||||||
|
// A calculator that converts Landmark proto to RenderData proto for
|
||||||
|
// visualization. The input should be LandmarkList proto. It is also possible
|
||||||
|
// to specify the connections between landmarks.
|
||||||
|
//
|
||||||
|
// Example config:
|
||||||
|
// node {
|
||||||
|
// calculator: "LandmarksToMaskCalculator"
|
||||||
|
// input_stream: "NORM_LANDMARKS:landmarks"
|
||||||
|
// output_stream: "RENDER_DATA:render_data"
|
||||||
|
// options {
|
||||||
|
// [LandmarksToRenderDataCalculatorOptions.ext] {
|
||||||
|
// landmark_connections: [0, 1, 1, 2]
|
||||||
|
// landmark_color { r: 0 g: 255 b: 0 }
|
||||||
|
// connection_color { r: 0 g: 255 b: 0 }
|
||||||
|
// thickness: 4.0
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
class LandmarksToMaskCalculator : public CalculatorBase
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
LandmarksToMaskCalculator() = default;
|
||||||
|
~LandmarksToMaskCalculator() override = default;
|
||||||
|
LandmarksToMaskCalculator(const LandmarksToMaskCalculator &) =
|
||||||
|
delete;
|
||||||
|
LandmarksToMaskCalculator &operator=(
|
||||||
|
const LandmarksToMaskCalculator &) = delete;
|
||||||
|
|
||||||
|
static absl::Status GetContract(CalculatorContract *cc);
|
||||||
|
|
||||||
|
absl::Status Open(CalculatorContext *cc) override;
|
||||||
|
|
||||||
|
absl::Status Process(CalculatorContext *cc) override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
absl::Status RenderToCpu(CalculatorContext *cc,
|
||||||
|
std::unordered_map<std::string, cv::Mat> &all_masks);
|
||||||
|
|
||||||
|
absl::Status GetFaceBox(CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat);
|
||||||
|
|
||||||
|
absl::Status GetMasks(CalculatorContext *cc, std::unordered_map<std::string, cv::Mat> &all_masks, std::unique_ptr<cv::Mat> &image_mat);
|
||||||
|
|
||||||
|
absl::Status CreateRenderTargetCpu(
|
||||||
|
CalculatorContext *cc, std::unique_ptr<cv::Mat> &image_mat,
|
||||||
|
ImageFormat::Format *target_format);
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace mediapipe
|
||||||
|
#endif // MEDIAPIPE_CALCULATORS_UTIL_LANDMARKS_TO_MASK_CALCULATOR_H_
|
||||||
Loading…
Reference in New Issue
Block a user