diff --git a/mediapipe/examples/desktop/face_mesh_dll/BUILD b/mediapipe/examples/desktop/face_mesh_dll/BUILD new file mode 100644 index 000000000..ff5709093 --- /dev/null +++ b/mediapipe/examples/desktop/face_mesh_dll/BUILD @@ -0,0 +1,65 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("windows_dll_library.bzl", "windows_dll_library") + +licenses(["notice"]) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//examples:__pkg__"], +) + +package(default_visibility = ["//mediapipe/examples:__subpackages__"]) + +# Define the shared library +windows_dll_library( + name = "face_mesh_lib", + srcs = ["face_mesh_lib.cpp"], + hdrs = ["face_mesh_lib.h"], + # Define COMPILING_DLL to export symbols during compiling the DLL. + copts = ["-DCOMPILING_DLL"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/port:file_helpers", + "//mediapipe/framework/port:opencv_highgui", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_video", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", + + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/modules/face_landmark:face_landmark_front_cpu_with_face_counter", + + + ] +) + +# **Implicitly link to face_mesh_lib.dll** + +## Link to face_mesh_lib.dll through its import library. +cc_binary( + name = "face_mesh_cpu", + srcs = ["face_mesh_cpu.cpp"], + deps = [ + ":face_mesh_lib", + ], +) \ No newline at end of file diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp new file mode 100644 index 000000000..e44375864 --- /dev/null +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp @@ -0,0 +1,56 @@ +#include "face_mesh_lib.h" + +int main(int argc, char **argv) { + google::InitGoogleLogging(argv[0]); + absl::ParseCommandLine(argc, argv); + + + cv::VideoCapture capture; + capture.open(0); + if (!capture.isOpened()) { + return -1; + } + + constexpr char kWindowName[] = "MediaPipe"; + + cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); +#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2) + capture.set(cv::CAP_PROP_FRAME_WIDTH, 640); + capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480); + capture.set(cv::CAP_PROP_FPS, 30); +#endif + + LOG(INFO) << "VideoCapture initialized."; + + FaceMeshDetector *faceMeshDetector = FaceMeshDetector_Construct(); + + LOG(INFO) << "FaceMeshDetector constructed."; + + LOG(INFO) << "Start grabbing and processing frames."; + bool grab_frames = true; + + while (grab_frames) { + // Capture opencv camera. + cv::Mat camera_frame_raw; + capture >> camera_frame_raw; + if (camera_frame_raw.empty()) { + LOG(INFO) << "Ignore empty frames from camera."; + continue; + } + cv::Mat camera_frame; + cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); + cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); + + FaceMeshDetector_ProcessFrame(faceMeshDetector, camera_frame); + + const int pressed_key = cv::waitKey(5); + if (pressed_key >= 0 && pressed_key != 255) + grab_frames = false; + + cv::imshow(kWindowName, camera_frame_raw); + } + + LOG(INFO) << "Shutting down."; + + FaceMeshDetector_Destruct(faceMeshDetector); +} \ No newline at end of file diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp new file mode 100644 index 000000000..a918e3719 --- /dev/null +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp @@ -0,0 +1,179 @@ +#include + +#include "face_mesh_lib.h" + +#define DEBUG + +FaceMeshDetector::FaceMeshDetector() { + const auto status = InitFaceMeshDetector(); + if (!status.ok()) { + LOG(INFO) << "Failed constructing FaceMeshDetector."; + } +} + +absl::Status FaceMeshDetector::InitFaceMeshDetector() { + LOG(INFO) << "Get calculator graph config contents: " << graphConfig; + + mediapipe::CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie( + graphConfig); + + LOG(INFO) << "Initialize the calculator graph."; + + MP_RETURN_IF_ERROR(graph.Initialize(config)); + + LOG(INFO) << "Start running the calculator graph."; + + ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller landmarks_poller, + graph.AddOutputStreamPoller(kOutputStream_landmarks)); + ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller face_count_poller, + graph.AddOutputStreamPoller(kOutputStream_faceCount)); + + landmarks_poller_ptr = std::make_unique( + std::move(landmarks_poller)); + face_count_poller_ptr = std::make_unique( + std::move(face_count_poller)); + + MP_RETURN_IF_ERROR(graph.StartRun({})); + + return absl::Status(); +} + +absl::Status FaceMeshDetector::ProcessFrameWithStatus(cv::Mat &camera_frame) { + // Wrap Mat into an ImageFrame. + auto input_frame = absl::make_unique( + mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, + mediapipe::ImageFrame::kDefaultAlignmentBoundary); + cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get()); + camera_frame.copyTo(input_frame_mat); + + // Send image packet into the graph. + + size_t frame_timestamp_us = + (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6; + MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( + kInputStream, mediapipe::Adopt(input_frame.release()) + .At(mediapipe::Timestamp(frame_timestamp_us)))); + LOG(INFO) << "Pushed new frame."; + +#ifdef DEBUG + LOG(INFO) << "Pushed new frame."; +#endif + mediapipe::Packet face_count_packet; + if (!face_count_poller_ptr || + !face_count_poller_ptr->Next(&face_count_packet)) { + LOG(INFO) << "Failed during getting next face_count_packet."; + + return absl::Status(); + } + auto &face_count = face_count_packet.Get(); + +#ifdef DEBUG + LOG(INFO) << "Got face_count: " << face_count; +#endif + + if (!face_count) { + return absl::Status(); + } + + mediapipe::Packet face_landmarks_packet; + if (!landmarks_poller_ptr || + !landmarks_poller_ptr->Next(&face_landmarks_packet)) { + LOG(INFO) << "Failed during getting next landmarks_packet."; + + return absl::Status(); + } + + auto &output_landmarks_vector = + face_landmarks_packet + .Get<::std::vector<::mediapipe::NormalizedLandmarkList>>(); + + auto &output_landmarks = output_landmarks_vector[0]; + +#ifdef DEBUG + LOG(INFO) << "Got landmarks_packet: " << output_landmarks.landmark_size(); +#endif + + auto &landmark = output_landmarks.landmark(0); +#ifdef DEBUG + LOG(INFO) << "First landmark: x - " << landmark.x() << ", y - " + << landmark.y() << ", z - " << landmark.z(); +#endif + + return absl::Status(); +} + +std::vector * +FaceMeshDetector::ProcessFrame(cv::Mat &camera_frame) { + ProcessFrameWithStatus(camera_frame); + + return new std::vector(); +} + +extern "C" { +DLLEXPORT FaceMeshDetector *FaceMeshDetector_Construct() { + return new FaceMeshDetector(); +} + +DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector) { + delete detector; +} + +DLLEXPORT void *FaceMeshDetector_ProcessFrame(FaceMeshDetector *detector, + cv::Mat &camera_frame) { + return reinterpret_cast(detector->ProcessFrame(camera_frame)); +} +} + +const char FaceMeshDetector::kInputStream[] = "input_video"; +const char FaceMeshDetector::kOutputStream_landmarks[] = "multi_face_landmarks"; +const char FaceMeshDetector::kOutputStream_faceCount[] = "face_count"; + +const std::string FaceMeshDetector::graphConfig = R"pb( +# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU. + +# Input image. (ImageFrame) +input_stream: "input_video" + +# Collection of detected/processed faces, each represented as a list of +# landmarks. (std::vector) +output_stream: "multi_face_landmarks" + +# Detected faces count. (int) +output_stream: "face_count" + +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:multi_face_landmarks" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Defines side packets for further use in the graph. +node { + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:num_faces" + node_options: { + [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { + packet { int_value: 1 } + } + } +} + +# Subgraph that detects faces and corresponding landmarks. +node { + calculator: "FaceLandmarkFrontCpuWithFaceCounter" + input_stream: "IMAGE:throttled_input_video" + input_side_packet: "NUM_FACES:num_faces" + output_stream: "LANDMARKS:multi_face_landmarks" + output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" + output_stream: "DETECTIONS:face_detections" + output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" + output_stream: "FACE_COUNT_FROM_LANDMARKS:face_count" +} + +)pb"; diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h new file mode 100644 index 000000000..7c62b01e4 --- /dev/null +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h @@ -0,0 +1,64 @@ +#ifndef FACE_MESH_LIBRARY_H +#define FACE_MESH_LIBRARY_H + +#ifdef COMPILING_DLL +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif + +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_graph.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status.h" + +class FaceMeshDetector { +public: + FaceMeshDetector(); + ~FaceMeshDetector() = default; + std::vector *ProcessFrame(cv::Mat &camera_frame); + +private: + absl::Status InitFaceMeshDetector(); + absl::Status ProcessFrameWithStatus(cv::Mat &camera_frame); + + static const char kInputStream[]; + static const char kOutputStream_landmarks[]; + static const char kOutputStream_faceCount[]; + + static const std::string graphConfig; + + mediapipe::CalculatorGraph graph; + + std::unique_ptr landmarks_poller_ptr; + std::unique_ptr face_count_poller_ptr; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +DLLEXPORT FaceMeshDetector *FaceMeshDetector_Construct(); + +DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector); + +DLLEXPORT void *FaceMeshDetector_ProcessFrame(FaceMeshDetector *detector, + cv::Mat &camera_frame); + +#ifdef __cplusplus +}; +#endif +#endif \ No newline at end of file diff --git a/mediapipe/examples/desktop/face_mesh_dll/windows_dll_library.bzl b/mediapipe/examples/desktop/face_mesh_dll/windows_dll_library.bzl new file mode 100644 index 000000000..69c243d60 --- /dev/null +++ b/mediapipe/examples/desktop/face_mesh_dll/windows_dll_library.bzl @@ -0,0 +1,62 @@ +""" +This is a simple windows_dll_library rule for builing a DLL Windows +that can be depended on by other cc rules. +Example useage: + windows_dll_library( + name = "hellolib", + srcs = [ + "hello-library.cpp", + ], + hdrs = ["hello-library.h"], + # Define COMPILING_DLL to export symbols during compiling the DLL. + copts = ["/DCOMPILING_DLL"], + ) +""" + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_import", "cc_library") + +def windows_dll_library( + name, + srcs = [], + deps = [], + hdrs = [], + visibility = None, + **kwargs): + """A simple windows_dll_library rule for builing a DLL Windows.""" + dll_name = name + ".dll" + import_lib_name = name + "_import_lib" + import_target_name = name + "_dll_import" + + # Build the shared library + cc_binary( + name = dll_name, + srcs = srcs + hdrs, + deps = deps, + linkshared = 1, + **kwargs + ) + + # Get the import library for the dll + native.filegroup( + name = import_lib_name, + srcs = [":" + dll_name], + output_group = "interface_library", + ) + + # Because we cannot directly depend on cc_binary from other cc rules in deps attribute, + # we use cc_import as a bridge to depend on the dll. + cc_import( + name = import_target_name, + interface_library = ":" + import_lib_name, + shared_library = ":" + dll_name, + ) + + # Create a new cc_library to also include the headers needed for the shared library + cc_library( + name = name, + hdrs = hdrs, + visibility = visibility, + deps = deps + [ + ":" + import_target_name, + ], + ) \ No newline at end of file