diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..ef4dc2f6a --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "files.associations": { + "*.inc": "cpp" + } +} \ No newline at end of file diff --git a/WORKSPACE b/WORKSPACE index 5341b094a..dbec861dc 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -323,7 +323,7 @@ http_archive( new_local_repository( name = "linux_opencv", build_file = "@//third_party:opencv_linux.BUILD", - path = "/usr", + path = "/usr/local", ) new_local_repository( diff --git a/mediapipe/examples/desktop/ multi_hand_tracking_run_graph_cpu_main.cc b/mediapipe/examples/desktop/ multi_hand_tracking_run_graph_cpu_main.cc new file mode 100644 index 000000000..b9b197fc7 --- /dev/null +++ b/mediapipe/examples/desktop/ multi_hand_tracking_run_graph_cpu_main.cc @@ -0,0 +1,187 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// An example of sending OpenCV webcam frames into a MediaPipe graph. +#include +#include +#include + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/port/commandlineflags.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status.h" + +constexpr char kWindowName[] = "MediaPipe"; +constexpr char kCalculatorGraphConfigFile[] = + "mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt"; +// Input and output streams. +constexpr char kInputStream[] = "input_video"; +constexpr char kOutputStream[] = "output_video"; +constexpr char kMultiHandLandmarksOutputStream[] = "multi_hand_landmarks"; + +DEFINE_string(input_video_path, "", + "Full path of video to load. " + "If not provided, attempt to use a webcam."); +DEFINE_string(output_video_path, "", + "Full path of where to save result (.mp4 only). " + "If not provided, show result in a window."); + +::mediapipe::Status RunMPPGraph( + std::unique_ptr<::mediapipe::CalculatorGraph> graph) { + + LOG(INFO) << "Initialize the camera or load the video."; + cv::VideoCapture capture; + const bool load_video = !FLAGS_input_video_path.empty(); + if (load_video) { + capture.open(FLAGS_input_video_path); + } else { + capture.open(0); + } + RET_CHECK(capture.isOpened()); + + cv::VideoWriter writer; + const bool save_video = !FLAGS_output_video_path.empty(); + if (!save_video) { + cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); +#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2) + capture.set(cv::CAP_PROP_FRAME_WIDTH, 640); + capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480); + capture.set(cv::CAP_PROP_FPS, 30); +#endif + } + + LOG(INFO) << "Start running the calculator graph."; + ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller poller, + graph->AddOutputStreamPoller(kOutputStream)); + ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller multi_hand_landmarks_poller, + graph->AddOutputStreamPoller(kMultiHandLandmarksOutputStream)); + MP_RETURN_IF_ERROR(graph->StartRun({})); + + LOG(INFO) << "Start grabbing and processing frames."; + bool grab_frames = true; + while (grab_frames) { + // Capture opencv camera or video frame. + cv::Mat camera_frame_raw; + capture >> camera_frame_raw; + if (camera_frame_raw.empty()) break; // End of video. + cv::Mat camera_frame; + cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); + if (!load_video) { + cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); + } + + // Wrap Mat into an ImageFrame. + auto input_frame = absl::make_unique<::mediapipe::ImageFrame>( + ::mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, + ::mediapipe::ImageFrame::kDefaultAlignmentBoundary); + cv::Mat input_frame_mat = ::mediapipe::formats::MatView(input_frame.get()); + camera_frame.copyTo(input_frame_mat); + + // Send image packet into the graph. + size_t frame_timestamp_us = + (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6; + MP_RETURN_IF_ERROR(graph->AddPacketToInputStream( + kInputStream, ::mediapipe::Adopt(input_frame.release()) + .At(::mediapipe::Timestamp(frame_timestamp_us)))); + + // Get the graph result packet, or stop if that fails. + ::mediapipe::Packet packet; + if (!poller.Next(&packet)) break; + auto& output_frame = packet.Get<::mediapipe::ImageFrame>(); + + // Get the packet containing multi_hand_landmarks. + ::mediapipe::Packet multi_hand_landmarks_packet; + if (!multi_hand_landmarks_poller.Next(&multi_hand_landmarks_packet)) break; + const auto& multi_hand_landmarks = + multi_hand_landmarks_packet.Get< + std::vector<::mediapipe::NormalizedLandmarkList>>(); + + LOG(INFO) << "#Multi Hand landmarks: " << multi_hand_landmarks.size(); + int hand_id = 0; + for (const auto& single_hand_landmarks: multi_hand_landmarks) { + ++hand_id; + LOG(INFO) << "Hand [" << hand_id << "]:"; + for (int i = 0; i < single_hand_landmarks.landmark_size(); ++i) { + const auto& landmark = single_hand_landmarks.landmark(i); + LOG(INFO) << "\tLandmark [" << i << "]: (" + << landmark.x() << ", " + << landmark.y() << ", " + << landmark.z() << ")"; + } + } + + // Convert back to opencv for display or saving. + cv::Mat output_frame_mat = ::mediapipe::formats::MatView(&output_frame); + cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); + if (save_video) { + if (!writer.isOpened()) { + LOG(INFO) << "Prepare video writer."; + writer.open(FLAGS_output_video_path, + ::mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 + capture.get(cv::CAP_PROP_FPS), output_frame_mat.size()); + RET_CHECK(writer.isOpened()); + } + writer.write(output_frame_mat); + } else { + cv::imshow(kWindowName, output_frame_mat); + // Press any key to exit. + const int pressed_key = cv::waitKey(5); + if (pressed_key >= 0 && pressed_key != 255) grab_frames = false; + } + } + + LOG(INFO) << "Shutting down."; + if (writer.isOpened()) writer.release(); + MP_RETURN_IF_ERROR(graph->CloseInputStream(kInputStream)); + return graph->WaitUntilDone(); +} + +::mediapipe::Status InitializeAndRunMPPGraph() { + + std::string calculator_graph_config_contents; + MP_RETURN_IF_ERROR(::mediapipe::file::GetContents( + kCalculatorGraphConfigFile, &calculator_graph_config_contents)); + LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; + mediapipe::CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie( + calculator_graph_config_contents); + + LOG(INFO) << "Initialize the calculator graph."; + std::unique_ptr<::mediapipe::CalculatorGraph> graph = + absl::make_unique<::mediapipe::CalculatorGraph>(); + MP_RETURN_IF_ERROR(graph->Initialize(config)); + + return RunMPPGraph(std::move(graph)); +} + +int main(int argc, char** argv) { + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + ::mediapipe::Status run_status = InitializeAndRunMPPGraph(); + if (!run_status.ok()) { + LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + return EXIT_FAILURE; + } else { + LOG(INFO) << "Success!"; + } + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/mediapipe/examples/desktop/BUILD b/mediapipe/examples/desktop/BUILD index 06a30d610..3983dc05b 100644 --- a/mediapipe/examples/desktop/BUILD +++ b/mediapipe/examples/desktop/BUILD @@ -75,6 +75,63 @@ cc_library( ], ) +cc_library( + name = "demo_run_graph_main_out_face", + srcs = ["demo_run_graph_main_out_face.cc"], + deps = [ + "//mediapipe/calculators/util:landmarks_to_render_data_calculator", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/port:commandlineflags", + "//mediapipe/framework/port:file_helpers", + "//mediapipe/framework/port:opencv_highgui", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_video", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + ], +) + +cc_library( + name = "demo_run_graph_main_out_pose", + srcs = ["demo_run_graph_main_out_pose.cc"], + deps = [ + "//mediapipe/calculators/util:landmarks_to_render_data_calculator", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/port:commandlineflags", + "//mediapipe/framework/port:file_helpers", + "//mediapipe/framework/port:opencv_highgui", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_video", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + ], +) + +cc_library( + name = "demo_run_graph_main_out_holistic", + srcs = ["demo_run_graph_main_out_holistic.cc"], + deps = [ + "//mediapipe/calculators/util:landmarks_to_render_data_calculator", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/port:commandlineflags", + "//mediapipe/framework/port:file_helpers", + "//mediapipe/framework/port:opencv_highgui", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_video", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + ], +) + cc_binary( name = "multi_hand_tracking_cpu", srcs = ["multi_hand_tracking_run_graph_cpu_main.cc"], diff --git a/mediapipe/examples/desktop/demo_run_graph_main_out_face.cc b/mediapipe/examples/desktop/demo_run_graph_main_out_face.cc new file mode 100644 index 000000000..6cc84fd47 --- /dev/null +++ b/mediapipe/examples/desktop/demo_run_graph_main_out_face.cc @@ -0,0 +1,214 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// An example of sending OpenCV webcam frames into a MediaPipe graph. + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/port/commandlineflags.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status.h" + +// Take stream from /mediapipe/graphs/face_tracking/face_detection_desktop_live.pbtxt +// RendererSubgraph - LANDMARKS:face_landmarks +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" + +// input and output streams to be used/retrieved by calculators +constexpr char kInputStream[] = "input_video"; +constexpr char kOutputStream[] = "output_video"; +constexpr char kLandmarksStream[] = "multi_face_landmarks"; +constexpr char kWindowName[] = "MediaPipe"; + +// cli inputs +DEFINE_string( + calculator_graph_config_file, "", + "Name of file containing text format CalculatorGraphConfig proto."); +DEFINE_string(input_video_path, "", + "Full path of video to load. " + "If not provided, attempt to use a webcam."); +DEFINE_string(output_video_path, "", + "Full path of where to save result (.mp4 only). " + "If not provided, show result in a window."); + + + +::mediapipe::Status RunMPPGraph() +{ + std::string calculator_graph_config_contents; + MP_RETURN_IF_ERROR(mediapipe::file::GetContents( + FLAGS_calculator_graph_config_file, &calculator_graph_config_contents)); + LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; + mediapipe::CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie( + calculator_graph_config_contents); + + LOG(INFO) << "Initialize the calculator graph."; + mediapipe::CalculatorGraph graph; + MP_RETURN_IF_ERROR(graph.Initialize(config)); + + LOG(INFO) << "2222"; + + LOG(INFO) << "Initialize the camera or load the video."; + cv::VideoCapture capture; + const bool load_video = !FLAGS_input_video_path.empty(); + if (load_video) + { + capture.open(FLAGS_input_video_path); + } + else + { + capture.open(0); + } + RET_CHECK(capture.isOpened()); + + cv::VideoWriter writer; + const bool save_video = !FLAGS_output_video_path.empty(); + if (save_video) + { + LOG(INFO) << "Prepare video writer."; + cv::Mat test_frame; + capture.read(test_frame); // Consume first frame. + capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning. + writer.open(FLAGS_output_video_path, + mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 + capture.get(cv::CAP_PROP_FPS), test_frame.size()); + RET_CHECK(writer.isOpened()); + } + else + { + cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); + } + + // pollers to retrieve streams from graph + // output stream (i.e. rendered landmark frame) + + LOG(INFO) << "Start running the calculator graph."; + ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller poller, + graph.AddOutputStreamPoller(kOutputStream)); + ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller poller_landmark, + graph.AddOutputStreamPoller(kLandmarksStream)); + MP_RETURN_IF_ERROR(graph.StartRun({})); + + LOG(INFO) << "Start grabbing and processing frames."; + size_t frame_timestamp = 0; + bool grab_frames = true; + while (grab_frames) + { + // Capture opencv camera or video frame. + cv::Mat camera_frame_raw; + capture >> camera_frame_raw; + if (camera_frame_raw.empty()) + break; // End of video. + cv::Mat camera_frame; + cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); + if (!load_video) + { + cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); + } + + // Wrap Mat into an ImageFrame. + auto input_frame = absl::make_unique( + mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, + mediapipe::ImageFrame::kDefaultAlignmentBoundary); + cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get()); + camera_frame.copyTo(input_frame_mat); + + // Send image packet into the graph. + MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( + kInputStream, mediapipe::Adopt(input_frame.release()) + .At(mediapipe::Timestamp(frame_timestamp++)))); + + // Get the graph result packet, or stop if that fails. + ::mediapipe::Packet packet; + if (!poller.Next(&packet)) + break; + auto &output_frame = packet.Get<::mediapipe::ImageFrame>(); + + // Get the packet containing multi_face_landmarks. + ::mediapipe::Packet landmarks_packet; + if (!poller_landmark.Next(&landmarks_packet)) + break; + const auto &multi_face_landmarks = + landmarks_packet.Get< + std::vector<::mediapipe::NormalizedLandmarkList>>(); + + LOG(INFO) << "#face landmarks: " << multi_face_landmarks.size(); + int face_id = 0; + for (const auto &single_face_landmarks : multi_face_landmarks) + { + ++face_id; + LOG(INFO) << "face [" << face_id << "]:"; + for (int i = 0; i < single_face_landmarks.landmark_size(); ++i) + { + const auto &landmark = single_face_landmarks.landmark(i); + LOG(INFO) << "\tLandmark [" << i << "]: (" + << landmark.x() << ", " + << landmark.y() << ", " + << landmark.z() << ")"; + } + } + + // Use packet.Get to recover values from packet + + // Convert back to opencv for display or saving. + cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame); + cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); + if (save_video) + { + writer.write(output_frame_mat); + } + else + { + cv::imshow(kWindowName, output_frame_mat); + // Press any key to exit. + const int pressed_key = cv::waitKey(5); + if (pressed_key >= 0 && pressed_key != 255) + grab_frames = false; + } + } + + LOG(INFO) << "1111"; + + LOG(INFO) << "Shutting down."; + if (writer.isOpened()) + writer.release(); + MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); + return graph.WaitUntilDone(); +} + +int main(int argc, char **argv) +{ + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + ::mediapipe::Status run_status = RunMPPGraph(); + + LOG(INFO) << "0000"; + + if (!run_status.ok()) + { + LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + } + else + { + LOG(INFO) << "Success!"; + } + return 0; +} \ No newline at end of file diff --git a/mediapipe/examples/desktop/demo_run_graph_main_out_hand.cc b/mediapipe/examples/desktop/demo_run_graph_main_out_hand.cc index 8d0bc04d2..00f181735 100644 --- a/mediapipe/examples/desktop/demo_run_graph_main_out_hand.cc +++ b/mediapipe/examples/desktop/demo_run_graph_main_out_hand.cc @@ -25,15 +25,14 @@ #include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/status.h" -//Take stream from /mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt -// RendererSubgraph - LANDMARKS:hand_landmarks +// Take stream from /mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt +// RendererSubgraph - LANDMARKS:hand_landmarks #include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" #include "mediapipe/framework/formats/landmark.pb.h" // input and output streams to be used/retrieved by calculators constexpr char kInputStream[] = "input_video"; constexpr char kOutputStream[] = "output_video"; -// constexpr char kLandmarksStream[] = "hand_landmarks"; constexpr char kLandmarksStream[] = "landmarks"; constexpr char kWindowName[] = "MediaPipe"; @@ -42,18 +41,19 @@ DEFINE_string( calculator_graph_config_file, "", "Name of file containing text format CalculatorGraphConfig proto."); DEFINE_string(input_video_path, "", - "Full path of video to load. " - "If not provided, attempt to use a webcam."); + "Full path of video to load. " + "If not provided, attempt to use a webcam."); DEFINE_string(output_video_path, "", - "Full path of where to save result (.mp4 only). " - "If not provided, show result in a window."); + "Full path of where to save result (.mp4 only). " + "If not provided, show result in a window."); -::mediapipe::Status RunMPPGraph() { +::mediapipe::Status RunMPPGraph() +{ std::string calculator_graph_config_contents; MP_RETURN_IF_ERROR(mediapipe::file::GetContents( FLAGS_calculator_graph_config_file, &calculator_graph_config_contents)); LOG(INFO) << "Get calculator graph config contents: " - << calculator_graph_config_contents; + << calculator_graph_config_contents; mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie( calculator_graph_config_contents); @@ -65,60 +65,58 @@ DEFINE_string(output_video_path, "", LOG(INFO) << "Initialize the camera or load the video."; cv::VideoCapture capture; const bool load_video = !FLAGS_input_video_path.empty(); - if (load_video) { + if (load_video) + { capture.open(FLAGS_input_video_path); } - else { + else + { capture.open(0); } RET_CHECK(capture.isOpened()); cv::VideoWriter writer; const bool save_video = !FLAGS_output_video_path.empty(); - if (save_video) { + if (save_video) + { LOG(INFO) << "Prepare video writer."; cv::Mat test_frame; - capture.read(test_frame); // Consume first frame. - capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning. + capture.read(test_frame); // Consume first frame. + capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning. writer.open(FLAGS_output_video_path, - mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 - capture.get(cv::CAP_PROP_FPS), test_frame.size()); + mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 + capture.get(cv::CAP_PROP_FPS), test_frame.size()); RET_CHECK(writer.isOpened()); } - else { + else + { cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); } // pollers to retrieve streams from graph // output stream (i.e. rendered landmark frame) - // ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, - // graph.AddOutputStreamPoller(kOutputStream)); - // // hand landmarks stream - // ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller_landmark, - // graph.AddOutputStreamPoller(kLandmarksStream)); LOG(INFO) << "Start running the calculator graph."; ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller poller, - graph.AddOutputStreamPoller(kOutputStream)); + graph.AddOutputStreamPoller(kOutputStream)); ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller poller_landmark, - graph.AddOutputStreamPoller(kLandmarksStream)); + graph.AddOutputStreamPoller(kLandmarksStream)); MP_RETURN_IF_ERROR(graph.StartRun({})); - - // LOG(INFO) << "Start running the calculator graph."; - // MP_RETURN_IF_ERROR(graph.StartRun({})); - LOG(INFO) << "Start grabbing and processing frames."; size_t frame_timestamp = 0; bool grab_frames = true; - while (grab_frames) { + while (grab_frames) + { // Capture opencv camera or video frame. cv::Mat camera_frame_raw; capture >> camera_frame_raw; - if (camera_frame_raw.empty()) break; // End of video. + if (camera_frame_raw.empty()) + break; // End of video. cv::Mat camera_frame; cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); - if (!load_video) { + if (!load_video) + { cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); } @@ -132,78 +130,75 @@ DEFINE_string(output_video_path, "", // Send image packet into the graph. MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( kInputStream, mediapipe::Adopt(input_frame.release()) - .At(mediapipe::Timestamp(frame_timestamp++)))); - - // // Get the graph result packet, or stop if that fails. - // mediapipe::Packet packet; - // mediapipe::Packet landmark_packet; + .At(mediapipe::Timestamp(frame_timestamp++)))); // Get the graph result packet, or stop if that fails. ::mediapipe::Packet packet; - if (!poller.Next(&packet)) break; - auto& output_frame = packet.Get<::mediapipe::ImageFrame>(); + if (!poller.Next(&packet)) + break; + auto &output_frame = packet.Get<::mediapipe::ImageFrame>(); - // //Polling the poller to get landmark packet - // if (!poller.Next(&packet)) break; - // if (!poller_landmark.Next(&landmark_packet)) break; - - // Get the packet containing multi_hand_landmarks. + // Get the packet containing multi_hand_landmarks. ::mediapipe::Packet landmarks_packet; - if (!poller_landmark.Next(&landmarks_packet)) break; - const auto& multi_hand_landmarks = + if (!poller_landmark.Next(&landmarks_packet)) + break; + const auto &multi_hand_landmarks = landmarks_packet.Get< std::vector<::mediapipe::NormalizedLandmarkList>>(); LOG(INFO) << "#Multi Hand landmarks: " << multi_hand_landmarks.size(); int hand_id = 0; - for (const auto& single_hand_landmarks: multi_hand_landmarks) { - ++hand_id; - LOG(INFO) << "Hand [" << hand_id << "]:"; - for (int i = 0; i < single_hand_landmarks.landmark_size(); ++i) { - const auto& landmark = single_hand_landmarks.landmark(i); - LOG(INFO) << "\tLandmark [" << i << "]: (" - << landmark.x() << ", " - << landmark.y() << ", " - << landmark.z() << ")"; - } - } + for (const auto &single_hand_landmarks : multi_hand_landmarks) + { + ++hand_id; + LOG(INFO) << "Hand [" << hand_id << "]:"; + for (int i = 0; i < single_hand_landmarks.landmark_size(); ++i) + { + const auto &landmark = single_hand_landmarks.landmark(i); + LOG(INFO) << "\tLandmark [" << i << "]: (" + << landmark.x() << ", " + << landmark.y() << ", " + << landmark.z() << ")"; + } + } // Use packet.Get to recover values from packet - // auto& output_frame = packet.Get(); - // auto& output_landmarks = landmark_packet.Get>(); // Convert back to opencv for display or saving. cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame); cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); - if (save_video) { + if (save_video) + { writer.write(output_frame_mat); } - else { + else + { cv::imshow(kWindowName, output_frame_mat); // Press any key to exit. const int pressed_key = cv::waitKey(5); - if (pressed_key >= 0 && pressed_key != 255) grab_frames = false; + if (pressed_key >= 0 && pressed_key != 255) + grab_frames = false; } - // // printout landmark values - // for (const ::mediapipe::NormalizedLandmarkList& landmark : output_landmarks) { - // std::cout << landmark.DebugString(); - // } } LOG(INFO) << "Shutting down."; - if (writer.isOpened()) writer.release(); + if (writer.isOpened()) + writer.release(); MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); return graph.WaitUntilDone(); } -int main(int argc, char** argv) { +int main(int argc, char **argv) +{ google::InitGoogleLogging(argv[0]); gflags::ParseCommandLineFlags(&argc, &argv, true); ::mediapipe::Status run_status = RunMPPGraph(); - if (!run_status.ok()) { + if (!run_status.ok()) + { LOG(ERROR) << "Failed to run the graph: " << run_status.message(); } - else { + else + { LOG(INFO) << "Success!"; } return 0; diff --git a/mediapipe/examples/desktop/demo_run_graph_main_out_holistic.cc b/mediapipe/examples/desktop/demo_run_graph_main_out_holistic.cc new file mode 100644 index 000000000..63d31a5fd --- /dev/null +++ b/mediapipe/examples/desktop/demo_run_graph_main_out_holistic.cc @@ -0,0 +1,206 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// An example of sending OpenCV webcam frames into a MediaPipe graph. + +// #include +// using namespace std; + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/port/commandlineflags.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status.h" + +// Take stream from /mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt +// RendererSubgraph - LANDMARKS:hand_landmarks +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" + +// input and output streams to be used/retrieved by calculators +constexpr char kInputStream[] = "input_video"; +constexpr char kOutputStream[] = "output_video"; +constexpr char kLandmarksStream[] = "pose_landmarks"; +constexpr char kWindowName[] = "MediaPipe"; + +// cli inputs +DEFINE_string( + calculator_graph_config_file, "", + "Name of file containing text format CalculatorGraphConfig proto."); +DEFINE_string(input_video_path, "", + "Full path of video to load. " + "If not provided, attempt to use a webcam."); +DEFINE_string(output_video_path, "", + "Full path of where to save result (.mp4 only). " + "If not provided, show result in a window."); + +::mediapipe::Status RunMPPGraph() +{ + std::string calculator_graph_config_contents; + MP_RETURN_IF_ERROR(mediapipe::file::GetContents( + FLAGS_calculator_graph_config_file, &calculator_graph_config_contents)); + LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; + mediapipe::CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie( + calculator_graph_config_contents); + + LOG(INFO) << "Initialize the calculator graph."; + mediapipe::CalculatorGraph graph; + MP_RETURN_IF_ERROR(graph.Initialize(config)); + + LOG(INFO) << "Initialize the camera or load the video."; + cv::VideoCapture capture; + const bool load_video = !FLAGS_input_video_path.empty(); + if (load_video) + { + capture.open(FLAGS_input_video_path); + } + else + { + capture.open(0); + } + RET_CHECK(capture.isOpened()); + + cv::VideoWriter writer; + const bool save_video = !FLAGS_output_video_path.empty(); + if (save_video) + { + LOG(INFO) << "Prepare video writer."; + cv::Mat test_frame; + capture.read(test_frame); // Consume first frame. + capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning. + writer.open(FLAGS_output_video_path, + mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 + capture.get(cv::CAP_PROP_FPS), test_frame.size()); + RET_CHECK(writer.isOpened()); + } + else + { + cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); + } + + // pollers to retrieve streams from graph + // output stream (i.e. rendered landmark frame) + + LOG(INFO) << "Start running the calculator graph."; + ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller poller, + graph.AddOutputStreamPoller(kOutputStream)); + ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller poller_landmark, + graph.AddOutputStreamPoller(kLandmarksStream)); + MP_RETURN_IF_ERROR(graph.StartRun({})); + + LOG(INFO) << "Start grabbing and processing frames."; + size_t frame_timestamp = 0; + bool grab_frames = true; + while (grab_frames) + { + // Capture opencv camera or video frame. + cv::Mat camera_frame_raw; + capture >> camera_frame_raw; + if (camera_frame_raw.empty()) + break; // End of video. + cv::Mat camera_frame; + cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); + if (!load_video) + { + cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); + } + + // Wrap Mat into an ImageFrame. + auto input_frame = absl::make_unique( + mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, + mediapipe::ImageFrame::kDefaultAlignmentBoundary); + cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get()); + camera_frame.copyTo(input_frame_mat); + + // Send image packet into the graph. + MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( + kInputStream, mediapipe::Adopt(input_frame.release()) + .At(mediapipe::Timestamp(frame_timestamp++)))); + + // Get the graph result packet, or stop if that fails. + ::mediapipe::Packet packet; + if (!poller.Next(&packet)) + break; + auto &output_frame = packet.Get<::mediapipe::ImageFrame>(); + + // Get the packet containing multi_hand_landmarks. + ::mediapipe::Packet landmarks_packet; + if (!poller_landmark.Next(&landmarks_packet)) + break; + + // const auto &multi_hand_landmarks = + // landmarks_packet.Get< + // std::vector<::mediapipe::NormalizedLandmarkList>>(); + + const auto &pose_landmarks = landmarks_packet.Get(); + + LOG(INFO) << "#Pose landmarks: " << pose_landmarks.landmark_size(); + + for (int i = 0; i < pose_landmarks.landmark_size(); ++i) + { + const auto &landmark = pose_landmarks.landmark(i); + LOG(INFO) << "\tLandmark [" << i << "]: (" + << landmark.x() << ", " + << landmark.y() << ", " + << landmark.z() << ")"; + } + + // Use packet.Get to recover values from packet + + // Convert back to opencv for display or saving. + cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame); + cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); + if (save_video) + { + writer.write(output_frame_mat); + } + else + { + cv::imshow(kWindowName, output_frame_mat); + // Press any key to exit. + const int pressed_key = cv::waitKey(5); + if (pressed_key >= 0 && pressed_key != 255) + grab_frames = false; + } + } + + LOG(INFO) << "Shutting down."; + if (writer.isOpened()) + writer.release(); + MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); + return graph.WaitUntilDone(); +} + +int main(int argc, char **argv) +{ + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + ::mediapipe::Status run_status = RunMPPGraph(); + if (!run_status.ok()) + { + LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + } + else + { + LOG(INFO) << "Success!"; + } + return 0; +} \ No newline at end of file diff --git a/mediapipe/examples/desktop/demo_run_graph_main_out_pose.cc b/mediapipe/examples/desktop/demo_run_graph_main_out_pose.cc new file mode 100644 index 000000000..058636871 --- /dev/null +++ b/mediapipe/examples/desktop/demo_run_graph_main_out_pose.cc @@ -0,0 +1,200 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// An example of sending OpenCV webcam frames into a MediaPipe graph. + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/port/commandlineflags.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status.h" + +// Take stream from /mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt +// RendererSubgraph - LANDMARKS:hand_landmarks +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" + +// input and output streams to be used/retrieved by calculators +constexpr char kInputStream[] = "input_video"; +constexpr char kOutputStream[] = "output_video"; +constexpr char kLandmarksStream[] = "pose_landmarks"; +constexpr char kWindowName[] = "MediaPipe"; + +// cli inputs +DEFINE_string( + calculator_graph_config_file, "", + "Name of file containing text format CalculatorGraphConfig proto."); +DEFINE_string(input_video_path, "", + "Full path of video to load. " + "If not provided, attempt to use a webcam."); +DEFINE_string(output_video_path, "", + "Full path of where to save result (.mp4 only). " + "If not provided, show result in a window."); + +::mediapipe::Status RunMPPGraph() +{ + std::string calculator_graph_config_contents; + MP_RETURN_IF_ERROR(mediapipe::file::GetContents( + FLAGS_calculator_graph_config_file, &calculator_graph_config_contents)); + LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; + mediapipe::CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie( + calculator_graph_config_contents); + + LOG(INFO) << "Initialize the calculator graph."; + mediapipe::CalculatorGraph graph; + MP_RETURN_IF_ERROR(graph.Initialize(config)); + + LOG(INFO) << "Initialize the camera or load the video."; + cv::VideoCapture capture; + const bool load_video = !FLAGS_input_video_path.empty(); + if (load_video) + { + capture.open(FLAGS_input_video_path); + } + else + { + capture.open(0); + } + RET_CHECK(capture.isOpened()); + + cv::VideoWriter writer; + const bool save_video = !FLAGS_output_video_path.empty(); + if (save_video) + { + LOG(INFO) << "Prepare video writer."; + cv::Mat test_frame; + capture.read(test_frame); // Consume first frame. + capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning. + writer.open(FLAGS_output_video_path, + mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 + capture.get(cv::CAP_PROP_FPS), test_frame.size()); + RET_CHECK(writer.isOpened()); + } + else + { + cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); + } + + // pollers to retrieve streams from graph + // output stream (i.e. rendered landmark frame) + + LOG(INFO) << "Start running the calculator graph."; + ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller poller, + graph.AddOutputStreamPoller(kOutputStream)); + ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller poller_landmark, + graph.AddOutputStreamPoller(kLandmarksStream)); + MP_RETURN_IF_ERROR(graph.StartRun({})); + + LOG(INFO) << "Start grabbing and processing frames."; + size_t frame_timestamp = 0; + bool grab_frames = true; + while (grab_frames) + { + // Capture opencv camera or video frame. + cv::Mat camera_frame_raw; + capture >> camera_frame_raw; + if (camera_frame_raw.empty()) + break; // End of video. + cv::Mat camera_frame; + cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); + if (!load_video) + { + cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); + } + + // Wrap Mat into an ImageFrame. + auto input_frame = absl::make_unique( + mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, + mediapipe::ImageFrame::kDefaultAlignmentBoundary); + cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get()); + camera_frame.copyTo(input_frame_mat); + + // Send image packet into the graph. + MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( + kInputStream, mediapipe::Adopt(input_frame.release()) + .At(mediapipe::Timestamp(frame_timestamp++)))); + + // Get the graph result packet, or stop if that fails. + ::mediapipe::Packet packet; + if (!poller.Next(&packet)) + break; + auto &output_frame = packet.Get<::mediapipe::ImageFrame>(); + + // Get the packet containing multi_hand_landmarks. + ::mediapipe::Packet landmarks_packet; + if (!poller_landmark.Next(&landmarks_packet)) + break; + const auto &pose_landmarks = + landmarks_packet.Get< + mediapipe::NormalizedLandmarkList>(); + + LOG(INFO) << "#Pose landmarks: " << pose_landmarks.landmark_size(); + + for (int i = 0; i < pose_landmarks.landmark_size(); ++i) + { + const auto &landmark = pose_landmarks.landmark(i); + LOG(INFO) << "\tLandmark [" << i << "]: (" + << landmark.x() << ", " + << landmark.y() << ", " + << landmark.z() << ")"; + } + + // Use packet.Get to recover values from packet + + // Convert back to opencv for display or saving. + cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame); + cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); + if (save_video) + { + writer.write(output_frame_mat); + } + else + { + cv::imshow(kWindowName, output_frame_mat); + // Press any key to exit. + const int pressed_key = cv::waitKey(5); + if (pressed_key >= 0 && pressed_key != 255) + grab_frames = false; + } + } + + LOG(INFO) << "Shutting down."; + if (writer.isOpened()) + writer.release(); + MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); + return graph.WaitUntilDone(); +} + +int main(int argc, char **argv) +{ + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + ::mediapipe::Status run_status = RunMPPGraph(); + if (!run_status.ok()) + { + LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + } + else + { + LOG(INFO) << "Success!"; + } + return 0; +} \ No newline at end of file diff --git a/mediapipe/examples/desktop/face_mesh/BUILD b/mediapipe/examples/desktop/face_mesh/BUILD index 162cb6b65..1462e9941 100644 --- a/mediapipe/examples/desktop/face_mesh/BUILD +++ b/mediapipe/examples/desktop/face_mesh/BUILD @@ -30,7 +30,16 @@ cc_binary( data = ["//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite"], deps = [ "//mediapipe/examples/desktop:demo_run_graph_main", - "//mediapipe/graphs/face_mesh:desktop_live_calculators", + "//mediapipe/graphs/face_mesh:desktop_calculators", + ], +) + +cc_binary( + name = "face_mesh_out_cpu", + data = ["//mediapipe/modules/face_landmark:face_landmark_with_attention.tflite"], + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main_out_face", + "//mediapipe/graphs/face_mesh:desktop_calculators", ], ) diff --git a/mediapipe/examples/desktop/hand_tracking/demo_run_graph_main_out2.cc b/mediapipe/examples/desktop/hand_tracking/demo_run_graph_main_out2.cc new file mode 100644 index 000000000..f9b3218b1 --- /dev/null +++ b/mediapipe/examples/desktop/hand_tracking/demo_run_graph_main_out2.cc @@ -0,0 +1,176 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// An example of sending OpenCV webcam frames into a MediaPipe graph. + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/port/commandlineflags.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status.h" + +//Take stream from /mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt +// RendererSubgraph - LANDMARKS:hand_landmarks +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" + +// input and output streams to be used/retrieved by calculators +constexpr char kInputStream[] = "input_video"; +constexpr char kOutputStream[] = "output_video"; +// constexpr char kLandmarksStream[] = "hand_landmarks"; +constexpr char kLandmarksStream[] = "landmarks"; +constexpr char kWindowName[] = "MediaPipe"; + +// cli inputs +DEFINE_string( + calculator_graph_config_file, "", + "Name of file containing text format CalculatorGraphConfig proto."); +DEFINE_string(input_video_path, "", + "Full path of video to load. " + "If not provided, attempt to use a webcam."); +DEFINE_string(output_video_path, "", + "Full path of where to save result (.mp4 only). " + "If not provided, show result in a window."); + +::mediapipe::Status RunMPPGraph() { + std::string calculator_graph_config_contents; + MP_RETURN_IF_ERROR(mediapipe::file::GetContents( + FLAGS_calculator_graph_config_file, &calculator_graph_config_contents)); + LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; + mediapipe::CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie( + calculator_graph_config_contents); + + LOG(INFO) << "Initialize the calculator graph."; + mediapipe::CalculatorGraph graph; + MP_RETURN_IF_ERROR(graph.Initialize(config)); + + LOG(INFO) << "Initialize the camera or load the video."; + cv::VideoCapture capture; + const bool load_video = !FLAGS_input_video_path.empty(); + if (load_video) { + capture.open(FLAGS_input_video_path); + } + else { + capture.open(0); + } + RET_CHECK(capture.isOpened()); + + cv::VideoWriter writer; + const bool save_video = !FLAGS_output_video_path.empty(); + if (save_video) { + LOG(INFO) << "Prepare video writer."; + cv::Mat test_frame; + capture.read(test_frame); // Consume first frame. + capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning. + writer.open(FLAGS_output_video_path, + mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 + capture.get(cv::CAP_PROP_FPS), test_frame.size()); + RET_CHECK(writer.isOpened()); + } + else { + cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); + } + + // pollers to retrieve streams from graph + // output stream (i.e. rendered landmark frame) + ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, + graph.AddOutputStreamPoller(kOutputStream)); + // hand landmarks stream + ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller_landmark, + graph.AddOutputStreamPoller(kLandmarksStream)); + + LOG(INFO) << "Start running the calculator graph."; + MP_RETURN_IF_ERROR(graph.StartRun({})); + + LOG(INFO) << "Start grabbing and processing frames."; + size_t frame_timestamp = 0; + bool grab_frames = true; + while (grab_frames) { + // Capture opencv camera or video frame. + cv::Mat camera_frame_raw; + capture >> camera_frame_raw; + if (camera_frame_raw.empty()) break; // End of video. + cv::Mat camera_frame; + cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); + if (!load_video) { + cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); + } + + // Wrap Mat into an ImageFrame. + auto input_frame = absl::make_unique( + mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, + mediapipe::ImageFrame::kDefaultAlignmentBoundary); + cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get()); + camera_frame.copyTo(input_frame_mat); + + // Send image packet into the graph. + MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( + kInputStream, mediapipe::Adopt(input_frame.release()) + .At(mediapipe::Timestamp(frame_timestamp++)))); + + // Get the graph result packet, or stop if that fails. + mediapipe::Packet packet; + mediapipe::Packet landmark_packet; + + //Polling the poller to get landmark packet + if (!poller.Next(&packet)) break; + if (!poller_landmark.Next(&landmark_packet)) break; + + // Use packet.Get to recover values from packet + auto& output_frame = packet.Get(); + auto& output_landmarks = landmark_packet.Get>(); + + // Convert back to opencv for display or saving. + cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame); + cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); + if (save_video) { + writer.write(output_frame_mat); + } + else { + cv::imshow(kWindowName, output_frame_mat); + // Press any key to exit. + const int pressed_key = cv::waitKey(5); + if (pressed_key >= 0 && pressed_key != 255) grab_frames = false; + } + // printout landmark values + for (const ::mediapipe::NormalizedLandmarkList& landmark : output_landmarks) { + std::cout << landmark.DebugString(); + } + } + + LOG(INFO) << "Shutting down."; + if (writer.isOpened()) writer.release(); + MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream)); + return graph.WaitUntilDone(); +} + +int main(int argc, char** argv) { + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + ::mediapipe::Status run_status = RunMPPGraph(); + if (!run_status.ok()) { + LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + } + else { + LOG(INFO) << "Success!"; + } + return 0; +} \ No newline at end of file diff --git a/mediapipe/examples/desktop/holistic_tracking/BUILD b/mediapipe/examples/desktop/holistic_tracking/BUILD index bb5d747cc..a9f807846 100644 --- a/mediapipe/examples/desktop/holistic_tracking/BUILD +++ b/mediapipe/examples/desktop/holistic_tracking/BUILD @@ -31,6 +31,21 @@ cc_binary( ], ) +cc_binary( + name = "holistic_tracking_cpu_out", + data = [ + "//mediapipe/modules/face_landmark:face_landmark.tflite", + "//mediapipe/modules/hand_landmark:hand_landmark_full.tflite", + "//mediapipe/modules/holistic_landmark:hand_recrop.tflite", + "//mediapipe/modules/pose_detection:pose_detection.tflite", + "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite", + ], + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main_out_holistic", + "//mediapipe/graphs/holistic_tracking:holistic_tracking_cpu_graph_deps", + ], +) + # Linux only cc_binary( name = "holistic_tracking_gpu", diff --git a/mediapipe/examples/desktop/multi_hand_tracking/ multi_hand_tracking_run_graph_cpu_main.cc b/mediapipe/examples/desktop/multi_hand_tracking/ multi_hand_tracking_run_graph_cpu_main.cc new file mode 100644 index 000000000..b9b197fc7 --- /dev/null +++ b/mediapipe/examples/desktop/multi_hand_tracking/ multi_hand_tracking_run_graph_cpu_main.cc @@ -0,0 +1,187 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// An example of sending OpenCV webcam frames into a MediaPipe graph. +#include +#include +#include + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/port/commandlineflags.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status.h" + +constexpr char kWindowName[] = "MediaPipe"; +constexpr char kCalculatorGraphConfigFile[] = + "mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt"; +// Input and output streams. +constexpr char kInputStream[] = "input_video"; +constexpr char kOutputStream[] = "output_video"; +constexpr char kMultiHandLandmarksOutputStream[] = "multi_hand_landmarks"; + +DEFINE_string(input_video_path, "", + "Full path of video to load. " + "If not provided, attempt to use a webcam."); +DEFINE_string(output_video_path, "", + "Full path of where to save result (.mp4 only). " + "If not provided, show result in a window."); + +::mediapipe::Status RunMPPGraph( + std::unique_ptr<::mediapipe::CalculatorGraph> graph) { + + LOG(INFO) << "Initialize the camera or load the video."; + cv::VideoCapture capture; + const bool load_video = !FLAGS_input_video_path.empty(); + if (load_video) { + capture.open(FLAGS_input_video_path); + } else { + capture.open(0); + } + RET_CHECK(capture.isOpened()); + + cv::VideoWriter writer; + const bool save_video = !FLAGS_output_video_path.empty(); + if (!save_video) { + cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); +#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2) + capture.set(cv::CAP_PROP_FRAME_WIDTH, 640); + capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480); + capture.set(cv::CAP_PROP_FPS, 30); +#endif + } + + LOG(INFO) << "Start running the calculator graph."; + ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller poller, + graph->AddOutputStreamPoller(kOutputStream)); + ASSIGN_OR_RETURN(::mediapipe::OutputStreamPoller multi_hand_landmarks_poller, + graph->AddOutputStreamPoller(kMultiHandLandmarksOutputStream)); + MP_RETURN_IF_ERROR(graph->StartRun({})); + + LOG(INFO) << "Start grabbing and processing frames."; + bool grab_frames = true; + while (grab_frames) { + // Capture opencv camera or video frame. + cv::Mat camera_frame_raw; + capture >> camera_frame_raw; + if (camera_frame_raw.empty()) break; // End of video. + cv::Mat camera_frame; + cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); + if (!load_video) { + cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); + } + + // Wrap Mat into an ImageFrame. + auto input_frame = absl::make_unique<::mediapipe::ImageFrame>( + ::mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, + ::mediapipe::ImageFrame::kDefaultAlignmentBoundary); + cv::Mat input_frame_mat = ::mediapipe::formats::MatView(input_frame.get()); + camera_frame.copyTo(input_frame_mat); + + // Send image packet into the graph. + size_t frame_timestamp_us = + (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6; + MP_RETURN_IF_ERROR(graph->AddPacketToInputStream( + kInputStream, ::mediapipe::Adopt(input_frame.release()) + .At(::mediapipe::Timestamp(frame_timestamp_us)))); + + // Get the graph result packet, or stop if that fails. + ::mediapipe::Packet packet; + if (!poller.Next(&packet)) break; + auto& output_frame = packet.Get<::mediapipe::ImageFrame>(); + + // Get the packet containing multi_hand_landmarks. + ::mediapipe::Packet multi_hand_landmarks_packet; + if (!multi_hand_landmarks_poller.Next(&multi_hand_landmarks_packet)) break; + const auto& multi_hand_landmarks = + multi_hand_landmarks_packet.Get< + std::vector<::mediapipe::NormalizedLandmarkList>>(); + + LOG(INFO) << "#Multi Hand landmarks: " << multi_hand_landmarks.size(); + int hand_id = 0; + for (const auto& single_hand_landmarks: multi_hand_landmarks) { + ++hand_id; + LOG(INFO) << "Hand [" << hand_id << "]:"; + for (int i = 0; i < single_hand_landmarks.landmark_size(); ++i) { + const auto& landmark = single_hand_landmarks.landmark(i); + LOG(INFO) << "\tLandmark [" << i << "]: (" + << landmark.x() << ", " + << landmark.y() << ", " + << landmark.z() << ")"; + } + } + + // Convert back to opencv for display or saving. + cv::Mat output_frame_mat = ::mediapipe::formats::MatView(&output_frame); + cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR); + if (save_video) { + if (!writer.isOpened()) { + LOG(INFO) << "Prepare video writer."; + writer.open(FLAGS_output_video_path, + ::mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4 + capture.get(cv::CAP_PROP_FPS), output_frame_mat.size()); + RET_CHECK(writer.isOpened()); + } + writer.write(output_frame_mat); + } else { + cv::imshow(kWindowName, output_frame_mat); + // Press any key to exit. + const int pressed_key = cv::waitKey(5); + if (pressed_key >= 0 && pressed_key != 255) grab_frames = false; + } + } + + LOG(INFO) << "Shutting down."; + if (writer.isOpened()) writer.release(); + MP_RETURN_IF_ERROR(graph->CloseInputStream(kInputStream)); + return graph->WaitUntilDone(); +} + +::mediapipe::Status InitializeAndRunMPPGraph() { + + std::string calculator_graph_config_contents; + MP_RETURN_IF_ERROR(::mediapipe::file::GetContents( + kCalculatorGraphConfigFile, &calculator_graph_config_contents)); + LOG(INFO) << "Get calculator graph config contents: " + << calculator_graph_config_contents; + mediapipe::CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie( + calculator_graph_config_contents); + + LOG(INFO) << "Initialize the calculator graph."; + std::unique_ptr<::mediapipe::CalculatorGraph> graph = + absl::make_unique<::mediapipe::CalculatorGraph>(); + MP_RETURN_IF_ERROR(graph->Initialize(config)); + + return RunMPPGraph(std::move(graph)); +} + +int main(int argc, char** argv) { + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + ::mediapipe::Status run_status = InitializeAndRunMPPGraph(); + if (!run_status.ok()) { + LOG(ERROR) << "Failed to run the graph: " << run_status.message(); + return EXIT_FAILURE; + } else { + LOG(INFO) << "Success!"; + } + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/mediapipe/examples/desktop/multi_hand_tracking/BUILD b/mediapipe/examples/desktop/multi_hand_tracking/BUILD new file mode 100644 index 000000000..919e31a2c --- /dev/null +++ b/mediapipe/examples/desktop/multi_hand_tracking/BUILD @@ -0,0 +1,36 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) + +package(default_visibility = [ + "//visibility:public", +]) + +cc_binary( + name = "multi_hand_tracking_cpu", + srcs = ["multi_hand_tracking_run_graph_cpu_main.cc"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:commandlineflags", + "//mediapipe/framework/port:file_helpers", + "//mediapipe/framework/port:opencv_highgui", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_video", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + ], +) \ No newline at end of file diff --git a/mediapipe/examples/desktop/pose_tracking/BUILD b/mediapipe/examples/desktop/pose_tracking/BUILD index 06f790264..fa735b95f 100644 --- a/mediapipe/examples/desktop/pose_tracking/BUILD +++ b/mediapipe/examples/desktop/pose_tracking/BUILD @@ -28,6 +28,19 @@ cc_binary( ], ) +cc_binary( + name = "pose_tracking_out_cpu", + data = [ + "//mediapipe/modules/pose_detection:pose_detection.tflite", + "//mediapipe/modules/pose_landmark:pose_landmark_full.tflite", + ], + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main_out_pose", + "//mediapipe/graphs/pose_tracking:pose_tracking_cpu_deps", + ], +) + + # Linux only cc_binary( name = "pose_tracking_gpu", diff --git a/mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt b/mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt index 2cc563424..75475ad2b 100644 --- a/mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt +++ b/mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt @@ -63,4 +63,4 @@ node { input_stream: "NORM_RECTS:face_rects_from_landmarks" input_stream: "DETECTIONS:face_detections" output_stream: "IMAGE:output_video" -} +} \ No newline at end of file diff --git a/mediapipe/modules/face_detection/face_detection_short_range.tflite b/mediapipe/modules/face_detection/face_detection_short_range.tflite new file mode 100644 index 000000000..659bce896 Binary files /dev/null and b/mediapipe/modules/face_detection/face_detection_short_range.tflite differ diff --git a/mediapipe/modules/face_landmark/face_landmark.tflite b/mediapipe/modules/face_landmark/face_landmark.tflite new file mode 100644 index 000000000..277d26f2f Binary files /dev/null and b/mediapipe/modules/face_landmark/face_landmark.tflite differ diff --git a/mediapipe/modules/face_landmark/face_landmark_with_attention.tflite b/mediapipe/modules/face_landmark/face_landmark_with_attention.tflite new file mode 100644 index 000000000..fe0a93a85 Binary files /dev/null and b/mediapipe/modules/face_landmark/face_landmark_with_attention.tflite differ diff --git a/mediapipe/modules/hand_landmark/hand_landmark_full.tflite b/mediapipe/modules/hand_landmark/hand_landmark_full.tflite new file mode 100644 index 000000000..01783cfd2 Binary files /dev/null and b/mediapipe/modules/hand_landmark/hand_landmark_full.tflite differ diff --git a/mediapipe/modules/holistic_landmark/hand_recrop.tflite b/mediapipe/modules/holistic_landmark/hand_recrop.tflite new file mode 100644 index 000000000..dcfd276cb Binary files /dev/null and b/mediapipe/modules/holistic_landmark/hand_recrop.tflite differ diff --git a/mediapipe/modules/palm_detection/palm_detection_full.tflite b/mediapipe/modules/palm_detection/palm_detection_full.tflite new file mode 100755 index 000000000..d37cd03c3 Binary files /dev/null and b/mediapipe/modules/palm_detection/palm_detection_full.tflite differ diff --git a/mediapipe/modules/pose_detection/pose_detection.tflite b/mediapipe/modules/pose_detection/pose_detection.tflite new file mode 100644 index 000000000..de8a158d3 Binary files /dev/null and b/mediapipe/modules/pose_detection/pose_detection.tflite differ diff --git a/mediapipe/modules/pose_landmark/pose_landmark_full.tflite b/mediapipe/modules/pose_landmark/pose_landmark_full.tflite new file mode 100644 index 000000000..e2ee84fc4 Binary files /dev/null and b/mediapipe/modules/pose_landmark/pose_landmark_full.tflite differ diff --git a/setup_opencv.sh b/setup_opencv.sh old mode 100644 new mode 100755 diff --git a/third_party/opencv_linux.BUILD b/third_party/opencv_linux.BUILD index 844585541..668d058cc 100644 --- a/third_party/opencv_linux.BUILD +++ b/third_party/opencv_linux.BUILD @@ -28,6 +28,9 @@ cc_library( #"include/opencv4/", ], linkopts = [ + "-L/usr/local/lib", + "-L/usr/local/lib", + "-L/usr/local/lib", "-l:libopencv_core.so", "-l:libopencv_calib3d.so", "-l:libopencv_features2d.so",