From 19983e3bd924c6547cc7e2728d0623107a135554 Mon Sep 17 00:00:00 2001 From: dom607 Date: Thu, 2 Jul 2020 13:57:54 +0900 Subject: [PATCH 1/5] Add counting_vector_size_calculator --- mediapipe/calculators/util/BUILD | 14 ++++ .../util/counting_vector_size_calculator.cc | 27 ++++++++ .../util/counting_vector_size_calculator.h | 69 +++++++++++++++++++ mediapipe/examples/desktop/BUILD | 1 + .../examples/desktop/demo_run_graph_main.cc | 41 +++++++++-- .../face_mesh/face_mesh_desktop_live.pbtxt | 2 + mediapipe/modules/face_landmark/BUILD | 1 + .../face_landmark_front_cpu.pbtxt | 10 +++ 8 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 mediapipe/calculators/util/counting_vector_size_calculator.cc create mode 100644 mediapipe/calculators/util/counting_vector_size_calculator.h diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD index b570e4ca2..ddbd7a643 100644 --- a/mediapipe/calculators/util/BUILD +++ b/mediapipe/calculators/util/BUILD @@ -311,6 +311,20 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "counting_vector_size_calculator", + srcs = ["counting_vector_size_calculator.cc"], + hdrs = ["counting_vector_size_calculator.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:landmark_cc_proto", + ], + alwayslink = 1, +) + cc_library( name = "annotation_overlay_calculator", srcs = ["annotation_overlay_calculator.cc"], diff --git a/mediapipe/calculators/util/counting_vector_size_calculator.cc b/mediapipe/calculators/util/counting_vector_size_calculator.cc new file mode 100644 index 000000000..d483240eb --- /dev/null +++ b/mediapipe/calculators/util/counting_vector_size_calculator.cc @@ -0,0 +1,27 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/util/counting_vector_size_calculator.h" + +#include "mediapipe/framework/formats/landmark.pb.h" + +namespace mediapipe { + +typedef CountingVectorSizeCalculator> + CountingNormalizedLandmarkListVectorSizeCalculator; + +REGISTER_CALCULATOR(CountingNormalizedLandmarkListVectorSizeCalculator); +} // namespace mediapipe + + diff --git a/mediapipe/calculators/util/counting_vector_size_calculator.h b/mediapipe/calculators/util/counting_vector_size_calculator.h new file mode 100644 index 000000000..51628113e --- /dev/null +++ b/mediapipe/calculators/util/counting_vector_size_calculator.h @@ -0,0 +1,69 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H +#define MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/landmark.pb.h" + +namespace mediapipe { + +// A calculator that count input landmarksList size. +// +// Count input landmark(std::vector) and return this +// value to ouput_stream. Input IMAGE has no effect on calculation, but is used to +// ensure that the calculator works even when the landmark is empty. And if the +// input landmark is empty, the number of faces found is zero. +// +// Example config: +// node { +// calculator: "CountingVectorSizeCalculator" +// input_stream: "IMAGE:input_image" +// input_stream: "LANDMARKS:multi_face_landmarks" +// output_stream: "COUNT:face_count" +// } + +template +class CountingVectorSizeCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + // Check tag. + RET_CHECK(cc->Inputs().HasTag("CLOCK")); + cc->Inputs().Tag("CLOCK").SetAny(); + RET_CHECK(cc->Inputs().HasTag("VECTOR")); + cc->Inputs().Tag("VECTOR").Set(); + RET_CHECK(cc->Outputs().HasTag("COUNT")); + cc->Outputs().Tag("COUNT").Set(); + + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) { + std::unique_ptr face_count; + if (!cc->Inputs().Tag("VECTOR").IsEmpty()) { + const auto& landmarks = cc->Inputs().Tag("VECTOR").Get(); + face_count = absl::make_unique(landmarks.size()); + } else { + face_count = absl::make_unique(0); + } + cc->Outputs().Tag("COUNT").Add(face_count.release(), cc->InputTimestamp()); + + return ::mediapipe::OkStatus(); + }; +}; + +} + +#endif // MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H \ No newline at end of file diff --git a/mediapipe/examples/desktop/BUILD b/mediapipe/examples/desktop/BUILD index 0e0335157..78476501c 100644 --- a/mediapipe/examples/desktop/BUILD +++ b/mediapipe/examples/desktop/BUILD @@ -41,6 +41,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/port:commandlineflags", "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:opencv_highgui", diff --git a/mediapipe/examples/desktop/demo_run_graph_main.cc b/mediapipe/examples/desktop/demo_run_graph_main.cc index 25f4bb4f1..3176c393c 100644 --- a/mediapipe/examples/desktop/demo_run_graph_main.cc +++ b/mediapipe/examples/desktop/demo_run_graph_main.cc @@ -18,6 +18,7 @@ #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/port/commandlineflags.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/opencv_highgui_inc.h" @@ -26,8 +27,11 @@ #include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/status.h" + constexpr char kInputStream[] = "input_video"; -constexpr char kOutputStream[] = "output_video"; +constexpr char kOutputImageStream[] = "output_video"; +constexpr char kOutputFaceCountStream[] = "face_count"; +constexpr char kOutputLandmarksStream[] = "multi_face_landmarks"; constexpr char kWindowName[] = "MediaPipe"; DEFINE_string( @@ -76,12 +80,17 @@ DEFINE_string(output_video_path, "", } LOG(INFO) << "Start running the calculator graph."; - ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, - graph.AddOutputStreamPoller(kOutputStream)); + ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller image_poller, + graph.AddOutputStreamPoller(kOutputImageStream)); + ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller face_count_poller, + graph.AddOutputStreamPoller(kOutputFaceCountStream)); + ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller landmarks_poller, + graph.AddOutputStreamPoller(kOutputLandmarksStream)); MP_RETURN_IF_ERROR(graph.StartRun({})); LOG(INFO) << "Start grabbing and processing frames."; bool grab_frames = true; + int prev_face_count = -1; while (grab_frames) { // Capture opencv camera or video frame. cv::Mat camera_frame_raw; @@ -108,9 +117,29 @@ DEFINE_string(output_video_path, "", .At(mediapipe::Timestamp(frame_timestamp_us)))); // Get the graph result packet, or stop if that fails. - mediapipe::Packet packet; - if (!poller.Next(&packet)) break; - auto& output_frame = packet.Get(); + mediapipe::Packet image_packet; + if (!image_poller.Next(&image_packet)) break; + auto& output_frame = image_packet.Get(); + + mediapipe::Packet face_count_packet; + if (!face_count_poller.Next(&face_count_packet)) break; + auto& face_count = face_count_packet.Get(); + + if (face_count != prev_face_count) { + LOG(INFO) << "Found face count : " << face_count; + prev_face_count = face_count; + + if (face_count != 0) { + mediapipe::Packet landmarks_packet; + if (!landmarks_poller.Next(&landmarks_packet)) break; + auto& landmarks = landmarks_packet.Get>(); + + LOG(INFO) << "Landmarks size : " << landmarks[0].landmark_size(); + + // Do something with landmarks + } + } + // Convert back to opencv for display or saving. cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame); diff --git a/mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt b/mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt index 57654436a..f4e11023c 100644 --- a/mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt +++ b/mediapipe/graphs/face_mesh/face_mesh_desktop_live.pbtxt @@ -8,6 +8,7 @@ output_stream: "output_video" # Collection of detected/processed faces, each represented as a list of # landmarks. (std::vector) output_stream: "multi_face_landmarks" +output_stream: "face_count" # Throttles the images flowing downstream for flow control. It passes through # the very first incoming image unaltered, and waits for downstream nodes @@ -50,6 +51,7 @@ node { output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" output_stream: "DETECTIONS:face_detections" output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" + output_stream: "FACE_COUNT_FROM_LANDMARKS:face_count" } # Subgraph that renders face-landmark annotation onto the input image. diff --git a/mediapipe/modules/face_landmark/BUILD b/mediapipe/modules/face_landmark/BUILD index 3dd41ecb4..5fff8dab5 100644 --- a/mediapipe/modules/face_landmark/BUILD +++ b/mediapipe/modules/face_landmark/BUILD @@ -75,6 +75,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/util:association_norm_rect_calculator", "//mediapipe/calculators/util:collection_has_min_size_calculator", "//mediapipe/modules/face_detection:face_detection_front_cpu", + "//mediapipe/calculators/util:counting_vector_size_calculator", ], ) diff --git a/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt index cdd8a03f2..573737d52 100644 --- a/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt +++ b/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt @@ -43,6 +43,8 @@ output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" # Regions of interest calculated based on face detections. # (std::vector) output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" +# (int) +output_stream: "FACE_COUNT_FROM_LANDMARKS:face_count" # Determines if an input vector of NormalizedRect has a size greater than or # equal to the provided num_faces. @@ -189,6 +191,14 @@ node { output_stream: "ITERABLE:multi_face_landmarks" } +node { + calculator: "CountingNormalizedLandmarkListVectorSizeCalculator" + input_stream: "CLOCK:image" + input_stream: "VECTOR:multi_face_landmarks" + output_stream: "COUNT:face_count" +} + + # Collects a NormalizedRect for each face into a vector. Upon receiving the # BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END # timestamp. From 1641986a7b0dcf9a6e8dc7bd153b3b4d475b9b3f Mon Sep 17 00:00:00 2001 From: dom607 Date: Thu, 2 Jul 2020 14:42:32 +0900 Subject: [PATCH 2/5] Change calculator's behaviour - Now clock input stream is optional - If there are clock stream, always emit count. If not, emit count when vector stream is come in. --- .../calculators/util/counting_vector_size_calculator.h | 7 ++++--- mediapipe/modules/face_landmark/BUILD | 3 ++- .../modules/face_landmark/face_landmark_front_gpu.pbtxt | 9 +++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/mediapipe/calculators/util/counting_vector_size_calculator.h b/mediapipe/calculators/util/counting_vector_size_calculator.h index 51628113e..7b49750b4 100644 --- a/mediapipe/calculators/util/counting_vector_size_calculator.h +++ b/mediapipe/calculators/util/counting_vector_size_calculator.h @@ -39,9 +39,10 @@ template class CountingVectorSizeCalculator : public CalculatorBase { public: static ::mediapipe::Status GetContract(CalculatorContract* cc) { - // Check tag. - RET_CHECK(cc->Inputs().HasTag("CLOCK")); - cc->Inputs().Tag("CLOCK").SetAny(); + if (cc->Inputs().HasTag("CLOCK")) { + cc->Inputs().Tag("CLOCK").SetAny(); + } + RET_CHECK(cc->Inputs().HasTag("VECTOR")); cc->Inputs().Tag("VECTOR").Set(); RET_CHECK(cc->Outputs().HasTag("COUNT")); diff --git a/mediapipe/modules/face_landmark/BUILD b/mediapipe/modules/face_landmark/BUILD index 5fff8dab5..88555ad07 100644 --- a/mediapipe/modules/face_landmark/BUILD +++ b/mediapipe/modules/face_landmark/BUILD @@ -74,8 +74,8 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/image:image_properties_calculator", "//mediapipe/calculators/util:association_norm_rect_calculator", "//mediapipe/calculators/util:collection_has_min_size_calculator", - "//mediapipe/modules/face_detection:face_detection_front_cpu", "//mediapipe/calculators/util:counting_vector_size_calculator", + "//mediapipe/modules/face_detection:face_detection_front_cpu", ], ) @@ -96,6 +96,7 @@ mediapipe_simple_subgraph( "//mediapipe/calculators/image:image_properties_calculator", "//mediapipe/calculators/util:association_norm_rect_calculator", "//mediapipe/calculators/util:collection_has_min_size_calculator", + "//mediapipe/calculators/util:counting_vector_size_calculator", "//mediapipe/modules/face_detection:face_detection_front_gpu", ], ) diff --git a/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt index d06aff1df..dde7048e2 100644 --- a/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt +++ b/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt @@ -43,6 +43,8 @@ output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" # Regions of interest calculated based on face detections. # (std::vector) output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" +# (int) +output_stream: "FACE_COUNT_FROM_LANDMARKS:face_count" # Determines if an input vector of NormalizedRect has a size greater than or # equal to the provided num_faces. @@ -189,6 +191,13 @@ node { output_stream: "ITERABLE:multi_face_landmarks" } +node { + calculator: "CountingNormalizedLandmarkListVectorSizeCalculator" + input_stream: "CLOCK:image" + input_stream: "VECTOR:multi_face_landmarks" + output_stream: "COUNT:face_count" +} + # Collects a NormalizedRect for each face into a vector. Upon receiving the # BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END # timestamp. From 74a81c581c1f7218d00ca3d4d1a3d4ccf3f06860 Mon Sep 17 00:00:00 2001 From: dom607 Date: Thu, 2 Jul 2020 15:12:43 +0900 Subject: [PATCH 3/5] Add comment. --- .../util/counting_vector_size_calculator.h | 27 ++++++++++++------- .../examples/desktop/demo_run_graph_main.cc | 1 - .../face_landmark_front_cpu.pbtxt | 3 ++- .../face_landmark_front_gpu.pbtxt | 2 ++ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/mediapipe/calculators/util/counting_vector_size_calculator.h b/mediapipe/calculators/util/counting_vector_size_calculator.h index 7b49750b4..f882b3296 100644 --- a/mediapipe/calculators/util/counting_vector_size_calculator.h +++ b/mediapipe/calculators/util/counting_vector_size_calculator.h @@ -20,19 +20,28 @@ namespace mediapipe { -// A calculator that count input landmarksList size. +// A calculator that counts the size of the input vector. It was created to +// aid in polling packets in the output stream synchronously. If there is +// a clock stream, it will output a value of 0 even if the input vector stream +// is empty. If not, it will output some value only if there is an input vector. +// The clock stream has the same time stamp as the input time stamp, and +// it must be a stream where packets must be transmitted during graph operation. +// (e.g. Any input stream of graph) // -// Count input landmark(std::vector) and return this -// value to ouput_stream. Input IMAGE has no effect on calculation, but is used to -// ensure that the calculator works even when the landmark is empty. And if the -// input landmark is empty, the number of faces found is zero. +// It is designed to be used like: // // Example config: // node { -// calculator: "CountingVectorSizeCalculator" -// input_stream: "IMAGE:input_image" -// input_stream: "LANDMARKS:multi_face_landmarks" -// output_stream: "COUNT:face_count" +// calculator: "CountingWithVectorSizeCalculator" +// input_stream: "CLOCK:triger_signal" +// input_stream: "VECTOR:input_vector" +// output_stream: "COUNT:vector_count" +// } +// +// node { +// calculator: "CountingWithVectorSizeCalculator" +// input_stream: "VECTOR:input_vector" +// output_stream: "COUNT:vector_count" // } template diff --git a/mediapipe/examples/desktop/demo_run_graph_main.cc b/mediapipe/examples/desktop/demo_run_graph_main.cc index 3176c393c..7d03172e6 100644 --- a/mediapipe/examples/desktop/demo_run_graph_main.cc +++ b/mediapipe/examples/desktop/demo_run_graph_main.cc @@ -27,7 +27,6 @@ #include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/status.h" - constexpr char kInputStream[] = "input_video"; constexpr char kOutputImageStream[] = "output_video"; constexpr char kOutputFaceCountStream[] = "face_count"; diff --git a/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt index 573737d52..815fa8ca6 100644 --- a/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt +++ b/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt @@ -191,6 +191,8 @@ node { output_stream: "ITERABLE:multi_face_landmarks" } +# Counting a multi_faceLandmarks vector size. The image stream is only used to +# make the calculator work even when there is no input vector. node { calculator: "CountingNormalizedLandmarkListVectorSizeCalculator" input_stream: "CLOCK:image" @@ -198,7 +200,6 @@ node { output_stream: "COUNT:face_count" } - # Collects a NormalizedRect for each face into a vector. Upon receiving the # BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END # timestamp. diff --git a/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt index dde7048e2..3426919d5 100644 --- a/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt +++ b/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt @@ -191,6 +191,8 @@ node { output_stream: "ITERABLE:multi_face_landmarks" } +# Counting a multi_faceLandmarks vector size. The image stream is only used to +# make the calculator work even when there is no input vector. node { calculator: "CountingNormalizedLandmarkListVectorSizeCalculator" input_stream: "CLOCK:image" From 1c4cf380d3afe8822046bcb16f46ef94d71965c0 Mon Sep 17 00:00:00 2001 From: dom607 Date: Thu, 2 Jul 2020 16:28:58 +0900 Subject: [PATCH 4/5] Revised comments to clarify meaning. --- mediapipe/calculators/util/counting_vector_size_calculator.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/calculators/util/counting_vector_size_calculator.h b/mediapipe/calculators/util/counting_vector_size_calculator.h index f882b3296..12bcbe548 100644 --- a/mediapipe/calculators/util/counting_vector_size_calculator.h +++ b/mediapipe/calculators/util/counting_vector_size_calculator.h @@ -24,8 +24,8 @@ namespace mediapipe { // aid in polling packets in the output stream synchronously. If there is // a clock stream, it will output a value of 0 even if the input vector stream // is empty. If not, it will output some value only if there is an input vector. -// The clock stream has the same time stamp as the input time stamp, and -// it must be a stream where packets must be transmitted during graph operation. +// The clock stream must have the same time stamp as the vector stream, and +// it must be the stream where packets are transmitted while the graph is running. // (e.g. Any input stream of graph) // // It is designed to be used like: From b74df9ce15e64ba46442d78aacf36e397e9d0c96 Mon Sep 17 00:00:00 2001 From: dom607 Date: Thu, 23 Jul 2020 11:47:22 +0900 Subject: [PATCH 5/5] Revert demo_run_graph_main.cc changes. --- .../examples/desktop/demo_run_graph_main.cc | 40 +++---------------- 1 file changed, 6 insertions(+), 34 deletions(-) diff --git a/mediapipe/examples/desktop/demo_run_graph_main.cc b/mediapipe/examples/desktop/demo_run_graph_main.cc index 7d03172e6..25f4bb4f1 100644 --- a/mediapipe/examples/desktop/demo_run_graph_main.cc +++ b/mediapipe/examples/desktop/demo_run_graph_main.cc @@ -18,7 +18,6 @@ #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" -#include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/port/commandlineflags.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/opencv_highgui_inc.h" @@ -28,9 +27,7 @@ #include "mediapipe/framework/port/status.h" constexpr char kInputStream[] = "input_video"; -constexpr char kOutputImageStream[] = "output_video"; -constexpr char kOutputFaceCountStream[] = "face_count"; -constexpr char kOutputLandmarksStream[] = "multi_face_landmarks"; +constexpr char kOutputStream[] = "output_video"; constexpr char kWindowName[] = "MediaPipe"; DEFINE_string( @@ -79,17 +76,12 @@ DEFINE_string(output_video_path, "", } LOG(INFO) << "Start running the calculator graph."; - ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller image_poller, - graph.AddOutputStreamPoller(kOutputImageStream)); - ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller face_count_poller, - graph.AddOutputStreamPoller(kOutputFaceCountStream)); - ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller landmarks_poller, - graph.AddOutputStreamPoller(kOutputLandmarksStream)); + ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, + graph.AddOutputStreamPoller(kOutputStream)); MP_RETURN_IF_ERROR(graph.StartRun({})); LOG(INFO) << "Start grabbing and processing frames."; bool grab_frames = true; - int prev_face_count = -1; while (grab_frames) { // Capture opencv camera or video frame. cv::Mat camera_frame_raw; @@ -116,29 +108,9 @@ DEFINE_string(output_video_path, "", .At(mediapipe::Timestamp(frame_timestamp_us)))); // Get the graph result packet, or stop if that fails. - mediapipe::Packet image_packet; - if (!image_poller.Next(&image_packet)) break; - auto& output_frame = image_packet.Get(); - - mediapipe::Packet face_count_packet; - if (!face_count_poller.Next(&face_count_packet)) break; - auto& face_count = face_count_packet.Get(); - - if (face_count != prev_face_count) { - LOG(INFO) << "Found face count : " << face_count; - prev_face_count = face_count; - - if (face_count != 0) { - mediapipe::Packet landmarks_packet; - if (!landmarks_poller.Next(&landmarks_packet)) break; - auto& landmarks = landmarks_packet.Get>(); - - LOG(INFO) << "Landmarks size : " << landmarks[0].landmark_size(); - - // Do something with landmarks - } - } - + mediapipe::Packet packet; + if (!poller.Next(&packet)) break; + auto& output_frame = packet.Get(); // Convert back to opencv for display or saving. cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame);