cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Avoid void* for the sake of forward declaration

Browse Source
This commit is contained in:
Maksym Walczak 2022-01-04 13:52:01 +01:00
parent 281199e278
commit 092e1ad899
2 changed files with 192 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

243
mediapipe/pose_tracking_dll/pose_tracking.cpp
View File

@ -1,14 +1,14 @@
#include "pose_tracking.h"
#include <cstdlib> #include <cstdlib>
#include <string> #include <string>
#include "pose_tracking.h"
#include "absl/flags/flag.h" #include "absl/flags/flag.h"
#include "absl/flags/parse.h" #include "absl/flags/parse.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h" #include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h"
@ -17,160 +17,147 @@
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
class PoseTrackingImpl { class PoseTrackingImpl {
public: public:
PoseTrackingImpl(const std::string& calculatorGraphConfigFile) { PoseTrackingImpl(const std::string& calculatorGraphConfigFile) {
auto status = initialize(calculatorGraphConfigFile); auto status = initialize(calculatorGraphConfigFile);
if (!status.ok()) { if (!status.ok()) {
LOG(WARNING) << "Warning: " << status; LOG(WARNING) << "Warning: " << status;
} }
} }
absl::Status initialize(const std::string& calculatorGraphConfigFile) { absl::Status initialize(const std::string& calculatorGraphConfigFile) {
std::string graphContents; std::string graphContents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents( MP_RETURN_IF_ERROR(mediapipe::file::GetContents(calculatorGraphConfigFile, &graphContents));
calculatorGraphConfigFile,
&graphContents));
mediapipe::CalculatorGraphConfig config = mediapipe::CalculatorGraphConfig config =
mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>( mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(graphContents);
graphContents);
MP_RETURN_IF_ERROR(graph.Initialize(config)); MP_RETURN_IF_ERROR(graph.Initialize(config));
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller, ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller,
graph.AddOutputStreamPoller(kOutputSegmentationStream)); graph.AddOutputStreamPoller(kOutputSegmentationStream));
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller landmarksPoller, ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller landmarksPoller,
graph.AddOutputStreamPoller(kOutpuLandmarksStream)); graph.AddOutputStreamPoller(kOutpuLandmarksStream));
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller posePresencePoller, ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller posePresencePoller,
graph.AddOutputStreamPoller(kOutpuPosePresenceStream)); graph.AddOutputStreamPoller(kOutpuPosePresenceStream));
maskPollerPtr = std::make_unique<mediapipe::OutputStreamPoller>(std::move(poller));
maskPollerPtr = std::make_unique<mediapipe::OutputStreamPoller>(std::move(poller)); landmarksPollerPtr =
std::make_unique<mediapipe::OutputStreamPoller>(std::move(landmarksPoller));
landmarksPollerPtr = std::make_unique<mediapipe::OutputStreamPoller>( posePresencePollerPtr =
std::move(landmarksPoller)); std::make_unique<mediapipe::OutputStreamPoller>(std::move(posePresencePoller));
posePresencePollerPtr = std::make_unique<mediapipe::OutputStreamPoller>( MP_RETURN_IF_ERROR(graph.StartRun({}));
std::move(posePresencePoller)); }
MP_RETURN_IF_ERROR(graph.StartRun({})); bool processFrame(const cv::Mat& inputRGB8Bit) {
} // Wrap Mat into an ImageFrame.
auto inputFrame = absl::make_unique<mediapipe::ImageFrame>(
mediapipe::ImageFormat::SRGB, inputRGB8Bit.cols, inputRGB8Bit.rows,
mediapipe::ImageFrame::kDefaultAlignmentBoundary);
cv::Mat inputFrameMat = mediapipe::formats::MatView(inputFrame.get());
inputRGB8Bit.copyTo(inputFrameMat);
bool processFrame(const cv::Mat& inputRGB8Bit) { // Send image packet into the graph.
// Wrap Mat into an ImageFrame. size_t frameTimestampUs =
auto inputFrame = absl::make_unique<mediapipe::ImageFrame>( static_cast<double>(cv::getTickCount()) / static_cast<double>(cv::getTickFrequency()) * 1e6;
mediapipe::ImageFormat::SRGB, inputRGB8Bit.cols, inputRGB8Bit.rows, auto status = graph.AddPacketToInputStream(
mediapipe::ImageFrame::kDefaultAlignmentBoundary); kInputStream,
cv::Mat inputFrameMat = mediapipe::formats::MatView(inputFrame.get()); mediapipe::Adopt(inputFrame.release()).At(mediapipe::Timestamp(frameTimestampUs)));
inputRGB8Bit.copyTo(inputFrameMat);
// Send image packet into the graph. if (!status.ok()) {
size_t frameTimestampUs = LOG(WARNING) << "Graph execution failed: " << status;
static_cast<double>(cv::getTickCount()) / static_cast<double>(cv::getTickFrequency()) * 1e6; return false;
auto status = graph.AddPacketToInputStream( }
kInputStream, mediapipe::Adopt(inputFrame.release())
.At(mediapipe::Timestamp(frameTimestampUs)));
if (!status.ok()) { mediapipe::Packet posePresencePacket;
LOG(WARNING) << "Graph execution failed: " << status; if (!posePresencePollerPtr || !posePresencePollerPtr->Next(&posePresencePacket)) return false;
return false; auto landmarksDetected = posePresencePacket.Get<bool>();
}
mediapipe::Packet posePresencePacket; if (!landmarksDetected) {
if (!posePresencePollerPtr || !posePresencePollerPtr->Next(&posePresencePacket)) return false; return false;
auto landmarksDetected = posePresencePacket.Get<bool>(); }
if (!landmarksDetected) { // Get the graph result packet, or stop if that fails.
return false; mediapipe::Packet maskPacket;
} if (!maskPollerPtr || !maskPollerPtr->Next(&maskPacket)) return false;
auto& outputFrame = maskPacket.Get<mediapipe::ImageFrame>();
// Get the graph result packet, or stop if that fails. // Get pose landmarks.
mediapipe::Packet maskPacket; if (!landmarksPollerPtr || !landmarksPollerPtr->Next(&poseLandmarksPacket)) {
if (!maskPollerPtr || !maskPollerPtr->Next(&maskPacket)) return false; return false;
auto& outputFrame = maskPacket.Get<mediapipe::ImageFrame>(); }
// Get pose landmarks. // Convert back to opencv for display or saving.
if (!landmarksPollerPtr || auto mask = mediapipe::formats::MatView(&outputFrame);
!landmarksPollerPtr->Next(&poseLandmarksPacket)) { segmentedMask = mask.clone();
return false;
}
// Convert back to opencv for display or saving. absl::Status landmarksStatus = detectLandmarksWithStatus(poseLandmarks);
auto mask = mediapipe::formats::MatView(&outputFrame);
segmentedMask = mask.clone();
absl::Status landmarksStatus = detectLandmarksWithStatus(poseLandmarks); return landmarksStatus.ok();
}
return landmarksStatus.ok(); absl::Status detectLandmarksWithStatus(nimagna::cv_wrapper::Point3f* poseLandmarks) {
} if (poseLandmarksPacket.IsEmpty()) {
return absl::CancelledError("Pose landmarks packet is empty.");
}
absl::Status detectLandmarksWithStatus( auto retrievedLandmarks = poseLandmarksPacket.Get<::mediapipe::NormalizedLandmarkList>();
nimagna::cv_wrapper::Point3f* poseLandmarks) {
if (poseLandmarksPacket.IsEmpty()) { // Convert landmarks to cv::Point3f**.
return absl::CancelledError("Pose landmarks packet is empty."); const auto landmarksCount = retrievedLandmarks.landmark_size();
}
auto retrievedLandmarks = for (int j = 0; j < landmarksCount; ++j) {
poseLandmarksPacket const auto& landmark = retrievedLandmarks.landmark(j);
.Get<::mediapipe::NormalizedLandmarkList>(); poseLandmarks[j].x = landmark.x();
poseLandmarks[j].y = landmark.y();
poseLandmarks[j].z = landmark.z();
}
// Convert landmarks to cv::Point3f**. return absl::OkStatus();
const auto landmarksCount = retrievedLandmarks.landmark_size(); }
for (int j = 0; j < landmarksCount; ++j) { nimagna::cv_wrapper::Point3f* lastDetectedLandmarks() { return poseLandmarks; }
const auto& landmark = retrievedLandmarks.landmark(j);
poseLandmarks[j].x = landmark.x();
poseLandmarks[j].y = landmark.y();
poseLandmarks[j].z = landmark.z();
}
return absl::OkStatus(); cv::Mat lastSegmentedFrame() { return segmentedMask; }
}
nimagna::cv_wrapper::Point3f* lastDetectedLandmarks() { static constexpr size_t kLandmarksCount = 33u;
return poseLandmarks;
}
cv::Mat lastSegmentedFrame() { private:
return segmentedMask; mediapipe::Packet poseLandmarksPacket;
} cv::Mat segmentedMask;
nimagna::cv_wrapper::Point3f poseLandmarks[kLandmarksCount];
static constexpr size_t kLandmarksCount = 33u; std::unique_ptr<mediapipe::OutputStreamPoller> posePresencePollerPtr;
std::unique_ptr<mediapipe::OutputStreamPoller> maskPollerPtr;
private: std::unique_ptr<mediapipe::OutputStreamPoller> landmarksPollerPtr;
mediapipe::Packet poseLandmarksPacket; mediapipe::CalculatorGraph graph;
cv::Mat segmentedMask; const char* kInputStream = "input_video";
nimagna::cv_wrapper::Point3f poseLandmarks[kLandmarksCount]; const char* kOutputSegmentationStream = "segmentation_mask";
std::unique_ptr<mediapipe::OutputStreamPoller> posePresencePollerPtr; const char* kOutpuLandmarksStream = "pose_landmarks";
std::unique_ptr<mediapipe::OutputStreamPoller> maskPollerPtr; const char* kOutpuPosePresenceStream = "pose_presence";
std::unique_ptr<mediapipe::OutputStreamPoller> landmarksPollerPtr;
mediapipe::CalculatorGraph graph;
const char* kInputStream = "input_video";
const char* kOutputSegmentationStream = "segmentation_mask";
const char* kOutpuLandmarksStream = "pose_landmarks";
const char* kOutpuPosePresenceStream = "pose_presence";
}; };
namespace nimagna { namespace nimagna {
PoseTracking::PoseTracking(const char* calculatorGraphConfigFile) { PoseTracking::PoseTracking(const char* calculatorGraphConfigFile) {
mImplementation = new PoseTrackingImpl(calculatorGraphConfigFile); mImplementation = new PoseTrackingImpl(calculatorGraphConfigFile);
}
bool PoseTracking::processFrame(const cv_wrapper::Mat& inputRGB8Bit) {
const auto frame = cv::Mat(inputRGB8Bit.rows, inputRGB8Bit.cols, CV_8UC3, inputRGB8Bit.data);
return mImplementation->processFrame(frame);
}
cv_wrapper::Point3f* PoseTracking::lastDetectedLandmarks() {
return mImplementation->lastDetectedLandmarks();
}
cv_wrapper::Mat PoseTracking::lastSegmentedFrame() {
const cv::Mat result = mImplementation->lastSegmentedFrame();
return cv_wrapper::Mat(result.rows, result.cols, result.data);
}
} }
bool PoseTracking::processFrame(const cv_wrapper::Mat& inputRGB8Bit) {
const auto frame = cv::Mat(inputRGB8Bit.rows, inputRGB8Bit.cols, CV_8UC3, inputRGB8Bit.data);
return mImplementation->processFrame(frame);
}
cv_wrapper::Point3f* PoseTracking::lastDetectedLandmarks() {
return mImplementation->lastDetectedLandmarks();
}
cv_wrapper::Mat PoseTracking::lastSegmentedFrame() {
const cv::Mat result = mImplementation->lastSegmentedFrame();
return cv_wrapper::Mat(result.rows, result.cols, result.data);
}
} // namespace nimagna

154
mediapipe/pose_tracking_dll/pose_tracking.h
View File

@ -10,92 +10,92 @@
class PoseTrackingImpl; class PoseTrackingImpl;
namespace nimagna { namespace nimagna {
namespace cv_wrapper { namespace cv_wrapper {
struct Point2f { struct Point2f {
float x = 0; float x = 0;
float y = 0; float y = 0;
Point2f() = default; Point2f() = default;
Point2f(float x, float y) : x(x), y(y) {} Point2f(float x, float y) : x(x), y(y) {}
}; };
struct Point3f { struct Point3f {
float x = 0; float x = 0;
float y = 0; float y = 0;
float z = 0; float z = 0;
Point3f() = default; Point3f() = default;
Point3f(float x, float y, float z) : x(x), y(y), z(z) {} Point3f(float x, float y, float z) : x(x), y(y), z(z) {}
}; };
struct Rect { struct Rect {
int x = 0; int x = 0;
int y = 0; int y = 0;
int width = 0; int width = 0;
int height = 0; int height = 0;
Rect() = default; Rect() = default;
Rect(int x, int y, int width, int height) : x(x), y(y), width(width), height(height) {} Rect(int x, int y, int width, int height) : x(x), y(y), width(width), height(height) {}
}; };
struct Mat { struct Mat {
int rows = 0; int rows = 0;
int cols = 0; int cols = 0;
unsigned char* data = 0; unsigned char* data = 0;
Mat(int rows, int cols, unsigned char* data) : rows(rows), cols(cols), data(data) {} Mat(int rows, int cols, unsigned char* data) : rows(rows), cols(cols), data(data) {}
}; };
} } // namespace cv_wrapper
class DLLEXPORT PoseTracking { class DLLEXPORT PoseTracking {
public: public:
static constexpr size_t landmarksCount = 33u; static constexpr size_t landmarksCount = 33u;
enum LandmarkNames { enum LandmarkNames {
NOSE = 0, NOSE = 0,
LEFT_EYE_INNER, LEFT_EYE_INNER,
LEFT_EYE, LEFT_EYE,
LEFT_EYE_OUTER, LEFT_EYE_OUTER,
RIGHT_EYE_INNER, RIGHT_EYE_INNER,
RIGHT_EYE, RIGHT_EYE,
RIGHT_EYE_OUTER, RIGHT_EYE_OUTER,
LEFT_EAR, LEFT_EAR,
RIGHT_EAR, RIGHT_EAR,
MOUTH_LEFT, MOUTH_LEFT,
MOUTH_RIGHT, MOUTH_RIGHT,
LEFT_SHOULDER, LEFT_SHOULDER,
RIGHT_SHOULDER, RIGHT_SHOULDER,
LEFT_ELBOW, LEFT_ELBOW,
RIGHT_ELBOW, RIGHT_ELBOW,
LEFT_WRIST, LEFT_WRIST,
RIGHT_WRIST, RIGHT_WRIST,
LEFT_PINKY, LEFT_PINKY,
RIGHT_PINKY, RIGHT_PINKY,
LEFT_INDEX, LEFT_INDEX,
RIGHT_INDEX, RIGHT_INDEX,
LEFT_THUMB, LEFT_THUMB,
RIGHT_THUMB, RIGHT_THUMB,
LEFT_HIP, LEFT_HIP,
RIGHT_HIP, RIGHT_HIP,
LEFT_KNEE, LEFT_KNEE,
RIGHT_KNEE, RIGHT_KNEE,
LEFT_ANKLE, LEFT_ANKLE,
RIGHT_ANKLE, RIGHT_ANKLE,
LEFT_HEEL, LEFT_HEEL,
RIGHT_HEEL, RIGHT_HEEL,
LEFT_FOOT_INDEX, LEFT_FOOT_INDEX,
RIGHT_FOOT_INDEX, RIGHT_FOOT_INDEX,
COUNT = landmarksCount COUNT = landmarksCount
}; };
PoseTracking(const char* calculatorGraphConfigFile); PoseTracking(const char* calculatorGraphConfigFile);
~PoseTracking() { delete mImplementation; } ~PoseTracking() { delete mImplementation; }
bool processFrame(const cv_wrapper::Mat& inputRGB8Bit); bool processFrame(const cv_wrapper::Mat& inputRGB8Bit);
cv_wrapper::Mat lastSegmentedFrame(); cv_wrapper::Mat lastSegmentedFrame();
cv_wrapper::Point3f* lastDetectedLandmarks(); cv_wrapper::Point3f* lastDetectedLandmarks();
private: private:
PoseTrackingImpl* mImplementation; PoseTrackingImpl* mImplementation;
}; };
} } // namespace nimagna
#endif #endif