Avoid void* for the sake of forward declaration
This commit is contained in:
Maksym Walczak
parent
281199e278
commit
092e1ad899
|
|
@ -1,14 +1,14 @@
|
|||
#include "pose_tracking.h"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
|
||||
#include "pose_tracking.h"
|
||||
|
||||
#include "absl/flags/flag.h"
|
||||
#include "absl/flags/parse.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_frame_opencv.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#include "mediapipe/framework/port/opencv_highgui_inc.h"
|
||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||
|
|
@ -17,160 +17,147 @@
|
|||
#include "mediapipe/framework/port/status.h"
|
||||
|
||||
class PoseTrackingImpl {
|
||||
public:
|
||||
PoseTrackingImpl(const std::string& calculatorGraphConfigFile) {
|
||||
auto status = initialize(calculatorGraphConfigFile);
|
||||
if (!status.ok()) {
|
||||
LOG(WARNING) << "Warning: " << status;
|
||||
}
|
||||
}
|
||||
public:
|
||||
PoseTrackingImpl(const std::string& calculatorGraphConfigFile) {
|
||||
auto status = initialize(calculatorGraphConfigFile);
|
||||
if (!status.ok()) {
|
||||
LOG(WARNING) << "Warning: " << status;
|
||||
}
|
||||
}
|
||||
|
||||
absl::Status initialize(const std::string& calculatorGraphConfigFile) {
|
||||
std::string graphContents;
|
||||
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
|
||||
calculatorGraphConfigFile,
|
||||
&graphContents));
|
||||
absl::Status initialize(const std::string& calculatorGraphConfigFile) {
|
||||
std::string graphContents;
|
||||
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(calculatorGraphConfigFile, &graphContents));
|
||||
|
||||
mediapipe::CalculatorGraphConfig config =
|
||||
mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
|
||||
graphContents);
|
||||
|
||||
MP_RETURN_IF_ERROR(graph.Initialize(config));
|
||||
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller,
|
||||
graph.AddOutputStreamPoller(kOutputSegmentationStream));
|
||||
mediapipe::CalculatorGraphConfig config =
|
||||
mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(graphContents);
|
||||
|
||||
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller landmarksPoller,
|
||||
graph.AddOutputStreamPoller(kOutpuLandmarksStream));
|
||||
MP_RETURN_IF_ERROR(graph.Initialize(config));
|
||||
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller,
|
||||
graph.AddOutputStreamPoller(kOutputSegmentationStream));
|
||||
|
||||
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller posePresencePoller,
|
||||
graph.AddOutputStreamPoller(kOutpuPosePresenceStream));
|
||||
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller landmarksPoller,
|
||||
graph.AddOutputStreamPoller(kOutpuLandmarksStream));
|
||||
|
||||
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller posePresencePoller,
|
||||
graph.AddOutputStreamPoller(kOutpuPosePresenceStream));
|
||||
|
||||
maskPollerPtr = std::make_unique<mediapipe::OutputStreamPoller>(std::move(poller));
|
||||
maskPollerPtr = std::make_unique<mediapipe::OutputStreamPoller>(std::move(poller));
|
||||
|
||||
landmarksPollerPtr = std::make_unique<mediapipe::OutputStreamPoller>(
|
||||
std::move(landmarksPoller));
|
||||
landmarksPollerPtr =
|
||||
std::make_unique<mediapipe::OutputStreamPoller>(std::move(landmarksPoller));
|
||||
|
||||
posePresencePollerPtr = std::make_unique<mediapipe::OutputStreamPoller>(
|
||||
std::move(posePresencePoller));
|
||||
posePresencePollerPtr =
|
||||
std::make_unique<mediapipe::OutputStreamPoller>(std::move(posePresencePoller));
|
||||
|
||||
MP_RETURN_IF_ERROR(graph.StartRun({}));
|
||||
}
|
||||
MP_RETURN_IF_ERROR(graph.StartRun({}));
|
||||
}
|
||||
|
||||
bool processFrame(const cv::Mat& inputRGB8Bit) {
|
||||
// Wrap Mat into an ImageFrame.
|
||||
auto inputFrame = absl::make_unique<mediapipe::ImageFrame>(
|
||||
mediapipe::ImageFormat::SRGB, inputRGB8Bit.cols, inputRGB8Bit.rows,
|
||||
mediapipe::ImageFrame::kDefaultAlignmentBoundary);
|
||||
cv::Mat inputFrameMat = mediapipe::formats::MatView(inputFrame.get());
|
||||
inputRGB8Bit.copyTo(inputFrameMat);
|
||||
bool processFrame(const cv::Mat& inputRGB8Bit) {
|
||||
// Wrap Mat into an ImageFrame.
|
||||
auto inputFrame = absl::make_unique<mediapipe::ImageFrame>(
|
||||
mediapipe::ImageFormat::SRGB, inputRGB8Bit.cols, inputRGB8Bit.rows,
|
||||
mediapipe::ImageFrame::kDefaultAlignmentBoundary);
|
||||
cv::Mat inputFrameMat = mediapipe::formats::MatView(inputFrame.get());
|
||||
inputRGB8Bit.copyTo(inputFrameMat);
|
||||
|
||||
// Send image packet into the graph.
|
||||
size_t frameTimestampUs =
|
||||
static_cast<double>(cv::getTickCount()) / static_cast<double>(cv::getTickFrequency()) * 1e6;
|
||||
auto status = graph.AddPacketToInputStream(
|
||||
kInputStream, mediapipe::Adopt(inputFrame.release())
|
||||
.At(mediapipe::Timestamp(frameTimestampUs)));
|
||||
// Send image packet into the graph.
|
||||
size_t frameTimestampUs =
|
||||
static_cast<double>(cv::getTickCount()) / static_cast<double>(cv::getTickFrequency()) * 1e6;
|
||||
auto status = graph.AddPacketToInputStream(
|
||||
kInputStream,
|
||||
mediapipe::Adopt(inputFrame.release()).At(mediapipe::Timestamp(frameTimestampUs)));
|
||||
|
||||
if (!status.ok()) {
|
||||
LOG(WARNING) << "Graph execution failed: " << status;
|
||||
return false;
|
||||
}
|
||||
if (!status.ok()) {
|
||||
LOG(WARNING) << "Graph execution failed: " << status;
|
||||
return false;
|
||||
}
|
||||
|
||||
mediapipe::Packet posePresencePacket;
|
||||
if (!posePresencePollerPtr || !posePresencePollerPtr->Next(&posePresencePacket)) return false;
|
||||
auto landmarksDetected = posePresencePacket.Get<bool>();
|
||||
mediapipe::Packet posePresencePacket;
|
||||
if (!posePresencePollerPtr || !posePresencePollerPtr->Next(&posePresencePacket)) return false;
|
||||
auto landmarksDetected = posePresencePacket.Get<bool>();
|
||||
|
||||
if (!landmarksDetected) {
|
||||
return false;
|
||||
}
|
||||
if (!landmarksDetected) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get the graph result packet, or stop if that fails.
|
||||
mediapipe::Packet maskPacket;
|
||||
if (!maskPollerPtr || !maskPollerPtr->Next(&maskPacket)) return false;
|
||||
auto& outputFrame = maskPacket.Get<mediapipe::ImageFrame>();
|
||||
// Get the graph result packet, or stop if that fails.
|
||||
mediapipe::Packet maskPacket;
|
||||
if (!maskPollerPtr || !maskPollerPtr->Next(&maskPacket)) return false;
|
||||
auto& outputFrame = maskPacket.Get<mediapipe::ImageFrame>();
|
||||
|
||||
// Get pose landmarks.
|
||||
if (!landmarksPollerPtr ||
|
||||
!landmarksPollerPtr->Next(&poseLandmarksPacket)) {
|
||||
return false;
|
||||
}
|
||||
// Get pose landmarks.
|
||||
if (!landmarksPollerPtr || !landmarksPollerPtr->Next(&poseLandmarksPacket)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Convert back to opencv for display or saving.
|
||||
auto mask = mediapipe::formats::MatView(&outputFrame);
|
||||
segmentedMask = mask.clone();
|
||||
// Convert back to opencv for display or saving.
|
||||
auto mask = mediapipe::formats::MatView(&outputFrame);
|
||||
segmentedMask = mask.clone();
|
||||
|
||||
absl::Status landmarksStatus = detectLandmarksWithStatus(poseLandmarks);
|
||||
absl::Status landmarksStatus = detectLandmarksWithStatus(poseLandmarks);
|
||||
|
||||
return landmarksStatus.ok();
|
||||
}
|
||||
return landmarksStatus.ok();
|
||||
}
|
||||
|
||||
absl::Status detectLandmarksWithStatus(
|
||||
nimagna::cv_wrapper::Point3f* poseLandmarks) {
|
||||
absl::Status detectLandmarksWithStatus(nimagna::cv_wrapper::Point3f* poseLandmarks) {
|
||||
if (poseLandmarksPacket.IsEmpty()) {
|
||||
return absl::CancelledError("Pose landmarks packet is empty.");
|
||||
}
|
||||
|
||||
if (poseLandmarksPacket.IsEmpty()) {
|
||||
return absl::CancelledError("Pose landmarks packet is empty.");
|
||||
}
|
||||
auto retrievedLandmarks = poseLandmarksPacket.Get<::mediapipe::NormalizedLandmarkList>();
|
||||
|
||||
auto retrievedLandmarks =
|
||||
poseLandmarksPacket
|
||||
.Get<::mediapipe::NormalizedLandmarkList>();
|
||||
// Convert landmarks to cv::Point3f**.
|
||||
const auto landmarksCount = retrievedLandmarks.landmark_size();
|
||||
|
||||
// Convert landmarks to cv::Point3f**.
|
||||
const auto landmarksCount = retrievedLandmarks.landmark_size();
|
||||
for (int j = 0; j < landmarksCount; ++j) {
|
||||
const auto& landmark = retrievedLandmarks.landmark(j);
|
||||
poseLandmarks[j].x = landmark.x();
|
||||
poseLandmarks[j].y = landmark.y();
|
||||
poseLandmarks[j].z = landmark.z();
|
||||
}
|
||||
|
||||
for (int j = 0; j < landmarksCount; ++j) {
|
||||
const auto& landmark = retrievedLandmarks.landmark(j);
|
||||
poseLandmarks[j].x = landmark.x();
|
||||
poseLandmarks[j].y = landmark.y();
|
||||
poseLandmarks[j].z = landmark.z();
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
nimagna::cv_wrapper::Point3f* lastDetectedLandmarks() { return poseLandmarks; }
|
||||
|
||||
nimagna::cv_wrapper::Point3f* lastDetectedLandmarks() {
|
||||
return poseLandmarks;
|
||||
}
|
||||
cv::Mat lastSegmentedFrame() { return segmentedMask; }
|
||||
|
||||
cv::Mat lastSegmentedFrame() {
|
||||
return segmentedMask;
|
||||
}
|
||||
static constexpr size_t kLandmarksCount = 33u;
|
||||
|
||||
static constexpr size_t kLandmarksCount = 33u;
|
||||
|
||||
private:
|
||||
mediapipe::Packet poseLandmarksPacket;
|
||||
cv::Mat segmentedMask;
|
||||
nimagna::cv_wrapper::Point3f poseLandmarks[kLandmarksCount];
|
||||
std::unique_ptr<mediapipe::OutputStreamPoller> posePresencePollerPtr;
|
||||
std::unique_ptr<mediapipe::OutputStreamPoller> maskPollerPtr;
|
||||
std::unique_ptr<mediapipe::OutputStreamPoller> landmarksPollerPtr;
|
||||
mediapipe::CalculatorGraph graph;
|
||||
const char* kInputStream = "input_video";
|
||||
const char* kOutputSegmentationStream = "segmentation_mask";
|
||||
const char* kOutpuLandmarksStream = "pose_landmarks";
|
||||
const char* kOutpuPosePresenceStream = "pose_presence";
|
||||
private:
|
||||
mediapipe::Packet poseLandmarksPacket;
|
||||
cv::Mat segmentedMask;
|
||||
nimagna::cv_wrapper::Point3f poseLandmarks[kLandmarksCount];
|
||||
std::unique_ptr<mediapipe::OutputStreamPoller> posePresencePollerPtr;
|
||||
std::unique_ptr<mediapipe::OutputStreamPoller> maskPollerPtr;
|
||||
std::unique_ptr<mediapipe::OutputStreamPoller> landmarksPollerPtr;
|
||||
mediapipe::CalculatorGraph graph;
|
||||
const char* kInputStream = "input_video";
|
||||
const char* kOutputSegmentationStream = "segmentation_mask";
|
||||
const char* kOutpuLandmarksStream = "pose_landmarks";
|
||||
const char* kOutpuPosePresenceStream = "pose_presence";
|
||||
};
|
||||
|
||||
namespace nimagna {
|
||||
PoseTracking::PoseTracking(const char* calculatorGraphConfigFile) {
|
||||
mImplementation = new PoseTrackingImpl(calculatorGraphConfigFile);
|
||||
}
|
||||
|
||||
bool PoseTracking::processFrame(const cv_wrapper::Mat& inputRGB8Bit) {
|
||||
const auto frame = cv::Mat(inputRGB8Bit.rows, inputRGB8Bit.cols, CV_8UC3, inputRGB8Bit.data);
|
||||
return mImplementation->processFrame(frame);
|
||||
}
|
||||
|
||||
cv_wrapper::Point3f* PoseTracking::lastDetectedLandmarks() {
|
||||
return mImplementation->lastDetectedLandmarks();
|
||||
}
|
||||
|
||||
cv_wrapper::Mat PoseTracking::lastSegmentedFrame() {
|
||||
const cv::Mat result = mImplementation->lastSegmentedFrame();
|
||||
|
||||
return cv_wrapper::Mat(result.rows, result.cols, result.data);
|
||||
}
|
||||
|
||||
PoseTracking::PoseTracking(const char* calculatorGraphConfigFile) {
|
||||
mImplementation = new PoseTrackingImpl(calculatorGraphConfigFile);
|
||||
}
|
||||
|
||||
bool PoseTracking::processFrame(const cv_wrapper::Mat& inputRGB8Bit) {
|
||||
const auto frame = cv::Mat(inputRGB8Bit.rows, inputRGB8Bit.cols, CV_8UC3, inputRGB8Bit.data);
|
||||
return mImplementation->processFrame(frame);
|
||||
}
|
||||
|
||||
cv_wrapper::Point3f* PoseTracking::lastDetectedLandmarks() {
|
||||
return mImplementation->lastDetectedLandmarks();
|
||||
}
|
||||
|
||||
cv_wrapper::Mat PoseTracking::lastSegmentedFrame() {
|
||||
const cv::Mat result = mImplementation->lastSegmentedFrame();
|
||||
|
||||
return cv_wrapper::Mat(result.rows, result.cols, result.data);
|
||||
}
|
||||
|
||||
} // namespace nimagna
|
||||
|
|
|
|||
|
|
@ -10,92 +10,92 @@
|
|||
class PoseTrackingImpl;
|
||||
|
||||
namespace nimagna {
|
||||
namespace cv_wrapper {
|
||||
struct Point2f {
|
||||
float x = 0;
|
||||
float y = 0;
|
||||
namespace cv_wrapper {
|
||||
struct Point2f {
|
||||
float x = 0;
|
||||
float y = 0;
|
||||
|
||||
Point2f() = default;
|
||||
Point2f(float x, float y) : x(x), y(y) {}
|
||||
};
|
||||
struct Point3f {
|
||||
float x = 0;
|
||||
float y = 0;
|
||||
float z = 0;
|
||||
Point2f() = default;
|
||||
Point2f(float x, float y) : x(x), y(y) {}
|
||||
};
|
||||
struct Point3f {
|
||||
float x = 0;
|
||||
float y = 0;
|
||||
float z = 0;
|
||||
|
||||
Point3f() = default;
|
||||
Point3f(float x, float y, float z) : x(x), y(y), z(z) {}
|
||||
};
|
||||
Point3f() = default;
|
||||
Point3f(float x, float y, float z) : x(x), y(y), z(z) {}
|
||||
};
|
||||
|
||||
struct Rect {
|
||||
int x = 0;
|
||||
int y = 0;
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
struct Rect {
|
||||
int x = 0;
|
||||
int y = 0;
|
||||
int width = 0;
|
||||
int height = 0;
|
||||
|
||||
Rect() = default;
|
||||
Rect(int x, int y, int width, int height) : x(x), y(y), width(width), height(height) {}
|
||||
};
|
||||
Rect() = default;
|
||||
Rect(int x, int y, int width, int height) : x(x), y(y), width(width), height(height) {}
|
||||
};
|
||||
|
||||
struct Mat {
|
||||
int rows = 0;
|
||||
int cols = 0;
|
||||
unsigned char* data = 0;
|
||||
struct Mat {
|
||||
int rows = 0;
|
||||
int cols = 0;
|
||||
unsigned char* data = 0;
|
||||
|
||||
Mat(int rows, int cols, unsigned char* data) : rows(rows), cols(cols), data(data) {}
|
||||
};
|
||||
}
|
||||
Mat(int rows, int cols, unsigned char* data) : rows(rows), cols(cols), data(data) {}
|
||||
};
|
||||
} // namespace cv_wrapper
|
||||
|
||||
class DLLEXPORT PoseTracking {
|
||||
public:
|
||||
static constexpr size_t landmarksCount = 33u;
|
||||
enum LandmarkNames {
|
||||
NOSE = 0,
|
||||
LEFT_EYE_INNER,
|
||||
LEFT_EYE,
|
||||
LEFT_EYE_OUTER,
|
||||
RIGHT_EYE_INNER,
|
||||
RIGHT_EYE,
|
||||
RIGHT_EYE_OUTER,
|
||||
LEFT_EAR,
|
||||
RIGHT_EAR,
|
||||
MOUTH_LEFT,
|
||||
MOUTH_RIGHT,
|
||||
LEFT_SHOULDER,
|
||||
RIGHT_SHOULDER,
|
||||
LEFT_ELBOW,
|
||||
RIGHT_ELBOW,
|
||||
LEFT_WRIST,
|
||||
RIGHT_WRIST,
|
||||
LEFT_PINKY,
|
||||
RIGHT_PINKY,
|
||||
LEFT_INDEX,
|
||||
RIGHT_INDEX,
|
||||
LEFT_THUMB,
|
||||
RIGHT_THUMB,
|
||||
LEFT_HIP,
|
||||
RIGHT_HIP,
|
||||
LEFT_KNEE,
|
||||
RIGHT_KNEE,
|
||||
LEFT_ANKLE,
|
||||
RIGHT_ANKLE,
|
||||
LEFT_HEEL,
|
||||
RIGHT_HEEL,
|
||||
LEFT_FOOT_INDEX,
|
||||
RIGHT_FOOT_INDEX,
|
||||
COUNT = landmarksCount
|
||||
};
|
||||
class DLLEXPORT PoseTracking {
|
||||
public:
|
||||
static constexpr size_t landmarksCount = 33u;
|
||||
enum LandmarkNames {
|
||||
NOSE = 0,
|
||||
LEFT_EYE_INNER,
|
||||
LEFT_EYE,
|
||||
LEFT_EYE_OUTER,
|
||||
RIGHT_EYE_INNER,
|
||||
RIGHT_EYE,
|
||||
RIGHT_EYE_OUTER,
|
||||
LEFT_EAR,
|
||||
RIGHT_EAR,
|
||||
MOUTH_LEFT,
|
||||
MOUTH_RIGHT,
|
||||
LEFT_SHOULDER,
|
||||
RIGHT_SHOULDER,
|
||||
LEFT_ELBOW,
|
||||
RIGHT_ELBOW,
|
||||
LEFT_WRIST,
|
||||
RIGHT_WRIST,
|
||||
LEFT_PINKY,
|
||||
RIGHT_PINKY,
|
||||
LEFT_INDEX,
|
||||
RIGHT_INDEX,
|
||||
LEFT_THUMB,
|
||||
RIGHT_THUMB,
|
||||
LEFT_HIP,
|
||||
RIGHT_HIP,
|
||||
LEFT_KNEE,
|
||||
RIGHT_KNEE,
|
||||
LEFT_ANKLE,
|
||||
RIGHT_ANKLE,
|
||||
LEFT_HEEL,
|
||||
RIGHT_HEEL,
|
||||
LEFT_FOOT_INDEX,
|
||||
RIGHT_FOOT_INDEX,
|
||||
COUNT = landmarksCount
|
||||
};
|
||||
|
||||
PoseTracking(const char* calculatorGraphConfigFile);
|
||||
~PoseTracking() { delete mImplementation; }
|
||||
PoseTracking(const char* calculatorGraphConfigFile);
|
||||
~PoseTracking() { delete mImplementation; }
|
||||
|
||||
bool processFrame(const cv_wrapper::Mat& inputRGB8Bit);
|
||||
cv_wrapper::Mat lastSegmentedFrame();
|
||||
cv_wrapper::Point3f* lastDetectedLandmarks();
|
||||
bool processFrame(const cv_wrapper::Mat& inputRGB8Bit);
|
||||
cv_wrapper::Mat lastSegmentedFrame();
|
||||
cv_wrapper::Point3f* lastDetectedLandmarks();
|
||||
|
||||
private:
|
||||
PoseTrackingImpl* mImplementation;
|
||||
};
|
||||
}
|
||||
private:
|
||||
PoseTrackingImpl* mImplementation;
|
||||
};
|
||||
} // namespace nimagna
|
||||
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user