cassdlcm/mediapipe-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

builds successfully

Browse Source
This commit is contained in:
Jules Youngberg 2022-06-06 22:06:03 -07:00
parent 0048ea7fa4
commit 9f758c730f
5 changed files with 151 additions and 4302 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -27,7 +27,7 @@ Build & install the mediagraph library.
```shell ```shell
bazel build --define MEDIAPIPE_DISABLE_GPU=1 mediapipe:libmediagraph.dylib bazel build --define MEDIAPIPE_DISABLE_GPU=1 mediapipe:libmediagraph.dylib
sudo cp bazel-bin/mediapipe/libmediagraph.dylib /usr/local/lib/libmediagraph.dylib sudo cp bazel-bin/mediapipe/libmediagraph.dylib /usr/local/lib/libmediagraph.dylib
cp mediapipe/cpuhost.h /usr/local/include/mediagraph.h cp mediapipe/mediagraph.h /usr/local/include/mediagraph.h
``` ```
### linux (untested) ### linux (untested)
@ -35,7 +35,7 @@ cp mediapipe/cpuhost.h /usr/local/include/mediagraph.h
```shell ```shell
bazel build --define MEDIAPIPE_DISABLE_GPU=1 mediapipe:mediagraph bazel build --define MEDIAPIPE_DISABLE_GPU=1 mediapipe:mediagraph
cp bazel-bin/mediapipe/libmediagraph.so /usr/local/lib/libmediagraph.so cp bazel-bin/mediapipe/libmediagraph.so /usr/local/lib/libmediagraph.so
cp mediapipe/cpuhost.h /usr/local/include/mediagraph.h cp mediapipe/mediagraph.h /usr/local/include/mediagraph.h
``` ```
## usage ## usage

44
build.rs
View File

@ -1,29 +1,29 @@
extern crate bindgen; extern crate bindgen;
// use std::path::PathBuf; use std::path::PathBuf;
fn main() { fn main() {
// println!("cargo:rustc-link-lib=stdc++"); println!("cargo:rustc-link-lib=stdc++");
// println!("cargo:rustc-link-lib=opencv4"); println!("cargo:rustc-link-lib=opencv4");
// println!("cargo:rustc-link-lib=mediagraph"); println!("cargo:rustc-link-lib=mediagraph");
// // println!("cargo:rerun-if-changed=wrapper.h"); // println!("cargo:rerun-if-changed=wrapper.h");
// let bindings = bindgen::Builder::default() let bindings = bindgen::Builder::default()
// .clang_arg("-xc++") .clang_arg("-xc++")
// .clang_arg("-std=c++14") .clang_arg("-std=c++14")
// .clang_arg("-I/usr/local/include/opencv4") .clang_arg("-I/usr/local/include/opencv4")
// .generate_comments(true) .generate_comments(true)
// .header("/usr/local/include/mediagraph.h") .header("/usr/local/include/mediagraph.h")
// // .whitelist_function("mediapipe_.*") .allowlist_function("mediagraph.*")
// // .whitelist_type("mediapipe.*") .allowlist_type("mediagraph.*")
// // .whitelist_var("mediapipe_.*") .allowlist_var("mediagraph.*")
// .detect_include_paths(true) .detect_include_paths(true)
// .generate_inline_functions(true) .generate_inline_functions(true)
// .generate() .generate()
// .expect("Unable to generate bindings"); .expect("Unable to generate bindings");
// let out_path = PathBuf::from("./src"); let out_path = PathBuf::from("./src");
// bindings bindings
// .write_to_file(out_path.join("bindings.rs")) .write_to_file(out_path.join("bindings.rs"))
// .expect("Couldn't write bindings!"); .expect("Couldn't write bindings!");
} }

32
examples/hello.rs
View File

@ -1,8 +1,6 @@
#![allow(unused_variables)] #![allow(unused_variables)]
#![allow(dead_code)] #![allow(dead_code)]
use mediapipe::*;
mod examples { mod examples {
use super::*; use super::*;
use opencv::prelude::*; use opencv::prelude::*;
@ -35,33 +33,6 @@ mod examples {
Ok(()) Ok(())
} }
// pub fn face_detection() -> Result<()> {
// let window = "video capture";
// highgui::named_window(window, highgui::WINDOW_AUTOSIZE)?;
// let mut cap = videoio::VideoCapture::new(0, videoio::CAP_ANY)?;
// if !cap.is_opened()? {
// panic!("Unable to open default cam")
// }
// let detector = mediapipe::face_detection::FaceDetector::default();
// loop {
// let mut frame = Mat::default();
// cap.read(&mut frame)?;
// let size = frame.size()?;
// if size.width > 0 {
// highgui::imshow(window, &mut frame)?
// }
// let key = highgui::wait_key(10)?;
// if key > 0 && key != 255 {
// break;
// }
// }
// Ok(())
// }
pub fn face_mesh() -> Result<()> { pub fn face_mesh() -> Result<()> {
let window = "video capture"; let window = "video capture";
@ -90,6 +61,7 @@ mod examples {
imgproc::cvt_color(&raw_frame, &mut rgb_frame, imgproc::COLOR_BGR2RGB, 0)?; imgproc::cvt_color(&raw_frame, &mut rgb_frame, imgproc::COLOR_BGR2RGB, 0)?;
opencv::core::flip(&rgb_frame, &mut flip_frame, 1)?; // horizontal opencv::core::flip(&rgb_frame, &mut flip_frame, 1)?; // horizontal
println!("processing");
detector.process(&flip_frame, &mut mesh); detector.process(&flip_frame, &mut mesh);
highgui::imshow(window, &mut flip_frame)?; highgui::imshow(window, &mut flip_frame)?;
@ -137,7 +109,7 @@ mod examples {
if size.width > 0 && !raw_frame.empty() { if size.width > 0 && !raw_frame.empty() {
imgproc::cvt_color(&raw_frame, &mut rgb_frame, imgproc::COLOR_BGR2RGB, 0)?; imgproc::cvt_color(&raw_frame, &mut rgb_frame, imgproc::COLOR_BGR2RGB, 0)?;
opencv::core::flip(&rgb_frame, &mut flip_frame, 1)?; // horizontal opencv::core::flip(&rgb_frame, &mut flip_frame, 1)?; // horizontal
detector.process(&flip_frame, &mut left, &mut right); detector.process(&flip_frame, &mut left, &mut right);
highgui::imshow(window, &mut flip_frame)?; highgui::imshow(window, &mut flip_frame)?;

4130
src/bindings.rs
View File

File diff suppressed because it is too large Load Diff

243
src/lib.rs
View File

@ -19,6 +19,9 @@ mod bindings;
pub use bindings::*; pub use bindings::*;
type Mediagraph = mediagraph_Mediagraph;
type Landmark = mediagraph_Landmark;
impl Default for Landmark { impl Default for Landmark {
fn default() -> Self { fn default() -> Self {
Self { Self {
@ -31,6 +34,10 @@ impl Default for Landmark {
} }
} }
struct Pose {
data: [Landmark; 33],
}
impl Default for Pose { impl Default for Pose {
fn default() -> Self { fn default() -> Self {
Self { Self {
@ -39,6 +46,10 @@ impl Default for Pose {
} }
} }
struct Hand {
data: [Landmark; 21],
}
impl Default for Hand { impl Default for Hand {
fn default() -> Self { fn default() -> Self {
Self { Self {
@ -47,6 +58,10 @@ impl Default for Hand {
} }
} }
struct FaceMesh {
data: [Landmark; 478],
}
impl Default for FaceMesh { impl Default for FaceMesh {
fn default() -> Self { fn default() -> Self {
Self { Self {
@ -99,7 +114,7 @@ pub mod pose {
pub smooth: bool, // true, pub smooth: bool, // true,
pub detection_con: f32, // 0.5 pub detection_con: f32, // 0.5
pub track_con: f32, // 0.5 pub track_con: f32, // 0.5
pub graph: PoseGraph, pub graph: *mut Mediagraph,
} }
impl PoseDetector { impl PoseDetector {
@ -108,8 +123,13 @@ pub mod pose {
CString::new(include_str!("pose_tracking_cpu.txt")).expect("CString::new failed"); CString::new(include_str!("pose_tracking_cpu.txt")).expect("CString::new failed");
let output_node = CString::new("pose_landmarks").expect("CString::new failed"); let output_node = CString::new("pose_landmarks").expect("CString::new failed");
let graph: PoseGraph = let graph: *mut Mediagraph = unsafe {
unsafe { PoseGraph::new(graph_config.as_ptr(), output_node.as_ptr()) }; Mediagraph::Create(
mediagraph_GraphType_POSE,
graph_config.as_ptr(),
output_node.as_ptr(),
)
};
Self { Self {
mode, mode,
@ -120,40 +140,20 @@ pub mod pose {
} }
} }
pub fn process(&mut self, input: &Mat, pose: *mut Pose) -> bool { pub fn process(&mut self, input: &Mat) -> bool {
unsafe { let mut data = input.clone();
let frame = input.as_raw() as *const cv_Mat; let landmarks = unsafe {
self.graph.process(frame, pose) mediagraph_Mediagraph_Process(
} self.graph as *mut std::ffi::c_void,
data.data_mut(),
data.cols(),
data.rows(),
)
};
// @todo read each landmark to build a pose struct
true
} }
// // draw true
// pub fn find_pose(&self, img: &[u8], draw: bool) {}
// // draw: true, bbox_with_hands: false
// pub fn find_position(&self, img: &[u8], draw: bool, bbox_with_hands: bool) {}
// // draw: true
// pub fn find_angle(
// &self,
// img: &[u8],
// p1: cgmath::Point2<f32>,
// p2: cgmath::Point2<f32>,
// draw: bool,
// ) {
// }
// pub fn find_distance(
// &self,
// p1: cgmath::Point2<f32>,
// p2: cgmath::Point2<f32>,
// img: Option<&[u8]>,
// r: f32,
// t: f32,
// ) {
// }
// pub fn anlge_check(&self, my_angle: f32, target_angle: f32, add_on: f32) {}
} }
impl Default for PoseDetector { impl Default for PoseDetector {
@ -171,7 +171,7 @@ pub mod face_mesh {
pub max_faces: usize, // 2 pub max_faces: usize, // 2
pub min_detection_con: f32, // 0.5 pub min_detection_con: f32, // 0.5
pub min_track_con: f32, // 0.5 pub min_track_con: f32, // 0.5
pub graph: FaceMeshGraph, pub graph: *mut Mediagraph,
} }
impl FaceMeshDetector { impl FaceMeshDetector {
@ -185,8 +185,14 @@ pub mod face_mesh {
.expect("CString::new failed"); .expect("CString::new failed");
let output_node = CString::new("multi_face_landmarks").expect("CString::new failed"); let output_node = CString::new("multi_face_landmarks").expect("CString::new failed");
let graph: FaceMeshGraph = let graph: *mut Mediagraph = unsafe {
unsafe { FaceMeshGraph::new(graph_config.as_ptr(), output_node.as_ptr()) }; Mediagraph::Create(
mediagraph_GraphType_FACE,
graph_config.as_ptr(),
output_node.as_ptr(),
)
};
Self { Self {
static_mode, static_mode,
max_faces, max_faces,
@ -196,22 +202,19 @@ pub mod face_mesh {
} }
} }
pub fn process(&mut self, input: &Mat, mesh: *mut FaceMesh) -> bool { pub fn process(&mut self, input: Mat) -> bool {
unsafe { let mut data = input.clone();
let frame = input.as_raw() as *const cv_Mat; let landmarks = unsafe {
self.graph.process(frame, mesh) mediagraph_Mediagraph_Process(
} self.graph as *mut std::ffi::c_void,
data.data_mut(),
data.cols(),
data.rows(),
)
};
// @todo read each landmark to build a face mesh struct
true
} }
// // draw: true
// pub fn find_face_mesh(&self, img: &[u8], draw: bool) {}
// pub fn find_distance(
// &self,
// p1: cgmath::Point2<f32>,
// p2: cgmath::Point2<f32>,
// img: Option<&[u8]>,
// ) {
// }
} }
impl Default for FaceMeshDetector { impl Default for FaceMeshDetector {
@ -221,49 +224,8 @@ pub mod face_mesh {
} }
} }
// pub mod face_detection {
// pub enum FaceKeyPoint {
// RIGHT_EYE = 0,
// LEFT_EYE = 1,
// NOSE_TIP = 2,
// MOUTH_CENTER = 3,
// RIGHT_EAR_TRAGION = 4,
// LEFT_EAR_TRAGION = 5,
// }
// pub struct FaceDetection {}
// impl FaceDetection {
// pub fn process(&self /* image */) /*NamedTuple*/ {}
// }
// pub struct FaceDetector {
// pub min_detection_con: f32, // 0.5
// pub face_detection: FaceDetection,
// }
// impl FaceDetector {
// pub fn new(min_detection_con: f32) -> Self {
// Self {
// min_detection_con,
// face_detection: todo!(),
// }
// }
// // draw: true
// pub fn find_faces(&self, img: &[u8], draw: bool) {}
// }
// impl Default for FaceDetector {
// fn default() -> Self {
// Self::new(0.5)
// }
// }
// }
pub mod hands { pub mod hands {
use super::*; use super::*;
// use mediapipe::*;
// use std::collections::HashMap;
pub enum HandLandmark { pub enum HandLandmark {
WRIST = 0, WRIST = 0,
@ -294,21 +256,22 @@ pub mod hands {
pub max_hands: usize, pub max_hands: usize,
pub detection_con: f32, // 0.5 pub detection_con: f32, // 0.5
pub min_track_con: f32, // 0.5 pub min_track_con: f32, // 0.5
pub graph: HandsGraph, pub graph: *mut Mediagraph,
} }
impl HandDetector { impl HandDetector {
pub fn new(mode: bool, max_hands: usize, detection_con: f32, min_track_con: f32) -> Self { pub fn new(mode: bool, max_hands: usize, detection_con: f32, min_track_con: f32) -> Self {
// // ::std::vector<::mediapipe::NormalizedLandmarkList>
// let graph_config = CString::new(include_str!("face_mesh_desktop_live.txt")).expect("CString::new failed");
// let output_node = CString::new("multi_face_landmarks").expect("CString::new failed");
let graph_config = CString::new(include_str!("hand_tracking_desktop_live.txt")) let graph_config = CString::new(include_str!("hand_tracking_desktop_live.txt"))
.expect("CString::new failed"); .expect("CString::new failed");
let output_node = CString::new("hand_landmarks").expect("CString::new failed"); let output_node = CString::new("hand_landmarks").expect("CString::new failed");
let graph: HandsGraph = let graph: *mut Mediagraph = unsafe {
unsafe { HandsGraph::new(graph_config.as_ptr(), output_node.as_ptr()) }; Mediagraph::Create(
mediagraph_GraphType_FACE,
graph_config.as_ptr(),
output_node.as_ptr(),
)
};
Self { Self {
mode, mode,
@ -319,26 +282,19 @@ pub mod hands {
} }
} }
pub fn process(&mut self, input: &Mat, left: *mut Hand, right: *mut Hand) -> bool { pub fn process(&mut self, input: Mat) -> bool {
unsafe { let mut data = input.clone();
let frame = input.as_raw() as *const cv_Mat; let landmarks = unsafe {
self.graph.process(frame, left, right) mediagraph_Mediagraph_Process(
} self.graph as *mut std::ffi::c_void,
data.data_mut(),
data.cols(),
data.rows(),
)
};
// @todo read each landmark to build a hands struct
true
} }
// // draw: true, flip_type: tru
// pub fn find_hands(&self, img: &[u8], draw: bool, flip_type: bool) {}
// pub fn fingers_up(&self, my_hand: &HashMap<String, String>) /*List of which fingers are up*/
// {
// }
// pub fn find_distance(
// &self,
// p1: cgmath::Point2<f32>,
// p2: cgmath::Point2<f32>,
// img: Option<&[u8]>,
// ) {
// }
} }
impl Default for HandDetector { impl Default for HandDetector {
@ -347,48 +303,3 @@ pub mod hands {
} }
} }
} }
// pub mod objectron {
// pub struct Objectron {}
// impl Objectron {
// pub fn process(&self /* image */) /*NamedTuple*/ {}
// }
// }
// pub mod selfie_segmentation {
// pub struct SelfieSegmentation {}
// impl SelfieSegmentation {
// pub fn process(&self /* image */) /*NamedTuple*/ {}
// }
// pub struct SelfieSegmentationDetector {
// pub model: usize, // 0 is general 1 is landscape(faster)
// pub selfie_segmentation: SelfieSegmentation,
// }
// impl SelfieSegmentationDetector {
// pub fn new(model: usize) -> Self {
// todo!()
// }
// // threshold: 0.1
// pub fn remove_bg(&self, img: &[u8], img_bg: [u8; 3], threshold: f32) {}
// }
// impl Default for SelfieSegmentationDetector {
// fn default() -> Self {
// Self::new(1)
// }
// }
// }
#[cfg(test)]
mod tests {
#[test]
fn it_works() {
let result = 2 + 2;
assert_eq!(result, 4);
}
}