added documentation

This commit is contained in:
Mautisim Munir 2022-10-21 15:23:05 +05:00
parent a49d98c171
commit 1c0cde6627
2 changed files with 228 additions and 199 deletions

View File

@ -10,12 +10,74 @@ import LinderaDetection
class ViewController: UIViewController {
//MARK: - UI Elements
@IBOutlet var liveView : UIView!
@IBOutlet var showLandmarksButton: UIButton!
@IBOutlet var chooseModelButton: UIButton!
@IBOutlet var titleview: UIView!
@IBOutlet var fpsLabel: UILabel!
//MARK: - UI Actions
@IBAction func setModelComplexity(){
let alert = UIAlertController(
title: nil,
message: nil,
preferredStyle: .actionSheet
)
alert.addAction(
.init(title: "MODEL (LITE)", style: .default) {[weak self] _ in
self?.lindera.setModelComplexityNow(complexity: 0)
self?.updateModelButtonText()
}
)
alert.addAction(
.init(title: "MODEL (FULL)", style: .default) { [weak self] _ in
self?.lindera.setModelComplexityNow(complexity: 1)
self?.updateModelButtonText()
}
)
alert.addAction(
.init(title: "MODEL (HEAVY)", style: .default) { [weak self] _ in
self?.lindera.setModelComplexityNow(complexity: 2)
self?.updateModelButtonText()
}
)
present(alert, animated: true)
}
@IBAction func showLandmarksButtonTouch(sender: UIButton){
lindera.showLandmarks(value: !lindera.areLandmarksShown());
updateLandmarksButtonText()
}
// MARK: - LinderaDelegate
/// A simple LinderaDelegate implementation that prints nose coordinates if detected
class LinderaDelegateImpl:LinderaDelegate{
func lindera(_ lindera: Lindera, didDetect event: Asensei3DPose.Event) {
// if let kpt = event.pose.nose{
// // Printing causes large drops in FPS
// print("LinderaDelegateImpl: Nose Keypoint (\(String(describing: kpt.position.x)),\(String(describing: kpt.position.y)),\(kpt.position.z)) with confidence \(kpt.confidence)")
// }
}
}
// MARK: - UI Text Modifications
func updateLandmarksButtonText(){
if (lindera.areLandmarksShown()){
showLandmarksButton.setTitle("LANDMARKS (ON)", for: UIControl.State.normal)
@ -25,7 +87,6 @@ class ViewController: UIViewController {
}
func updateModelButtonText(){
var text = "MODEL "
switch(lindera.getModelComplexity()){
@ -46,105 +107,56 @@ class ViewController: UIViewController {
chooseModelButton.setTitle(text, for: UIControl.State.normal)
}
@IBAction func setModelComplexity(){
let alert = UIAlertController(
title: nil,
message: nil,
preferredStyle: .actionSheet
)
alert.addAction(
.init(title: "MODEL (LITE)", style: .default) {[weak self] _ in
self?.lindera.setModelComplexityNow(complexity: 0)
self?.updateModelButtonText()
}
)
alert.addAction(
.init(title: "MODEL (FULL)", style: .default) { [weak self] _ in
self?.lindera.setModelComplexityNow(complexity: 1)
self?.updateModelButtonText()
}
)
alert.addAction(
.init(title: "MODEL (HEAVY)", style: .default) { [weak self] _ in
self?.lindera.setModelComplexityNow(complexity: 2)
self?.updateModelButtonText()
// MARK: - State Objects
}
)
present(alert, animated: true)
}
@IBAction func showLandmarksButtonTouch(sender: UIButton){
lindera.showLandmarks(value: !lindera.areLandmarksShown());
updateLandmarksButtonText()
}
let lindera = Lindera()
/// A simple LinderaDelegate implementation that prints nose coordinates if detected
class LinderaDelegateImpl:LinderaDelegate{
func lindera(_ lindera: Lindera, didDetect event: Asensei3DPose.Event) {
// if let kpt = event.pose.nose{
// // Printing causes large drops in FPS
// print("LinderaDelegateImpl: Nose Keypoint (\(String(describing: kpt.position.x)),\(String(describing: kpt.position.y)),\(kpt.position.z)) with confidence \(kpt.confidence)")
// }
}
}
let linderaDelegate = LinderaDelegateImpl()
// MARK: - UI Setup
override func viewDidLoad() {
super.viewDidLoad()
// // Do any additional setup after loading the view.
//
//
self.lindera.delegate = linderaDelegate
// add lindera camera view to our app's UIView i.e. liveView
// Expand our cameraView frame to liveView frame
if let view = self.liveView{
// add lindera camera view to our app's UIView i.e. liveView
view.addSubview(lindera.cameraView)
// Expand our cameraView frame to liveView frame
self.lindera.cameraView.frame = view.bounds
// Setting Up Constraints (No necessary with above statement)
self.lindera.cameraView.translatesAutoresizingMaskIntoConstraints = false
NSLayoutConstraint.activate([
self.lindera.cameraView.leadingAnchor.constraint(equalTo: view.leadingAnchor),
self.lindera.cameraView.topAnchor.constraint(equalTo: view.topAnchor),
self.lindera.cameraView.trailingAnchor.constraint(equalTo: view.trailingAnchor),
self.lindera.cameraView.bottomAnchor.constraint(equalTo: view.bottomAnchor)
])
NSLayoutConstraint.activate([
self.lindera.cameraView.leadingAnchor.constraint(equalTo: view.leadingAnchor),
self.lindera.cameraView.topAnchor.constraint(equalTo: view.topAnchor),
self.lindera.cameraView.trailingAnchor.constraint(equalTo: view.trailingAnchor),
self.lindera.cameraView.bottomAnchor.constraint(equalTo: view.bottomAnchor)
])
}
lindera.startCamera()
// This function is called whenver there is an fps update
self.lindera.setFpsDelegate(fpsDelegate: {[weak self] fps in
DispatchQueue.main.async {
self?.fpsLabel.text = "\(Int(fps)) fps"
}
})
// Otherwise they are hidden
self.liveView.bringSubviewToFront(titleview)
self.liveView.bringSubviewToFront(fpsLabel)
// Make the Landmarks and Model button text reflect the state in lindera object
updateLandmarksButtonText()
updateModelButtonText()
lindera.startCamera()
}

View File

@ -6,34 +6,40 @@ import UIKit
/// A helper class to run the Pose Tracking API
/// TFLite models are also loaded when you initialize this class
public final class Lindera{
// initalize the PoseTracking api and load models
var poseTracking:PoseTracking = PoseTracking(poseTrackingOptions: PoseTrackingOptions(showLandmarks: true,modelComplexity: 1))
let fpsHelper = FPSHelper(smoothingFactor: 0.95)
public func setFpsDelegate(fpsDelegate: @escaping (_ fps:Double)->Void){
fpsHelper.onFpsUpdate = fpsDelegate;
}
// attach Mediapipe camera helper to our class
let cameraSource = MPPCameraInputSource()
//MARK: - Public Class API
// A delegate to handle results
public weak var delegate: LinderaDelegate?
/// This function sets up your callback function to happen whenver there is an fps update
public func setFpsDelegate(fpsDelegate: @escaping (_ fps:Double)->Void){
fpsHelper.onFpsUpdate = fpsDelegate;
}
// Get the camera UI View that may contain landmarks drawing
public var cameraView: UIView {
return self.linderaExerciseSession
}
// Show Landmarks - works instantaneously!
public func showLandmarks(value:Bool){
self.poseTracking.showLandmarks(value)
}
// Are landmarks already drawn?
public func areLandmarksShown() -> Bool{
return self.poseTracking.areLandmarksShown()
}
// Current Model Complexity 0 -> lite; 1 -> full ; 2 -> heavy
public func getModelComplexity() -> Int {
return Int(self.poseTracking.poseTrackingOptions.modelComplexity);
}
// Set the model complexity and restart detection to load new models
public func setModelComplexityNow(complexity:Int){
let poseTrackingOptions = poseTracking.poseTrackingOptions
@ -43,32 +49,70 @@ public final class Lindera{
startPoseTracking()
startCamera()
}
public required init(){
startPoseTracking()
}
public func startCamera(_ completion: ((Result<Void, Error>) -> Void)? = nil) {
// set our rendering layer frame according to cameraView boundry
self.poseTracking.renderer.layer.frame = cameraView.layer.bounds
// attach render CALayer on cameraView to render output to
self.cameraView.layer.addSublayer(self.poseTracking.renderer.layer)
self.cameraSource.requestCameraAccess(
completionHandler: {(granted:Bool)->Void in
if (granted){
self.poseTracking.videoQueue.async(execute:{ [weak self] in
self?.cameraSource.start()
} )
completion?(.success(Void()))
}else{
completion?(.failure(preconditionFailure("Camera Access Not Granted")))
}
})
}
/// Choose front or back camera. Must restart camera after use if already started
public func selectCamera(_ position: AVCaptureDevice.Position, _ completion: ((Result<Void, Error>) -> Void)? = nil) {
self.poseTracking.videoQueue.async { [weak self] in
self?.cameraSource.cameraPosition = position
completion?(.success(Void()))
}
}
// MARK: - Private Class Functions
// Set your custom view heree
private lazy var linderaExerciseSession: UIView = {
// this will be the main camera view
// this will be the main camera view; Change it to custom view class to get desired results
let liveView = UIView()
startPoseTracking()
return liveView
}()
private func startPoseTracking(){
// set camera preferences
// set camera preferences
self.cameraSource.sessionPreset = AVCaptureSession.Preset.high.rawValue
self.cameraSource.cameraPosition = AVCaptureDevice.Position.front
self.cameraSource.orientation = AVCaptureVideoOrientation.portrait
if (self.cameraSource.orientation == AVCaptureVideoOrientation.portrait){
self.cameraSource.videoMirrored = true
self.cameraSource.videoMirrored = true
}
// call LinderaDelegate on pose tracking results
self.poseTracking.poseTrackingResultsListener = {[weak self] results in
@ -88,44 +132,13 @@ public final class Lindera{
// attach camera's output with poseTracking object and its videoQueue
self.cameraSource.setDelegate(self.poseTracking, queue: self.poseTracking.videoQueue)
}
public required init(){
}
public func startCamera(_ completion: ((Result<Void, Error>) -> Void)? = nil) {
// set our rendering layer frame according to cameraView boundry
self.poseTracking.renderer.layer.frame = cameraView.layer.bounds
// attach render CALayer on cameraView to render output to
self.cameraView.layer.addSublayer(self.poseTracking.renderer.layer)
self.cameraSource.requestCameraAccess(
completionHandler: {(granted:Bool)->Void in
if (granted){
self.poseTracking.videoQueue.async(execute:{ [weak self] in
self?.cameraSource.start()
} )
completion?(.success(Void()))
}else{
completion?(.failure(preconditionFailure("Camera Access Not Granted")))
}
})
}
func stopCamera(){
if (self.cameraSource.isRunning){
self.poseTracking.videoQueue.async { [weak self] in
self?.cameraSource.stop()
}
self.poseTracking.videoQueue.async { [weak self] in
self?.cameraSource.stop()
}
}
}
@ -155,21 +168,25 @@ public final class Lindera{
}
}
}
/// Choose front or back camera. Must restart camera after use if already started
public func selectCamera(_ position: AVCaptureDevice.Position, _ completion: ((Result<Void, Error>) -> Void)? = nil) {
self.poseTracking.videoQueue.async { [weak self] in
self?.cameraSource.cameraPosition = position
completion?(.success(Void()))
}
}
// MARK: - Private Class Objects
// initalize the PoseTracking api and load models
var poseTracking:PoseTracking = PoseTracking(poseTrackingOptions: PoseTrackingOptions(showLandmarks: true,modelComplexity: 1))
// Needed to get fps of model
let fpsHelper = FPSHelper(smoothingFactor: 0.95)
// attach Mediapipe camera helper to our class
let cameraSource = MPPCameraInputSource()
}
public protocol LinderaDelegate: AnyObject {
func lindera(_ lindera: Lindera, didDetect event: Asensei3DPose.Event)
@ -187,53 +204,53 @@ extension Asensei3DPose {
self.nose = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_NOSE])
self.leftEyeInner = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE_INNER])
self.leftEye = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE])
self.leftEyeOuter = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE_OUTER])
self.leftEyeInner = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE_INNER])
self.leftEye = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE])
self.leftEyeOuter = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE_OUTER])
self.rightEyeInner = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE_OUTER])
self.rightEye = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE])
self.rightEyeOuter = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE_OUTER])
self.rightEyeInner = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE_OUTER])
self.rightEye = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE])
self.rightEyeOuter = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE_OUTER])
self.leftEar = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EAR])
self.rightEar = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EAR])
self.leftEar = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EAR])
self.rightEar = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EAR])
self.mouthLeft = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_MOUTH_LEFT])
self.mouthRight = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_MOUTH_RIGHT])
self.mouthLeft = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_MOUTH_LEFT])
self.mouthRight = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_MOUTH_RIGHT])
self.leftShoulder = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_SHOULDER])
self.rightShoulder = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_SHOULDER])
self.leftShoulder = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_SHOULDER])
self.rightShoulder = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_SHOULDER])
self.leftElbow = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_ELBOW])
self.rightElbow = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_ELBOW])
self.leftElbow = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_ELBOW])
self.rightElbow = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_ELBOW])
self.leftWrist = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_WRIST])
self.rightWrist = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_WRIST])
self.leftWrist = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_WRIST])
self.rightWrist = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_WRIST])
self.leftPinky = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_PINKY])
self.rightPinky = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_PINKY])
self.leftPinky = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_PINKY])
self.rightPinky = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_PINKY])
self.leftIndex = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_INDEX])
self.rightIndex = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_INDEX])
self.leftIndex = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_INDEX])
self.rightIndex = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_INDEX])
self.leftThumb = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_THUMB])
self.rightThumb = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_THUMB])
self.leftThumb = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_THUMB])
self.rightThumb = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_THUMB])
self.leftHip = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_HIP])
self.rightHip = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_HIP])
self.leftHip = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_HIP])
self.rightHip = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_HIP])
self.leftKnee = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_KNEE])
self.rightKnee = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_KNEE])
self.leftKnee = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_KNEE])
self.rightKnee = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_KNEE])
self.rightAnkle = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_ANKLE])
self.leftAnkle = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_ANKLE])
self.rightAnkle = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_ANKLE])
self.leftAnkle = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_ANKLE])
self.rightHeel = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_HEEL])
self.leftHeel = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_HEEL])
self.rightHeel = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_HEEL])
self.leftHeel = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_HEEL])
self.rightFoot = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_FOOT])
self.leftFoot = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_FOOT])
self.rightFoot = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_FOOT])
self.leftFoot = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_FOOT])