diff --git a/mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/ViewController.swift b/mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/ViewController.swift index 6c7a37858..13ec1ee48 100644 --- a/mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/ViewController.swift +++ b/mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/ViewController.swift @@ -9,13 +9,75 @@ import LinderaDetection class ViewController: UIViewController { - + + //MARK: - UI Elements + + @IBOutlet var liveView : UIView! @IBOutlet var showLandmarksButton: UIButton! @IBOutlet var chooseModelButton: UIButton! @IBOutlet var titleview: UIView! @IBOutlet var fpsLabel: UILabel! + + //MARK: - UI Actions + + @IBAction func setModelComplexity(){ + let alert = UIAlertController( + title: nil, + message: nil, + preferredStyle: .actionSheet + ) + + alert.addAction( + .init(title: "MODEL (LITE)", style: .default) {[weak self] _ in + self?.lindera.setModelComplexityNow(complexity: 0) + self?.updateModelButtonText() + + } + ) + + alert.addAction( + .init(title: "MODEL (FULL)", style: .default) { [weak self] _ in + self?.lindera.setModelComplexityNow(complexity: 1) + self?.updateModelButtonText() + + + } + ) + alert.addAction( + .init(title: "MODEL (HEAVY)", style: .default) { [weak self] _ in + self?.lindera.setModelComplexityNow(complexity: 2) + self?.updateModelButtonText() + + + } + ) + + present(alert, animated: true) + } + + @IBAction func showLandmarksButtonTouch(sender: UIButton){ + + lindera.showLandmarks(value: !lindera.areLandmarksShown()); + updateLandmarksButtonText() + + } + + // MARK: - LinderaDelegate + + /// A simple LinderaDelegate implementation that prints nose coordinates if detected + class LinderaDelegateImpl:LinderaDelegate{ + func lindera(_ lindera: Lindera, didDetect event: Asensei3DPose.Event) { + // if let kpt = event.pose.nose{ + // // Printing causes large drops in FPS + // print("LinderaDelegateImpl: Nose Keypoint (\(String(describing: kpt.position.x)),\(String(describing: kpt.position.y)),\(kpt.position.z)) with confidence \(kpt.confidence)") + // } + } + + + } + // MARK: - UI Text Modifications func updateLandmarksButtonText(){ if (lindera.areLandmarksShown()){ showLandmarksButton.setTitle("LANDMARKS (ON)", for: UIControl.State.normal) @@ -25,7 +87,6 @@ class ViewController: UIViewController { } - func updateModelButtonText(){ var text = "MODEL " switch(lindera.getModelComplexity()){ @@ -46,107 +107,58 @@ class ViewController: UIViewController { chooseModelButton.setTitle(text, for: UIControl.State.normal) } - @IBAction func setModelComplexity(){ - let alert = UIAlertController( - title: nil, - message: nil, - preferredStyle: .actionSheet - ) - - alert.addAction( - .init(title: "MODEL (LITE)", style: .default) {[weak self] _ in - self?.lindera.setModelComplexityNow(complexity: 0) - self?.updateModelButtonText() - - } - ) - - alert.addAction( - .init(title: "MODEL (FULL)", style: .default) { [weak self] _ in - self?.lindera.setModelComplexityNow(complexity: 1) - self?.updateModelButtonText() - - - } - ) - alert.addAction( - .init(title: "MODEL (HEAVY)", style: .default) { [weak self] _ in - self?.lindera.setModelComplexityNow(complexity: 2) - self?.updateModelButtonText() - - - } - ) - - present(alert, animated: true) - } - @IBAction func showLandmarksButtonTouch(sender: UIButton){ - - lindera.showLandmarks(value: !lindera.areLandmarksShown()); - updateLandmarksButtonText() - - - - - } + + // MARK: - State Objects + let lindera = Lindera() - - /// A simple LinderaDelegate implementation that prints nose coordinates if detected - class LinderaDelegateImpl:LinderaDelegate{ - func lindera(_ lindera: Lindera, didDetect event: Asensei3DPose.Event) { -// if let kpt = event.pose.nose{ -// // Printing causes large drops in FPS -// print("LinderaDelegateImpl: Nose Keypoint (\(String(describing: kpt.position.x)),\(String(describing: kpt.position.y)),\(kpt.position.z)) with confidence \(kpt.confidence)") -// } - } - - - } let linderaDelegate = LinderaDelegateImpl() - + // MARK: - UI Setup override func viewDidLoad() { super.viewDidLoad() -// // Do any additional setup after loading the view. -// -// self.lindera.delegate = linderaDelegate - - // add lindera camera view to our app's UIView i.e. liveView - - // Expand our cameraView frame to liveView frame + + if let view = self.liveView{ + // add lindera camera view to our app's UIView i.e. liveView view.addSubview(lindera.cameraView) + // Expand our cameraView frame to liveView frame self.lindera.cameraView.frame = view.bounds - + + // Setting Up Constraints (No necessary with above statement) self.lindera.cameraView.translatesAutoresizingMaskIntoConstraints = false - NSLayoutConstraint.activate([ - self.lindera.cameraView.leadingAnchor.constraint(equalTo: view.leadingAnchor), - self.lindera.cameraView.topAnchor.constraint(equalTo: view.topAnchor), - self.lindera.cameraView.trailingAnchor.constraint(equalTo: view.trailingAnchor), - self.lindera.cameraView.bottomAnchor.constraint(equalTo: view.bottomAnchor) - ]) + NSLayoutConstraint.activate([ + self.lindera.cameraView.leadingAnchor.constraint(equalTo: view.leadingAnchor), + self.lindera.cameraView.topAnchor.constraint(equalTo: view.topAnchor), + self.lindera.cameraView.trailingAnchor.constraint(equalTo: view.trailingAnchor), + self.lindera.cameraView.bottomAnchor.constraint(equalTo: view.bottomAnchor) + ]) } - - - lindera.startCamera() + + // This function is called whenver there is an fps update self.lindera.setFpsDelegate(fpsDelegate: {[weak self] fps in DispatchQueue.main.async { self?.fpsLabel.text = "\(Int(fps)) fps" } }) + + // Otherwise they are hidden self.liveView.bringSubviewToFront(titleview) self.liveView.bringSubviewToFront(fpsLabel) + // Make the Landmarks and Model button text reflect the state in lindera object updateLandmarksButtonText() updateModelButtonText() + + lindera.startCamera() + } - - + + } diff --git a/mediapipe/swift/solutions/lindera/Lindera.swift b/mediapipe/swift/solutions/lindera/Lindera.swift index eeffc1602..2943c2416 100644 --- a/mediapipe/swift/solutions/lindera/Lindera.swift +++ b/mediapipe/swift/solutions/lindera/Lindera.swift @@ -6,34 +6,40 @@ import UIKit /// A helper class to run the Pose Tracking API /// TFLite models are also loaded when you initialize this class public final class Lindera{ - // initalize the PoseTracking api and load models - var poseTracking:PoseTracking = PoseTracking(poseTrackingOptions: PoseTrackingOptions(showLandmarks: true,modelComplexity: 1)) - let fpsHelper = FPSHelper(smoothingFactor: 0.95) + + + + //MARK: - Public Class API + + + // A delegate to handle results + public weak var delegate: LinderaDelegate? + + /// This function sets up your callback function to happen whenver there is an fps update public func setFpsDelegate(fpsDelegate: @escaping (_ fps:Double)->Void){ fpsHelper.onFpsUpdate = fpsDelegate; } - // attach Mediapipe camera helper to our class - let cameraSource = MPPCameraInputSource() - - // A delegate to handle results - public weak var delegate: LinderaDelegate? - - + // Get the camera UI View that may contain landmarks drawing public var cameraView: UIView { return self.linderaExerciseSession } + + + // Show Landmarks - works instantaneously! public func showLandmarks(value:Bool){ self.poseTracking.showLandmarks(value) } - - + // Are landmarks already drawn? public func areLandmarksShown() -> Bool{ return self.poseTracking.areLandmarksShown() } + // Current Model Complexity 0 -> lite; 1 -> full ; 2 -> heavy public func getModelComplexity() -> Int { return Int(self.poseTracking.poseTrackingOptions.modelComplexity); } + + // Set the model complexity and restart detection to load new models public func setModelComplexityNow(complexity:Int){ let poseTrackingOptions = poseTracking.poseTrackingOptions @@ -43,37 +49,75 @@ public final class Lindera{ startPoseTracking() startCamera() - - - + } + + public required init(){ + + startPoseTracking() } - + public func startCamera(_ completion: ((Result) -> Void)? = nil) { + // set our rendering layer frame according to cameraView boundry + self.poseTracking.renderer.layer.frame = cameraView.layer.bounds + // attach render CALayer on cameraView to render output to + self.cameraView.layer.addSublayer(self.poseTracking.renderer.layer) + + self.cameraSource.requestCameraAccess( + completionHandler: {(granted:Bool)->Void in + if (granted){ + self.poseTracking.videoQueue.async(execute:{ [weak self] in + + self?.cameraSource.start() + + } ) + completion?(.success(Void())) + }else{ + + completion?(.failure(preconditionFailure("Camera Access Not Granted"))) + + } + }) + + + + + } + /// Choose front or back camera. Must restart camera after use if already started + public func selectCamera(_ position: AVCaptureDevice.Position, _ completion: ((Result) -> Void)? = nil) { + self.poseTracking.videoQueue.async { [weak self] in + self?.cameraSource.cameraPosition = position + completion?(.success(Void())) + } + + } + + + // MARK: - Private Class Functions + + // Set your custom view heree private lazy var linderaExerciseSession: UIView = { - // this will be the main camera view + // this will be the main camera view; Change it to custom view class to get desired results let liveView = UIView() - startPoseTracking() - - - return liveView }() + + private func startPoseTracking(){ - // set camera preferences + // set camera preferences self.cameraSource.sessionPreset = AVCaptureSession.Preset.high.rawValue self.cameraSource.cameraPosition = AVCaptureDevice.Position.front self.cameraSource.orientation = AVCaptureVideoOrientation.portrait if (self.cameraSource.orientation == AVCaptureVideoOrientation.portrait){ - self.cameraSource.videoMirrored = true + self.cameraSource.videoMirrored = true } // call LinderaDelegate on pose tracking results self.poseTracking.poseTrackingResultsListener = {[weak self] results in - + guard let self = self, let results = results else { return } @@ -88,44 +132,13 @@ public final class Lindera{ // attach camera's output with poseTracking object and its videoQueue self.cameraSource.setDelegate(self.poseTracking, queue: self.poseTracking.videoQueue) } - public required init(){ - - - } - public func startCamera(_ completion: ((Result) -> Void)? = nil) { - // set our rendering layer frame according to cameraView boundry - self.poseTracking.renderer.layer.frame = cameraView.layer.bounds - // attach render CALayer on cameraView to render output to - self.cameraView.layer.addSublayer(self.poseTracking.renderer.layer) - - self.cameraSource.requestCameraAccess( - completionHandler: {(granted:Bool)->Void in - if (granted){ - self.poseTracking.videoQueue.async(execute:{ [weak self] in - - self?.cameraSource.start() - - } ) - completion?(.success(Void())) - }else{ - - completion?(.failure(preconditionFailure("Camera Access Not Granted"))) - - } - }) - - - - - } - func stopCamera(){ if (self.cameraSource.isRunning){ - self.poseTracking.videoQueue.async { [weak self] in - self?.cameraSource.stop() - } + self.poseTracking.videoQueue.async { [weak self] in + self?.cameraSource.stop() + } } } @@ -139,7 +152,7 @@ public final class Lindera{ self.startCamera(completion) switch(self.cameraSource.cameraPosition){ - + case .unspecified: completion?(.failure(preconditionFailure("Unkown Camera Position"))) case .back: @@ -148,30 +161,34 @@ public final class Lindera{ self.selectCamera(AVCaptureDevice.Position.back,completion) @unknown default: completion?(.failure(preconditionFailure("Unkown Camera Position"))) - + } - + } - + } - } - - /// Choose front or back camera. Must restart camera after use if already started - public func selectCamera(_ position: AVCaptureDevice.Position, _ completion: ((Result) -> Void)? = nil) { - self.poseTracking.videoQueue.async { [weak self] in - self?.cameraSource.cameraPosition = position - completion?(.success(Void())) - } - } + + + // MARK: - Private Class Objects + // initalize the PoseTracking api and load models + var poseTracking:PoseTracking = PoseTracking(poseTrackingOptions: PoseTrackingOptions(showLandmarks: true,modelComplexity: 1)) + + // Needed to get fps of model + let fpsHelper = FPSHelper(smoothingFactor: 0.95) + + // attach Mediapipe camera helper to our class + let cameraSource = MPPCameraInputSource() } -public protocol LinderaDelegate: AnyObject { + +public protocol LinderaDelegate: AnyObject { + func lindera(_ lindera: Lindera, didDetect event: Asensei3DPose.Event) } @@ -182,61 +199,61 @@ func landmarkToBodyJointDetails(landmark: PoseLandmark) -> Asensei3DPose.BodyJoi } // MARK: - Helpers extension Asensei3DPose { - + init(_ pose: PoseTrackingResults) { self.nose = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_NOSE]) - - self.leftEyeInner = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE_INNER]) - self.leftEye = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE]) - self.leftEyeOuter = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE_OUTER]) - - self.rightEyeInner = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE_OUTER]) - self.rightEye = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE]) - self.rightEyeOuter = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE_OUTER]) - - self.leftEar = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EAR]) - self.rightEar = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EAR]) - - self.mouthLeft = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_MOUTH_LEFT]) - self.mouthRight = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_MOUTH_RIGHT]) - - self.leftShoulder = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_SHOULDER]) - self.rightShoulder = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_SHOULDER]) - - self.leftElbow = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_ELBOW]) - self.rightElbow = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_ELBOW]) - - self.leftWrist = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_WRIST]) - self.rightWrist = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_WRIST]) - - self.leftPinky = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_PINKY]) - self.rightPinky = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_PINKY]) - - self.leftIndex = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_INDEX]) - self.rightIndex = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_INDEX]) - - self.leftThumb = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_THUMB]) - self.rightThumb = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_THUMB]) - - self.leftHip = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_HIP]) - self.rightHip = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_HIP]) - - self.leftKnee = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_KNEE]) - self.rightKnee = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_KNEE]) - - self.rightAnkle = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_ANKLE]) - self.leftAnkle = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_ANKLE]) - - - self.rightHeel = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_HEEL]) - self.leftHeel = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_HEEL]) - - self.rightFoot = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_FOOT]) - self.leftFoot = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_FOOT]) + + self.leftEyeInner = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE_INNER]) + self.leftEye = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE]) + self.leftEyeOuter = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EYE_OUTER]) + + self.rightEyeInner = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE_OUTER]) + self.rightEye = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE]) + self.rightEyeOuter = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EYE_OUTER]) + + self.leftEar = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_EAR]) + self.rightEar = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_EAR]) + + self.mouthLeft = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_MOUTH_LEFT]) + self.mouthRight = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_MOUTH_RIGHT]) + + self.leftShoulder = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_SHOULDER]) + self.rightShoulder = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_SHOULDER]) + + self.leftElbow = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_ELBOW]) + self.rightElbow = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_ELBOW]) + + self.leftWrist = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_WRIST]) + self.rightWrist = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_WRIST]) + + self.leftPinky = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_PINKY]) + self.rightPinky = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_PINKY]) + + self.leftIndex = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_INDEX]) + self.rightIndex = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_INDEX]) + + self.leftThumb = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_THUMB]) + self.rightThumb = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_THUMB]) + + self.leftHip = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_HIP]) + self.rightHip = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_HIP]) + + self.leftKnee = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_KNEE]) + self.rightKnee = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_KNEE]) + + self.rightAnkle = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_ANKLE]) + self.leftAnkle = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_ANKLE]) + + + self.rightHeel = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_HEEL]) + self.leftHeel = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_HEEL]) + + self.rightFoot = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_RIGHT_FOOT]) + self.leftFoot = landmarkToBodyJointDetails(landmark: pose.landmarks[POSE_LEFT_FOOT]) + - } }