From 208752d58e2852d1b50f503025ae9781e3fa9997 Mon Sep 17 00:00:00 2001 From: Mautisim Munir Date: Thu, 10 Nov 2022 00:02:09 +0500 Subject: [PATCH 1/2] added video api support in java --- .../posetracking_lindera/MainActivity.java | 58 +++++++++++------ .../app/src/main/res/layout/activity_main.xml | 11 +++- .../solutions/lindera/InputSource.java | 5 ++ .../mediapipe/solutions/lindera/Lindera.java | 64 +++++++++++++++++-- 4 files changed, 110 insertions(+), 28 deletions(-) create mode 100644 mediapipe/java/com/google/mediapipe/solutions/copperlabs/copperlabs-lindera/src/main/java/com/google/mediapipe/solutions/lindera/InputSource.java diff --git a/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java index f4d55641d..61adde466 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java +++ b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java @@ -33,9 +33,8 @@ import androidx.appcompat.app.AppCompatActivity; import com.afollestad.materialdialogs.MaterialDialog; import com.google.mediapipe.R; -import com.google.mediapipe.solutions.lindera.BodyJoints; import com.google.mediapipe.solutions.lindera.CameraRotation; -import com.google.mediapipe.solutions.lindera.ComputerVisionPlugin; +import com.google.mediapipe.solutions.lindera.InputSource; import com.google.mediapipe.solutions.lindera.Lindera; import org.json.JSONException; @@ -80,11 +79,11 @@ public class MainActivity extends AppCompatActivity { lindera = new Lindera(plugin); + lindera.setupVideoPicker(this); List cameras = lindera.getAvailableCameras(); // FRONT or BACK lindera.setCamera("FRONT"); lindera.setCameraRotation(CameraRotation.AUTOMATIC); - lindera.fpsHelper.onFpsUpdate = new Consumer() { @Override public void accept(Double fps) { @@ -99,36 +98,55 @@ public class MainActivity extends AppCompatActivity { } + private void startDetectionPipeline(InputSource inputSource){ + if (!isLinderaInitialized) { + modelLoadAsyncDialogue(()->{ + if (inputSource==InputSource.VIDEO){ + lindera.setInputSource(InputSource.VIDEO); + } + lindera.initialize(findViewById(R.id.preview_display_layout), MainActivity.this); + isLinderaInitialized = true; + findViewById(R.id.button_start_video).setVisibility(View.GONE); + findViewById(R.id.button_start_camera).setVisibility(View.GONE); + findViewById(R.id.button_set_model).setVisibility(View.VISIBLE); + findViewById(R.id.button_toggle_landmarks).setVisibility(View.VISIBLE); + findViewById(R.id.button_capture_logging).setVisibility(View.VISIBLE); + + updateLandmarkButtonText(); + updateModelComplexityButtonText(); + if (inputSource==InputSource.VIDEO) { + lindera.pickVideo(); + } + + }); + + + } + isDetectionStarted = !isDetectionStarted; + } /** * Sets up the UI components for the live demo with camera input. */ private void setupLiveDemoUiComponents() { - Button startDetectionButton = findViewById(R.id.button_start_detection); + Button startCameraButton = findViewById(R.id.button_start_camera); + Button startVideoButton = findViewById(R.id.button_start_video); Button toggleLandmarks = findViewById(R.id.button_toggle_landmarks); Button modelComplexity = findViewById(R.id.button_set_model); Button startCapture = findViewById(R.id.button_capture_logging); FrameLayout frameLayout = findViewById(R.id.preview_display_layout); - startDetectionButton.setOnClickListener( + startCameraButton.setOnClickListener( v -> { -// startCameraButton.setVisibility(View.GONE); - if (!isLinderaInitialized) { - modelLoadAsyncDialogue(()->{ - lindera.initialize(frameLayout, MainActivity.this); - isLinderaInitialized = true; - startDetectionButton.setVisibility(View.GONE); - findViewById(R.id.button_set_model).setVisibility(View.VISIBLE); - findViewById(R.id.button_toggle_landmarks).setVisibility(View.VISIBLE); - findViewById(R.id.button_capture_logging).setVisibility(View.VISIBLE); - - updateLandmarkButtonText(); - updateModelComplexityButtonText(); - }); + startDetectionPipeline(InputSource.CAMERA); - } - isDetectionStarted = !isDetectionStarted; + + }); + startVideoButton.setOnClickListener( + v -> { + startDetectionPipeline(InputSource.VIDEO); + }); diff --git a/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/res/layout/activity_main.xml b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/res/layout/activity_main.xml index 975690cb0..a2739ccc3 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/res/layout/activity_main.xml +++ b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/res/layout/activity_main.xml @@ -15,11 +15,18 @@ android:orientation="horizontal"> + + + + + @@ -63,8 +95,10 @@ + + diff --git a/mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/Infobazel.plist b/mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/Infobazel.plist new file mode 100644 index 000000000..108826e3b --- /dev/null +++ b/mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/Infobazel.plist @@ -0,0 +1,34 @@ + + + + + + UIApplicationSceneManifest + + CFBundleDevelopmentRegion + en + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSRequiresIPhoneOS + + UIApplicationSupportsMultipleScenes + + MainViewController + ViewController + UILaunchStoryboardName + LaunchScreen + + + diff --git a/mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/ViewController.swift b/mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/ViewController.swift index b9b449643..3cc30c86f 100644 --- a/mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/ViewController.swift +++ b/mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/ViewController.swift @@ -7,9 +7,34 @@ import UIKit import LinderaDetection //import LinderaDetection +import PhotosUI +class ViewController: UIViewController, PHPickerViewControllerDelegate { + + // Video Picker + func picker(_ picker: PHPickerViewController, didFinishPicking results: [PHPickerResult]) { + + for result in results { + result.itemProvider.loadFileRepresentation(forTypeIdentifier: UTType.movie.identifier){url,err in + if let url = url { + DispatchQueue.main.async { + [weak self] in -class ViewController: UIViewController { + picker.dismiss(animated: true) + self?.inputSourceView.isHidden = true + + } + let asset = AVAsset(url: url) + self.lindera.startVideo(asset: asset) + + } + } + } + +// + + } + //MARK: - UI Elements @@ -17,11 +42,30 @@ class ViewController: UIViewController { @IBOutlet var liveView : UIView! @IBOutlet var showLandmarksButton: UIButton! @IBOutlet var chooseModelButton: UIButton! + @IBOutlet var startCameraButton: UIButton! @IBOutlet var titleview: UIView! @IBOutlet var fpsLabel: UILabel! - + @IBOutlet var inputSourceView: UIView! //MARK: - UI Actions + @IBAction func startCamera(){ + lindera.startCamera() + inputSourceView.isHidden = true + + + + } + + @IBAction func pickVideo(){ + var configuration = PHPickerConfiguration(photoLibrary: .shared()) + let filter = PHPickerFilter.any(of: [ .videos]) + configuration.filter = filter + configuration.selectionLimit = 1 + let picker = PHPickerViewController(configuration: configuration) + picker.delegate = self + present(picker, animated: true) + + } @IBAction func setModelComplexity(){ let alert = UIAlertController( @@ -150,12 +194,16 @@ class ViewController: UIViewController { // Otherwise they are hidden self.liveView.bringSubviewToFront(titleview) self.liveView.bringSubviewToFront(fpsLabel) - + self.liveView.bringSubviewToFront(inputSourceView) // Make the Landmarks and Model button text reflect the state in lindera object updateLandmarksButtonText() updateModelButtonText() - lindera.startCamera() + + + + + } diff --git a/mediapipe/swift/solutions/lindera/Lindera.swift b/mediapipe/swift/solutions/lindera/Lindera.swift index 3a380c923..5f756daee 100644 --- a/mediapipe/swift/solutions/lindera/Lindera.swift +++ b/mediapipe/swift/solutions/lindera/Lindera.swift @@ -2,6 +2,12 @@ import UIKit #if arch(arm64) import MPPoseTracking +import AVFoundation + +public enum InputSource { + case CAMERA,VIDEO +} + /// A helper class to run the Pose Tracking API @@ -20,7 +26,45 @@ public final class Lindera{ public func setFpsDelegate(fpsDelegate: @escaping (_ fps:Double)->Void){ fpsHelper.onFpsUpdate = fpsDelegate; } + + public func startVideo(asset:AVAsset){ + self.videoSource = MPPPlayerInputSource(avAsset: asset) + self.videoSource.setDelegate(self.poseTracking, queue: self.poseTracking.videoQueue) + DispatchQueue.main.async { [weak self] in + + if let self = self { + // set our rendering layer frame according to cameraView boundry + self.poseTracking.renderer.layer.frame = self.cameraView.layer.bounds + // attach render CALayer on cameraView to render output to + self.cameraView.layer.addSublayer(self.poseTracking.renderer.layer) + } + } + + self.poseTracking.videoQueue.async(execute:{ [weak self] in + self?.videoSource.start() + + + }) + + } + + public func setInputSource(inputSource:InputSource){ + if (self.inputSource==inputSource) {return} + switch(inputSource){ + + case .CAMERA: + self.cameraSource.setDelegate(self.poseTracking, queue: self.poseTracking.videoQueue) + break + + case .VIDEO: + break + } + + + self.inputSource = inputSource + + } // Get the camera UI View that may contain landmarks drawing public var cameraView: UIView { return self.linderaExerciseSession @@ -57,6 +101,7 @@ public final class Lindera{ startPoseTracking() } + public func startCamera(_ completion: ((Result) -> Void)? = nil) { // set our rendering layer frame according to cameraView boundry @@ -182,8 +227,9 @@ public final class Lindera{ // attach Mediapipe camera helper to our class let cameraSource = MPPCameraInputSource() + var videoSource = MPPPlayerInputSource() - + var inputSource:InputSource = InputSource.CAMERA }