cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'video-api' into move-non-native-dependencies

Browse Source
This commit is contained in:
Mautisim Munir 2022-11-10 03:10:08 +05:00
commit e857430326
10 changed files with 280 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
mediapipe/MediaPipe.tulsiproj/Configs/MediaPipe.tulsigen
View File

@ -23,8 +23,7 @@
"mediapipe/objc/testing/app/BUILD"
],
"buildTargets" : [
"//mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera:posetracking-lindera",
"//mediapipe/swift/solutions/lindera:lindera"
"//mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera:posetracking-lindera"
],
"optionSet" : {
"BazelBuildOptionsDebug" : {

4
mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/BUILD.bazel
View File

@ -12,7 +12,7 @@ MIN_IOS_VERSION = "14.0"
swift_library(
name = "lindera_app_lib",
srcs = glob(["**/*.swift"]),
data =[
data = [
"Base.lproj/LaunchScreen.storyboard",
"Base.lproj/Main.storyboard",
],
@ -62,7 +62,7 @@ ios_application(
"ipad",
],
infoplists = [
"Info.plist",
"Infobazel.plist",
"//mediapipe/examples/ios/common:Info.plist",
],
linkopts = [

34
mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/Base.lproj/Main.storyboard
View File

@ -56,6 +56,38 @@
<color key="textColor" systemColor="secondarySystemGroupedBackgroundColor"/>
<nil key="highlightedColor"/>
</label>
<view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="xio-lL-zFK">
<rect key="frame" x="20" y="133" width="374" height="212"/>
<autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
<subviews>
<button opaque="NO" contentMode="scaleToFill" fixedFrame="YES" contentHorizontalAlignment="center" contentVerticalAlignment="center" buttonType="system" lineBreakMode="middleTruncation" translatesAutoresizingMaskIntoConstraints="NO" id="yeq-1E-9PL">
<rect key="frame" x="8" y="104" width="151" height="43"/>
<autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
<state key="normal" title="Button"/>
<buttonConfiguration key="configuration" style="gray" title="Start Camera"/>
<connections>
<action selector="startCamera" destination="BYZ-38-t0r" eventType="touchDown" id="bJ0-NK-Tiy"/>
</connections>
</button>
<label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Please Select Input Source" textAlignment="natural" lineBreakMode="tailTruncation" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="qIp-Q6-AgF">
<rect key="frame" x="78" y="8" width="218" height="58"/>
<autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
<fontDescription key="fontDescription" type="system" pointSize="17"/>
<nil key="textColor"/>
<nil key="highlightedColor"/>
</label>
<button opaque="NO" contentMode="scaleToFill" fixedFrame="YES" contentHorizontalAlignment="center" contentVerticalAlignment="center" buttonType="system" lineBreakMode="middleTruncation" translatesAutoresizingMaskIntoConstraints="NO" id="SV7-hA-BsP">
<rect key="frame" x="215" y="104" width="151" height="43"/>
<autoresizingMask key="autoresizingMask" flexibleMaxX="YES" flexibleMaxY="YES"/>
<state key="normal" title="Button"/>
<buttonConfiguration key="configuration" style="gray" title="Pick Video"/>
<connections>
<action selector="pickVideo" destination="BYZ-38-t0r" eventType="touchDown" id="YSl-m6-lbI"/>
</connections>
</button>
</subviews>
<color key="backgroundColor" systemColor="systemBackgroundColor"/>
</view>
</subviews>
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
<color key="backgroundColor" systemColor="systemBackgroundColor"/>
@ -63,8 +95,10 @@
<connections>
<outlet property="chooseModelButton" destination="oaC-Ax-V0a" id="Knp-ll-Zgu"/>
<outlet property="fpsLabel" destination="hSQ-so-ykH" id="sw5-ik-ro9"/>
<outlet property="inputSourceView" destination="xio-lL-zFK" id="uiz-Te-v97"/>
<outlet property="liveView" destination="8bC-Xf-vdC" id="COw-5j-lAL"/>
<outlet property="showLandmarksButton" destination="dv5-h1-tjP" id="xXW-UG-aSR"/>
<outlet property="startCameraButton" destination="yeq-1E-9PL" id="D90-1U-dYK"/>
<outlet property="titleview" destination="1BO-kg-lOt" id="uP4-0G-Gix"/>
</connections>
</viewController>

34
mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/Infobazel.plist Normal file
View File

@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>UIApplicationSceneManifest</key>
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>en</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
<key>LSRequiresIPhoneOS</key>
<true/>
<key>UIApplicationSupportsMultipleScenes</key>
<false/>
<key>MainViewController</key>
<string>ViewController</string>
<key>UILaunchStoryboardName</key>
<string>LaunchScreen</string>
</dict>
</dict>
</plist>

56
mediapipe/examples/ios/posetracking-lindera/PoseTrackingLindera/ViewController.swift
View File

@ -7,9 +7,34 @@
import UIKit
import LinderaDetection
//import LinderaDetection
import PhotosUI
class ViewController: UIViewController, PHPickerViewControllerDelegate {
// Video Picker
func picker(_ picker: PHPickerViewController, didFinishPicking results: [PHPickerResult]) {
for result in results {
result.itemProvider.loadFileRepresentation(forTypeIdentifier: UTType.movie.identifier){url,err in
if let url = url {
DispatchQueue.main.async {
[weak self] in
class ViewController: UIViewController {
picker.dismiss(animated: true)
self?.inputSourceView.isHidden = true
}
let asset = AVAsset(url: url)
self.lindera.startVideo(asset: asset)
}
}
}
//
}
//MARK: - UI Elements
@ -17,11 +42,30 @@ class ViewController: UIViewController {
@IBOutlet var liveView : UIView!
@IBOutlet var showLandmarksButton: UIButton!
@IBOutlet var chooseModelButton: UIButton!
@IBOutlet var startCameraButton: UIButton!
@IBOutlet var titleview: UIView!
@IBOutlet var fpsLabel: UILabel!
@IBOutlet var inputSourceView: UIView!
//MARK: - UI Actions
@IBAction func startCamera(){
lindera.startCamera()
inputSourceView.isHidden = true
}
@IBAction func pickVideo(){
var configuration = PHPickerConfiguration(photoLibrary: .shared())
let filter = PHPickerFilter.any(of: [ .videos])
configuration.filter = filter
configuration.selectionLimit = 1
let picker = PHPickerViewController(configuration: configuration)
picker.delegate = self
present(picker, animated: true)
}
@IBAction func setModelComplexity(){
let alert = UIAlertController(
@ -150,12 +194,16 @@ class ViewController: UIViewController {
// Otherwise they are hidden
self.liveView.bringSubviewToFront(titleview)
self.liveView.bringSubviewToFront(fpsLabel)
self.liveView.bringSubviewToFront(inputSourceView)
// Make the Landmarks and Model button text reflect the state in lindera object
updateLandmarksButtonText()
updateModelButtonText()
lindera.startCamera()
}

58
mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java
View File

@ -33,9 +33,8 @@ import androidx.appcompat.app.AppCompatActivity;
import com.afollestad.materialdialogs.MaterialDialog;
import com.google.mediapipe.R;
import com.google.mediapipe.solutions.lindera.BodyJoints;
import com.google.mediapipe.solutions.lindera.CameraRotation;
import com.google.mediapipe.solutions.lindera.ComputerVisionPlugin;
import com.google.mediapipe.solutions.lindera.InputSource;
import com.google.mediapipe.solutions.lindera.Lindera;
import org.json.JSONException;
@ -80,11 +79,11 @@ public class MainActivity extends AppCompatActivity {
lindera = new Lindera(plugin);
lindera.setupVideoPicker(this);
List<String> cameras = lindera.getAvailableCameras();
// FRONT or BACK
lindera.setCamera("BACK");
lindera.setCameraRotation(CameraRotation.AUTOMATIC);
lindera.fpsHelper.onFpsUpdate = new Consumer<Double>() {
@Override
public void accept(Double fps) {
@ -99,36 +98,55 @@ public class MainActivity extends AppCompatActivity {
}
private void startDetectionPipeline(InputSource inputSource){
if (!isLinderaInitialized) {
modelLoadAsyncDialogue(()->{
if (inputSource==InputSource.VIDEO){
lindera.setInputSource(InputSource.VIDEO);
}
lindera.initialize(findViewById(R.id.preview_display_layout), MainActivity.this);
isLinderaInitialized = true;
findViewById(R.id.button_start_video).setVisibility(View.GONE);
findViewById(R.id.button_start_camera).setVisibility(View.GONE);
findViewById(R.id.button_set_model).setVisibility(View.VISIBLE);
findViewById(R.id.button_toggle_landmarks).setVisibility(View.VISIBLE);
findViewById(R.id.button_capture_logging).setVisibility(View.VISIBLE);
updateLandmarkButtonText();
updateModelComplexityButtonText();
if (inputSource==InputSource.VIDEO) {
lindera.pickVideo();
}
});
}
isDetectionStarted = !isDetectionStarted;
}
/**
* Sets up the UI components for the live demo with camera input.
*/
private void setupLiveDemoUiComponents() {
Button startDetectionButton = findViewById(R.id.button_start_detection);
Button startCameraButton = findViewById(R.id.button_start_camera);
Button startVideoButton = findViewById(R.id.button_start_video);
Button toggleLandmarks = findViewById(R.id.button_toggle_landmarks);
Button modelComplexity = findViewById(R.id.button_set_model);
Button startCapture = findViewById(R.id.button_capture_logging);
FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
startDetectionButton.setOnClickListener(
startCameraButton.setOnClickListener(
v -> {
// startCameraButton.setVisibility(View.GONE);
if (!isLinderaInitialized) {
modelLoadAsyncDialogue(()->{
lindera.initialize(frameLayout, MainActivity.this);
isLinderaInitialized = true;
startDetectionButton.setVisibility(View.GONE);
findViewById(R.id.button_set_model).setVisibility(View.VISIBLE);
findViewById(R.id.button_toggle_landmarks).setVisibility(View.VISIBLE);
findViewById(R.id.button_capture_logging).setVisibility(View.VISIBLE);
updateLandmarkButtonText();
updateModelComplexityButtonText();
});
startDetectionPipeline(InputSource.CAMERA);
}
isDetectionStarted = !isDetectionStarted;
});
startVideoButton.setOnClickListener(
v -> {
startDetectionPipeline(InputSource.VIDEO);
});

11
mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/res/layout/activity_main.xml
View File

@ -15,11 +15,18 @@
android:orientation="horizontal">
<Button
android:id="@+id/button_start_detection"
android:id="@+id/button_start_camera"
style="?android:attr/buttonBarButtonStyle"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="Start Detection" />
android:text="Start Camera" />
<Button
android:id="@+id/button_start_video"
style="?android:attr/buttonBarButtonStyle"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="Start Video" />
<Button
android:id="@+id/button_toggle_landmarks"

5
mediapipe/java/com/google/mediapipe/solutions/copperlabs/copperlabs-lindera/src/main/java/com/google/mediapipe/solutions/lindera/InputSource.java Normal file
View File

@ -0,0 +1,5 @@
package com.google.mediapipe.solutions.lindera;
public enum InputSource {
VIDEO, CAMERA
}

64
mediapipe/java/com/google/mediapipe/solutions/copperlabs/copperlabs-lindera/src/main/java/com/google/mediapipe/solutions/lindera/Lindera.java
View File

@ -1,15 +1,21 @@
package com.google.mediapipe.solutions.lindera;
import android.app.Activity;
import android.content.Intent;
import android.provider.MediaStore;
import android.util.Log;
import android.view.View;
import android.view.ViewGroup;
import androidx.activity.result.ActivityResultLauncher;
import androidx.activity.result.contract.ActivityResultContracts;
import androidx.appcompat.app.AppCompatActivity;
import com.google.common.collect.ImmutableList;
import com.google.mediapipe.formats.proto.LandmarkProto;
import com.google.mediapipe.solutioncore.CameraInput;
import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
import com.google.mediapipe.solutioncore.VideoInput;
import com.google.mediapipe.solutions.posetracking.PoseTracking;
import com.google.mediapipe.solutions.posetracking.PoseTrackingOptions;
import com.google.mediapipe.solutions.posetracking.PoseTrackingResult;
@ -24,7 +30,7 @@ public class Lindera {
private ComputerVisionPlugin plugin;
public FpsHelper fpsHelper = new FpsHelper();
private PoseTracking poseTracking;
private InputSource inputSource = InputSource.CAMERA;
private CameraRotation cameraRotation = CameraRotation.AUTOMATIC;
// Live camera demo UI and camera components.
@ -34,7 +40,9 @@ public class Lindera {
private AppCompatActivity appCompatActivity;
private ViewGroup computerVisionContainerView;
private PoseTrackingResultGlRenderer solutionRenderer;
private VideoInput videoInput;
private ActivityResultLauncher<Intent> videoGetter;
public Lindera(ComputerVisionPlugin plugin){
this.plugin = plugin;
}
@ -72,13 +80,41 @@ public class Lindera {
this.computerVisionContainerView = computerVisionContainerView;
this.appCompatActivity = appCompatActivity;
startDetection();
}
public void setCameraRotation(CameraRotation cameraRotation){
this.cameraRotation = cameraRotation;
}
public void setInputSource(InputSource source){
this.inputSource = source;
}
/**
* Required for picking videos
* @param appCompatActivity
*/
public void setupVideoPicker(AppCompatActivity appCompatActivity){
videoGetter =
appCompatActivity.registerForActivityResult(
new ActivityResultContracts.StartActivityForResult(),
result -> {
Intent resultIntent = result.getData();
if (resultIntent != null) {
if (result.getResultCode() == Activity.RESULT_OK) {
glSurfaceView.post(
() ->
videoInput.start(
appCompatActivity,
resultIntent.getData(),
poseTracking.getGlContext(),
glSurfaceView.getWidth(),
glSurfaceView.getHeight()));
}
}
});
}
private void setupEventListener() {
poseTracking.setResultListener(
poseTrackingResult -> {
@ -132,6 +168,12 @@ public class Lindera {
);
}
public void pickVideo(){
inputSource = InputSource.VIDEO;
Intent pickVideoIntent = new Intent(Intent.ACTION_PICK);
pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*");
videoGetter.launch(pickVideoIntent);
}
public void startDetection(PoseTrackingOptions options){
// ensure that class is initalized
assert (appCompatActivity != null);
@ -142,10 +184,14 @@ public class Lindera {
options);
poseTracking.setErrorListener(
(message, e) -> Log.e("Lindera", "MediaPipe Pose Tracking error:" + message));
cameraInput = new CameraInput(appCompatActivity);
cameraInput.setNewFrameListener(textureFrame -> poseTracking.send(textureFrame));
if (inputSource==InputSource.CAMERA) {
cameraInput = new CameraInput(appCompatActivity);
cameraInput.setNewFrameListener(textureFrame -> poseTracking.send(textureFrame));
}else if (inputSource==InputSource.VIDEO){
videoInput = new VideoInput(appCompatActivity);
videoInput.setNewFrameListener(textureFrame -> poseTracking.send(textureFrame));
}
// Initializes a new Gl surface view with a user-defined PoseTrackingResultGlRenderer.
glSurfaceView =
new SolutionGlSurfaceView<>(
@ -162,7 +208,9 @@ public class Lindera {
// The runnable to start camera after the gl surface view is attached.
// For video input source, videoInput.start() will be called when the video uri is available.
glSurfaceView.post(this::startCamera);
if (inputSource==InputSource.CAMERA) {
glSurfaceView.post(this::startCamera);
}
// Updates the preview layout.
computerVisionContainerView.removeAllViewsInLayout();
@ -179,6 +227,10 @@ public class Lindera {
if (glSurfaceView != null) {
glSurfaceView.setVisibility(View.GONE);
}
if (videoInput != null) {
videoInput.setNewFrameListener(null);
videoInput.close();
}
if (poseTracking != null) {
poseTracking.close();
}

48
mediapipe/swift/solutions/lindera/Lindera.swift
View File

@ -2,6 +2,12 @@
import UIKit
#if arch(arm64)
import MPPoseTracking
import AVFoundation
public enum InputSource {
case CAMERA,VIDEO
}
/// A helper class to run the Pose Tracking API
@ -20,7 +26,45 @@ public final class Lindera{
public func setFpsDelegate(fpsDelegate: @escaping (_ fps:Double)->Void){
fpsHelper.onFpsUpdate = fpsDelegate;
}
public func startVideo(asset:AVAsset){
self.videoSource = MPPPlayerInputSource(avAsset: asset)
self.videoSource.setDelegate(self.poseTracking, queue: self.poseTracking.videoQueue)
DispatchQueue.main.async { [weak self] in
if let self = self {
// set our rendering layer frame according to cameraView boundry
self.poseTracking.renderer.layer.frame = self.cameraView.layer.bounds
// attach render CALayer on cameraView to render output to
self.cameraView.layer.addSublayer(self.poseTracking.renderer.layer)
}
}
self.poseTracking.videoQueue.async(execute:{ [weak self] in
self?.videoSource.start()
})
}
public func setInputSource(inputSource:InputSource){
if (self.inputSource==inputSource) {return}
switch(inputSource){
case .CAMERA:
self.cameraSource.setDelegate(self.poseTracking, queue: self.poseTracking.videoQueue)
break
case .VIDEO:
break
}
self.inputSource = inputSource
}
// Get the camera UI View that may contain landmarks drawing
public var cameraView: UIView {
return self.linderaExerciseSession
@ -57,6 +101,7 @@ public final class Lindera{
startPoseTracking()
}
public func startCamera(_ completion: ((Result<Void, Error>) -> Void)? = nil) {
// set our rendering layer frame according to cameraView boundry
@ -182,8 +227,9 @@ public final class Lindera{
// attach Mediapipe camera helper to our class
let cameraSource = MPPCameraInputSource()
var videoSource = MPPPlayerInputSource()
var inputSource:InputSource = InputSource.CAMERA
}