From 208752d58e2852d1b50f503025ae9781e3fa9997 Mon Sep 17 00:00:00 2001
From: Mautisim Munir <mautisim@edgestore.ai>
Date: Thu, 10 Nov 2022 00:02:09 +0500
Subject: [PATCH] added video api support in java

---
 .../posetracking_lindera/MainActivity.java    | 58 +++++++++++------
 .../app/src/main/res/layout/activity_main.xml | 11 +++-
 .../solutions/lindera/InputSource.java        |  5 ++
 .../mediapipe/solutions/lindera/Lindera.java  | 64 +++++++++++++++++--
 4 files changed, 110 insertions(+), 28 deletions(-)
 create mode 100644 mediapipe/java/com/google/mediapipe/solutions/copperlabs/copperlabs-lindera/src/main/java/com/google/mediapipe/solutions/lindera/InputSource.java

diff --git a/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java
index f4d55641d..61adde466 100644
--- a/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java
+++ b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java
@@ -33,9 +33,8 @@ import androidx.appcompat.app.AppCompatActivity;
 
 import com.afollestad.materialdialogs.MaterialDialog;
 import com.google.mediapipe.R;
-import com.google.mediapipe.solutions.lindera.BodyJoints;
 import com.google.mediapipe.solutions.lindera.CameraRotation;
-import com.google.mediapipe.solutions.lindera.ComputerVisionPlugin;
+import com.google.mediapipe.solutions.lindera.InputSource;
 import com.google.mediapipe.solutions.lindera.Lindera;
 
 import org.json.JSONException;
@@ -80,11 +79,11 @@ public class MainActivity extends AppCompatActivity {
 
 
         lindera = new Lindera(plugin);
+        lindera.setupVideoPicker(this);
         List<String> cameras = lindera.getAvailableCameras();
         // FRONT or BACK
         lindera.setCamera("FRONT");
         lindera.setCameraRotation(CameraRotation.AUTOMATIC);
-
         lindera.fpsHelper.onFpsUpdate = new Consumer<Double>() {
             @Override
             public void accept(Double fps) {
@@ -99,36 +98,55 @@ public class MainActivity extends AppCompatActivity {
     }
 
 
+    private void startDetectionPipeline(InputSource inputSource){
+        if (!isLinderaInitialized) {
+            modelLoadAsyncDialogue(()->{
+                if (inputSource==InputSource.VIDEO){
+                    lindera.setInputSource(InputSource.VIDEO);
+                }
+                lindera.initialize(findViewById(R.id.preview_display_layout), MainActivity.this);
+                isLinderaInitialized = true;
+                findViewById(R.id.button_start_video).setVisibility(View.GONE);
+                findViewById(R.id.button_start_camera).setVisibility(View.GONE);
+                findViewById(R.id.button_set_model).setVisibility(View.VISIBLE);
+                findViewById(R.id.button_toggle_landmarks).setVisibility(View.VISIBLE);
+                findViewById(R.id.button_capture_logging).setVisibility(View.VISIBLE);
+
+                updateLandmarkButtonText();
+                updateModelComplexityButtonText();
+                if (inputSource==InputSource.VIDEO) {
+                    lindera.pickVideo();
+                }
+
+            });
+
+
+        }
+        isDetectionStarted = !isDetectionStarted;
+    }
     /**
      * Sets up the UI components for the live demo with camera input.
      */
     private void setupLiveDemoUiComponents() {
 
-        Button startDetectionButton = findViewById(R.id.button_start_detection);
+        Button startCameraButton = findViewById(R.id.button_start_camera);
+        Button startVideoButton = findViewById(R.id.button_start_video);
         Button toggleLandmarks = findViewById(R.id.button_toggle_landmarks);
         Button modelComplexity = findViewById(R.id.button_set_model);
         Button startCapture = findViewById(R.id.button_capture_logging);
         FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
 
-        startDetectionButton.setOnClickListener(
+        startCameraButton.setOnClickListener(
                 v -> {
-//                    startCameraButton.setVisibility(View.GONE);
-                    if (!isLinderaInitialized) {
-                        modelLoadAsyncDialogue(()->{
-                            lindera.initialize(frameLayout, MainActivity.this);
-                            isLinderaInitialized = true;
-                            startDetectionButton.setVisibility(View.GONE);
-                            findViewById(R.id.button_set_model).setVisibility(View.VISIBLE);
-                            findViewById(R.id.button_toggle_landmarks).setVisibility(View.VISIBLE);
-                            findViewById(R.id.button_capture_logging).setVisibility(View.VISIBLE);
-
-                            updateLandmarkButtonText();
-                            updateModelComplexityButtonText();
-                        });
+                  startDetectionPipeline(InputSource.CAMERA);
 
 
-                    }
-                    isDetectionStarted = !isDetectionStarted;
+
+                });
+        startVideoButton.setOnClickListener(
+                v -> {
+                    startDetectionPipeline(InputSource.VIDEO);
+
 
 
                 });
diff --git a/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/res/layout/activity_main.xml b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/res/layout/activity_main.xml
index 975690cb0..a2739ccc3 100644
--- a/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/res/layout/activity_main.xml
+++ b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/app/src/main/res/layout/activity_main.xml
@@ -15,11 +15,18 @@
         android:orientation="horizontal">
 
         <Button
-            android:id="@+id/button_start_detection"
+            android:id="@+id/button_start_camera"
             style="?android:attr/buttonBarButtonStyle"
             android:layout_width="wrap_content"
             android:layout_height="wrap_content"
-            android:text="Start Detection" />
+            android:text="Start Camera" />
+
+        <Button
+            android:id="@+id/button_start_video"
+            style="?android:attr/buttonBarButtonStyle"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:text="Start Video" />
 
         <Button
             android:id="@+id/button_toggle_landmarks"
diff --git a/mediapipe/java/com/google/mediapipe/solutions/copperlabs/copperlabs-lindera/src/main/java/com/google/mediapipe/solutions/lindera/InputSource.java b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/copperlabs-lindera/src/main/java/com/google/mediapipe/solutions/lindera/InputSource.java
new file mode 100644
index 000000000..ac05f7e25
--- /dev/null
+++ b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/copperlabs-lindera/src/main/java/com/google/mediapipe/solutions/lindera/InputSource.java
@@ -0,0 +1,5 @@
+package com.google.mediapipe.solutions.lindera;
+
+public enum InputSource {
+    VIDEO, CAMERA
+}
diff --git a/mediapipe/java/com/google/mediapipe/solutions/copperlabs/copperlabs-lindera/src/main/java/com/google/mediapipe/solutions/lindera/Lindera.java b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/copperlabs-lindera/src/main/java/com/google/mediapipe/solutions/lindera/Lindera.java
index 3e861efec..358075f04 100644
--- a/mediapipe/java/com/google/mediapipe/solutions/copperlabs/copperlabs-lindera/src/main/java/com/google/mediapipe/solutions/lindera/Lindera.java
+++ b/mediapipe/java/com/google/mediapipe/solutions/copperlabs/copperlabs-lindera/src/main/java/com/google/mediapipe/solutions/lindera/Lindera.java
@@ -1,15 +1,21 @@
 package com.google.mediapipe.solutions.lindera;
 
+import android.app.Activity;
+import android.content.Intent;
+import android.provider.MediaStore;
 import android.util.Log;
 import android.view.View;
 import android.view.ViewGroup;
 
+import androidx.activity.result.ActivityResultLauncher;
+import androidx.activity.result.contract.ActivityResultContracts;
 import androidx.appcompat.app.AppCompatActivity;
 
 import com.google.common.collect.ImmutableList;
 import com.google.mediapipe.formats.proto.LandmarkProto;
 import com.google.mediapipe.solutioncore.CameraInput;
 import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
+import com.google.mediapipe.solutioncore.VideoInput;
 import com.google.mediapipe.solutions.posetracking.PoseTracking;
 import com.google.mediapipe.solutions.posetracking.PoseTrackingOptions;
 import com.google.mediapipe.solutions.posetracking.PoseTrackingResult;
@@ -24,7 +30,7 @@ public class Lindera {
     private ComputerVisionPlugin plugin;
     public FpsHelper fpsHelper = new FpsHelper();
     private PoseTracking poseTracking;
-    
+    private InputSource inputSource = InputSource.CAMERA;
     private CameraRotation cameraRotation = CameraRotation.AUTOMATIC;
     
     // Live camera demo UI and camera components.
@@ -34,7 +40,9 @@ public class Lindera {
     private AppCompatActivity appCompatActivity;
     private  ViewGroup computerVisionContainerView;
     private PoseTrackingResultGlRenderer solutionRenderer;
-    
+    private VideoInput videoInput;
+    private ActivityResultLauncher<Intent> videoGetter;
+
     public Lindera(ComputerVisionPlugin plugin){
         this.plugin = plugin;
     }
@@ -72,13 +80,41 @@ public class Lindera {
 
         this.computerVisionContainerView = computerVisionContainerView;
         this.appCompatActivity = appCompatActivity;
+
         startDetection();
     }
 
     public void setCameraRotation(CameraRotation cameraRotation){
         this.cameraRotation = cameraRotation;
     }
+    public void setInputSource(InputSource source){
+        this.inputSource = source;
+    }
 
+    /**
+     * Required for picking videos
+     * @param appCompatActivity
+     */
+    public void setupVideoPicker(AppCompatActivity appCompatActivity){
+        videoGetter =
+                appCompatActivity.registerForActivityResult(
+                        new ActivityResultContracts.StartActivityForResult(),
+                        result -> {
+                            Intent resultIntent = result.getData();
+                            if (resultIntent != null) {
+                                if (result.getResultCode() == Activity.RESULT_OK) {
+                                    glSurfaceView.post(
+                                            () ->
+                                                    videoInput.start(
+                                                            appCompatActivity,
+                                                            resultIntent.getData(),
+                                                            poseTracking.getGlContext(),
+                                                            glSurfaceView.getWidth(),
+                                                            glSurfaceView.getHeight()));
+                                }
+                            }
+                        });
+    }
     private void setupEventListener() {
         poseTracking.setResultListener(
             poseTrackingResult -> {
@@ -132,6 +168,12 @@ public class Lindera {
                 );    
     }
 
+    public void pickVideo(){
+        inputSource = InputSource.VIDEO;
+        Intent pickVideoIntent = new Intent(Intent.ACTION_PICK);
+        pickVideoIntent.setDataAndType(MediaStore.Video.Media.INTERNAL_CONTENT_URI, "video/*");
+        videoGetter.launch(pickVideoIntent);
+    }
     public void startDetection(PoseTrackingOptions options){
         // ensure that class is initalized
         assert (appCompatActivity != null);
@@ -142,10 +184,14 @@ public class Lindera {
                         options);
         poseTracking.setErrorListener(
                 (message, e) -> Log.e("Lindera", "MediaPipe Pose Tracking error:" + message));
-        cameraInput = new CameraInput(appCompatActivity);
-
-        cameraInput.setNewFrameListener(textureFrame -> poseTracking.send(textureFrame));
+        if (inputSource==InputSource.CAMERA) {
+            cameraInput = new CameraInput(appCompatActivity);
 
+            cameraInput.setNewFrameListener(textureFrame -> poseTracking.send(textureFrame));
+        }else if (inputSource==InputSource.VIDEO){
+            videoInput = new VideoInput(appCompatActivity);
+            videoInput.setNewFrameListener(textureFrame -> poseTracking.send(textureFrame));
+        }
         // Initializes a new Gl surface view with a user-defined PoseTrackingResultGlRenderer.
         glSurfaceView = 
             new SolutionGlSurfaceView<>(
@@ -162,7 +208,9 @@ public class Lindera {
         
         // The runnable to start camera after the gl surface view is attached.
         // For video input source, videoInput.start() will be called when the video uri is available.
-        glSurfaceView.post(this::startCamera);
+        if (inputSource==InputSource.CAMERA) {
+            glSurfaceView.post(this::startCamera);
+        }
         // Updates the preview layout.
 
         computerVisionContainerView.removeAllViewsInLayout();
@@ -179,6 +227,10 @@ public class Lindera {
         if (glSurfaceView != null) {
             glSurfaceView.setVisibility(View.GONE);
         }
+        if (videoInput != null) {
+            videoInput.setNewFrameListener(null);
+            videoInput.close();
+        }
         if (poseTracking != null) {
             poseTracking.close();
         }