lindera callback working

mediapipe/examples/android/solutions/posetracking-camera/src/main/java/com/google/mediapipe/examples/posetracking_camera/MainActivity.java

@@ -15,144 +15,142 @@
 package com.google.mediapipe.examples.posetracking_camera;
 
 import android.content.Intent;
-import android.graphics.Bitmap;
-import android.graphics.Matrix;
 import android.os.Bundle;
-import android.os.Looper;
-import android.provider.MediaStore;
-import androidx.appcompat.app.AppCompatActivity;
 import android.util.Log;
 import android.view.Surface;
 import android.view.View;
 import android.widget.Button;
 import android.widget.FrameLayout;
-import android.widget.ImageView;
 
 import androidx.activity.result.ActivityResultLauncher;
-import androidx.activity.result.contract.ActivityResultContracts;
+import androidx.appcompat.app.AppCompatActivity;
-import androidx.exifinterface.media.ExifInterface;
+
-// ContentResolver dependency
 import com.google.mediapipe.formats.proto.LandmarkProto;
 import com.google.mediapipe.solutioncore.CameraInput;
 import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
-import com.google.mediapipe.solutioncore.VideoInput;
 import com.google.mediapipe.solutions.posetracking.PoseTracking;
 import com.google.mediapipe.solutions.posetracking.PoseTrackingOptions;
 import com.google.mediapipe.solutions.posetracking.PoseTrackingResult;
 import com.google.mediapipe.solutions.posetracking.PoseTrackingResultGlRenderer;
-import java.io.IOException;
-import java.io.InputStream;
 
 
-/** Main activity of MediaPipe Face Detection app. */
+/**
+ * Main activity of MediaPipe Face Detection app.
+ */
 public class MainActivity extends AppCompatActivity {
-  private static final String TAG = "MainActivity";
+    private static final String TAG = "MainActivity";
-  private static final int rotation = Surface.ROTATION_0;
+    private static final int rotation = Surface.ROTATION_0;
-  private PoseTracking poseTracking;
+    private PoseTracking poseTracking;
 
 
+    private ActivityResultLauncher<Intent> videoGetter;
+    // Live camera demo UI and camera components.
+    private CameraInput cameraInput;
 
-  private ActivityResultLauncher<Intent> videoGetter;
+    private SolutionGlSurfaceView<PoseTrackingResult> glSurfaceView;
-  // Live camera demo UI and camera components.
-  private CameraInput cameraInput;
 
-  private SolutionGlSurfaceView<PoseTrackingResult> glSurfaceView;
+    @Override
+    protected void onCreate(Bundle savedInstanceState) {
+        super.onCreate(savedInstanceState);
+        setContentView(R.layout.activity_main);
+        disableRedundantUI();
 
-  @Override
+        setupLiveDemoUiComponents();
-  protected void onCreate(Bundle savedInstanceState) {
-    super.onCreate(savedInstanceState);
-    setContentView(R.layout.activity_main);
-    disableRedundantUI();
 
-    setupLiveDemoUiComponents();
-
-  }
-
-
-  /** Sets up the UI components for the live demo with camera input. */
-  private void setupLiveDemoUiComponents() {
-
-    Button startCameraButton = findViewById(R.id.button_start_camera);
-    startCameraButton.setOnClickListener(
-            v -> {
-              setupStreamingModePipeline();
-              startCameraButton.setVisibility(View.GONE);
-            });
-  }
-  /**Disables unecesary UI buttons*/
-  private  void disableRedundantUI(){
-    findViewById(R.id.button_load_picture).setVisibility(View.GONE);
-    findViewById(R.id.button_load_video).setVisibility(View.GONE);
-
-  }
-
-  /** Sets up core workflow for streaming mode. */
-  private void setupStreamingModePipeline() {
-    // Initializes a new MediaPipe Face Detection solution instance in the streaming mode.
-    poseTracking =
-            new PoseTracking(
-                    this,
-                    PoseTrackingOptions.builder()
-                            .setStaticImageMode(false)
-                            .setLandmarkVisibility(true)
-                            .setModelComplexity(0)
-                            .setSmoothLandmarks(true)
-                            .build());
-    poseTracking.setErrorListener(
-            (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
-     cameraInput = new CameraInput(this);
-
-     cameraInput.setNewFrameListener(textureFrame -> poseTracking.send(textureFrame));
-
-
-    // Initializes a new Gl surface view with a user-defined PoseTrackingResultGlRenderer.
-    glSurfaceView =
-            new SolutionGlSurfaceView<>(
-                    this, poseTracking.getGlContext(), poseTracking.getGlMajorVersion());
-    glSurfaceView.setSolutionResultRenderer(new PoseTrackingResultGlRenderer());
-    glSurfaceView.setRenderInputImage(true);
-    poseTracking.setResultListener(
-            poseTrackingResult -> {
-              logExampleKeypoint(poseTrackingResult);
-              glSurfaceView.setRenderData(poseTrackingResult);
-              glSurfaceView.requestRender();
-            });
-
-    // The runnable to start camera after the gl surface view is attached.
-    // For video input source, videoInput.start() will be called when the video uri is available.
-      glSurfaceView.post(this::startCamera);
-
-    // Updates the preview layout.
-      FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
-      frameLayout.removeAllViewsInLayout();
-      frameLayout.addView(glSurfaceView);
-    glSurfaceView.setVisibility(View.VISIBLE);
-    frameLayout.requestLayout();
-  }
-
-  private void startCamera() {
-    cameraInput.getConverter(poseTracking.getGlContext()).setRotation(rotation);
-    cameraInput.start(
-            this,
-            poseTracking.getGlContext(),
-            CameraInput.CameraFacing.FRONT,
-            glSurfaceView.getWidth(),
-            glSurfaceView.getHeight());
-  }
-
-
-  private void logExampleKeypoint(
-          PoseTrackingResult result) {
-    if (result.multiPoseLandmarks().isEmpty()) {
-      return;
     }
-    LandmarkProto.Landmark exampleLandmark = result.multiPoseLandmarks().get(PoseTrackingResult.NOSE);
-      Log.i(
-              TAG,
-              String.format(
-                      "Pose Landmark Landmark of Nose: x=%f, y=%f, z=%f",
-                      exampleLandmark.getX() , exampleLandmark.getY(),exampleLandmark.getZ() ));
 
 
-  }
+    /**
+     * Sets up the UI components for the live demo with camera input.
+     */
+    private void setupLiveDemoUiComponents() {
+
+        Button startCameraButton = findViewById(R.id.button_start_camera);
+        startCameraButton.setOnClickListener(
+                v -> {
+                    setupStreamingModePipeline();
+                    startCameraButton.setVisibility(View.GONE);
+                });
+    }
+
+    /**
+     * Disables unecesary UI buttons
+     */
+    private void disableRedundantUI() {
+        findViewById(R.id.button_load_picture).setVisibility(View.GONE);
+        findViewById(R.id.button_load_video).setVisibility(View.GONE);
+
+    }
+
+    /**
+     * Sets up core workflow for streaming mode.
+     */
+    private void setupStreamingModePipeline() {
+        // Initializes a new MediaPipe Face Detection solution instance in the streaming mode.
+        poseTracking =
+                new PoseTracking(
+                        this,
+                        PoseTrackingOptions.builder()
+                                .setStaticImageMode(false)
+                                .setLandmarkVisibility(true)
+                                .setModelComplexity(0)
+                                .setSmoothLandmarks(true)
+                                .build());
+        poseTracking.setErrorListener(
+                (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message));
+        cameraInput = new CameraInput(this);
+
+        cameraInput.setNewFrameListener(textureFrame -> poseTracking.send(textureFrame));
+
+
+        // Initializes a new Gl surface view with a user-defined PoseTrackingResultGlRenderer.
+        glSurfaceView =
+                new SolutionGlSurfaceView<>(
+                        this, poseTracking.getGlContext(), poseTracking.getGlMajorVersion());
+        glSurfaceView.setSolutionResultRenderer(new PoseTrackingResultGlRenderer());
+        glSurfaceView.setRenderInputImage(true);
+        poseTracking.setResultListener(
+                poseTrackingResult -> {
+                    logExampleKeypoint(poseTrackingResult);
+                    glSurfaceView.setRenderData(poseTrackingResult);
+                    glSurfaceView.requestRender();
+                });
+
+        // The runnable to start camera after the gl surface view is attached.
+        // For video input source, videoInput.start() will be called when the video uri is available.
+        glSurfaceView.post(this::startCamera);
+
+        // Updates the preview layout.
+        FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
+        frameLayout.removeAllViewsInLayout();
+        frameLayout.addView(glSurfaceView);
+        glSurfaceView.setVisibility(View.VISIBLE);
+        frameLayout.requestLayout();
+    }
+
+    private void startCamera() {
+        cameraInput.getConverter(poseTracking.getGlContext()).setRotation(rotation);
+        cameraInput.start(
+                this,
+                poseTracking.getGlContext(),
+                CameraInput.CameraFacing.FRONT,
+                glSurfaceView.getWidth(),
+                glSurfaceView.getHeight());
+    }
+
+
+    private void logExampleKeypoint(
+            PoseTrackingResult result) {
+        if (result.multiPoseLandmarks().isEmpty()) {
+            return;
+        }
+        LandmarkProto.Landmark exampleLandmark = result.multiPoseLandmarks().get(PoseTrackingResult.NOSE);
+        Log.i(
+                TAG,
+                String.format(
+                        "Pose Landmark Landmark of Nose: x=%f, y=%f, z=%f",
+                        exampleLandmark.getX(), exampleLandmark.getY(), exampleLandmark.getZ()));
+
+
+    }
 }

mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java

@@ -1,7 +1,18 @@
 package com.google.mediapipe.examples.posetracking_lindera;
 
+import android.util.Log;
+
+import com.google.mediapipe.solutions.posetracking.BodyJoints;
 import com.google.mediapipe.solutions.posetracking.ComputerVisionPlugin;
+import com.google.mediapipe.solutions.posetracking.XYZPointWithConfidence;
 
 public class ComputerVisionPluginImpl implements ComputerVisionPlugin {
 
+    @Override
+    public void bodyJoints(int timestamp, BodyJoints bodyJoints) {
+        XYZPointWithConfidence nose = bodyJoints.nose;
+        Log.v("ComputerVisionPluginImpl", String.format(
+
+                "Lindera BodyJoint of Nose: x=%f, y=%f, z=%f", nose.x, nose.y, nose.z));
+    }
 }

mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java

@@ -14,75 +14,67 @@
 
 package com.google.mediapipe.examples.posetracking_lindera;
 
-import android.content.Intent;
-
 import android.os.Bundle;
-
-import androidx.appcompat.app.AppCompatActivity;
-
-import android.os.Handler;
-import android.os.Looper;
-import android.view.Surface;
 import android.view.View;
 import android.widget.Button;
 import android.widget.FrameLayout;
 
-import androidx.activity.result.ActivityResultLauncher;
+import androidx.appcompat.app.AppCompatActivity;
 
-import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
 import com.google.mediapipe.solutions.posetracking.ComputerVisionPlugin;
 import com.google.mediapipe.solutions.posetracking.Lindera;
-import com.google.mediapipe.solutions.posetracking.PoseTrackingResultGlRenderer;
-
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 
 
-/** Main activity of MediaPipe Face Detection app. */
+/**
+ * Main activity of MediaPipe Face Detection app.
+ */
 public class MainActivity extends AppCompatActivity {
-  private static final String TAG = "MainActivity";
+    private static final String TAG = "MainActivity";
 
-  private Lindera lindera;
+    private Lindera lindera;
-  private ComputerVisionPlugin plugin;
+    private ComputerVisionPlugin plugin;
-  // Live camera demo UI and camera components.
+    // Live camera demo UI and camera components.
 
 
-
+    @Override
-  @Override
+    protected void onCreate(Bundle savedInstanceState) {
-  protected void onCreate(Bundle savedInstanceState) {
+        super.onCreate(savedInstanceState);
-    super.onCreate(savedInstanceState);
+        setContentView(R.layout.activity_main);
-    setContentView(R.layout.activity_main);
 
 
+        disableRedundantUI();
 
-    disableRedundantUI();
+        setupLiveDemoUiComponents();
-
+        plugin = new ComputerVisionPluginImpl();
-    setupLiveDemoUiComponents();
+        lindera = new Lindera(plugin);
-    plugin = new ComputerVisionPluginImpl();
+    }
-    lindera = new Lindera(plugin);
-  }
 
 
-  /** Sets up the UI components for the live demo with camera input. */
+    /**
-  private void setupLiveDemoUiComponents() {
+     * Sets up the UI components for the live demo with camera input.
+     */
+    private void setupLiveDemoUiComponents() {
 
-    Button startCameraButton = findViewById(R.id.button_start_camera);
+        Button startCameraButton = findViewById(R.id.button_start_camera);
-      FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
+        FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
 
-      startCameraButton.setOnClickListener(
+        startCameraButton.setOnClickListener(
-            v -> {
+                v -> {
-              startCameraButton.setVisibility(View.GONE);
+                    startCameraButton.setVisibility(View.GONE);
 
-              lindera.initialize(frameLayout, MainActivity.this);
+                    lindera.initialize(frameLayout, MainActivity.this);
 
-            });
+                });
-  }
+    }
-  /**Disables unecesary UI buttons*/
-  private  void disableRedundantUI(){
-    findViewById(R.id.button_load_picture).setVisibility(View.GONE);
-    findViewById(R.id.button_load_video).setVisibility(View.GONE);
 
-  }
+    /**
+     * Disables unecesary UI buttons
+     */
+    private void disableRedundantUI() {
+        findViewById(R.id.button_load_picture).setVisibility(View.GONE);
+        findViewById(R.id.button_load_video).setVisibility(View.GONE);
+
+    }
 
 
 }

mediapipe/java/com/google/mediapipe/solutions/posetracking/BUILD

@@ -117,7 +117,11 @@ android_library(
 
 android_library(
     name = "copperlabs-lindera-dummpy-interface",
-    srcs = ["ComputerVisionPlugin.java"],
+    srcs = [
+        "BodyJoints.java",
+        "ComputerVisionPlugin.java",
+        "XYZPointWithConfidence.java",
+    ],
     manifest = "AndroidManifest.xml",
     visibility = ["//visibility:public"],
     deps = [],

mediapipe/java/com/google/mediapipe/solutions/posetracking/ComputerVisionPlugin.java

@@ -1,4 +1,5 @@
 package com.google.mediapipe.solutions.posetracking;
 
 public interface ComputerVisionPlugin {
+    void bodyJoints(int timestamp, BodyJoints bodyJoints);
 }

mediapipe/java/com/google/mediapipe/solutions/posetracking/Lindera.java

@@ -7,6 +7,8 @@ import android.view.ViewGroup;
 
 import androidx.appcompat.app.AppCompatActivity;
 
+import com.google.common.collect.ImmutableList;
+import com.google.mediapipe.formats.proto.LandmarkProto;
 import com.google.mediapipe.solutioncore.CameraInput;
 import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
 
@@ -14,7 +16,8 @@ public class Lindera {
     private ComputerVisionPlugin plugin;
     private static final int rotation = Surface.ROTATION_0;
     private PoseTracking poseTracking;
-
+    // TODO: Verify that this is the timestamp used in Actual Plugin
+    private int timeStamp = 0;
     // Live camera demo UI and camera components.
     private CameraInput cameraInput;
     private SolutionGlSurfaceView<PoseTrackingResult> glSurfaceView;
@@ -28,6 +31,8 @@ public class Lindera {
         setupStreamingModePipeline(computerVisionContainerView,appCompatActivity);
 
     }
+
+    
     /** Sets up core workflow for streaming mode. */
     private void setupStreamingModePipeline(ViewGroup computerVisionContainerView,AppCompatActivity appCompatActivity) {
         // Initializes a new MediaPipe Face Detection solution instance in the streaming mode.
@@ -53,13 +58,8 @@ public class Lindera {
                         appCompatActivity, poseTracking.getGlContext(), poseTracking.getGlMajorVersion());
         glSurfaceView.setSolutionResultRenderer(new PoseTrackingResultGlRenderer());
         glSurfaceView.setRenderInputImage(true);
-        poseTracking.setResultListener(
-                poseTrackingResult -> {
-//                    logExampleKeypoint(poseTrackingResult);
-                    glSurfaceView.setRenderData(poseTrackingResult);
-                    glSurfaceView.requestRender();
-                });
 
+        setupEventListener();
         // The runnable to start camera after the gl surface view is attached.
         // For video input source, videoInput.start() will be called when the video uri is available.
         glSurfaceView.post(()->{this.startCamera(appCompatActivity);});
@@ -81,4 +81,81 @@ public class Lindera {
                 glSurfaceView.getHeight());
     }
 
+    public void setupEventListener() {
+    poseTracking.setResultListener(
+            poseTrackingResult -> {
+                glSurfaceView.setRenderData(poseTrackingResult);
+                glSurfaceView.requestRender();
+                ImmutableList<LandmarkProto.Landmark> landmarks = poseTrackingResult.multiPoseLandmarks();
+                timeStamp+=1;
+
+                if (landmarks.isEmpty()) return;
+
+                BodyJoints bodyJoints = new BodyJoints();
+                landmarksToBodyJoints(landmarks,bodyJoints);
+
+                plugin.bodyJoints(timeStamp, bodyJoints);
+            });
+    }
+    private void landmarkToXYZPointWithConfidence(LandmarkProto.Landmark landmark,XYZPointWithConfidence bodyJoint){
+        bodyJoint.x = landmark.getX();
+        bodyJoint.y = landmark.getY();
+        bodyJoint.z = landmark.getZ();
+        bodyJoint.confidence = landmark.getVisibility();
+    }
+    private void landmarksToBodyJoints(ImmutableList<LandmarkProto.Landmark> landmarks , BodyJoints bodyJoints){
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.NOSE), bodyJoints.nose);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE_INNER), bodyJoints.leftEyeInner);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE), bodyJoints.leftEye);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE_OUTER), bodyJoints.leftEyeOuter);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EYE_INNER), bodyJoints.rightEyeInner);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EYE), bodyJoints.rightEye);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EYE_OUTER), bodyJoints.rightEyeOuter);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EAR), bodyJoints.leftEar);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EAR), bodyJoints.rightEar);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.MOUTH_LEFT), bodyJoints.mouthLeft);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.MOUTH_RIGHT), bodyJoints.mouthRight);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_SHOULDER), bodyJoints.leftShoulder);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_SHOULDER), bodyJoints.rightShoulder);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_ELBOW), bodyJoints.leftElbow);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_ELBOW), bodyJoints.rightElbow);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_WRIST), bodyJoints.leftWrist);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_WRIST), bodyJoints.rightWrist);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_PINKY), bodyJoints.leftPinky);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_PINKY), bodyJoints.rightPinky);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_INDEX), bodyJoints.leftIndex);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_INDEX), bodyJoints.rightIndex);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_THUMB), bodyJoints.leftThumb);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_THUMB), bodyJoints.rightThumb);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_HIP), bodyJoints.leftHip);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_HIP), bodyJoints.rightHip);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_KNEE), bodyJoints.leftKnee);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_KNEE), bodyJoints.rightKnee);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_ANKLE), bodyJoints.rightAnkle);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_ANKLE), bodyJoints.leftAnkle);
+
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_HEEL), bodyJoints.rightHeel);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_HEEL), bodyJoints.leftHeel);
+
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_FOOT), bodyJoints.rightFoot);
+                landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_FOOT), bodyJoints.leftFoot);
+    }
+
+
+
+
 }

mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java

@@ -49,7 +49,7 @@ public class PoseTrackingResult extends ImageSolutionResult {
   public static final int LEFT_SHOULDER = 11;
   public static final int RIGHT_SHOULDER = 12;
   public static final int LEFT_ELBOW = 13;
-  public static final int RIGH_ELBOW = 14;
+  public static final int RIGHT_ELBOW = 14;
   public static final int LEFT_WRIST = 15;
   public static final int RIGHT_WRIST = 16;
   public static final int LEFT_PINKY = 17;