added interpolation with visualization

mediapipe/examples/android/solutions/posetracking-lindera/src/main/BUILD

@@ -20,6 +20,7 @@ android_binary(
     name = "posetracking-lindera",
     srcs = glob(["**/*.java"]),
     custom_package = "com.google.mediapipe.examples.posetracking_lindera",
+    dex_shards = 10,
     manifest = "AndroidManifest.xml",
     manifest_values = {
         "applicationId": "com.google.mediapipe.examples.posetracking_lindera",

mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java

@@ -88,7 +88,6 @@ public class ComputerVisionPluginImpl implements ComputerVisionPlugin {
 
                 }
             }
-            bodyJointsString = bodyJointsString.concat(interpolateJoints(bodyJoints));
             // remove the last equal sign
             bodyJointsString = bodyJointsString.substring(0,bodyJointsString.length()-1);
 
@@ -106,68 +105,11 @@ public class ComputerVisionPluginImpl implements ComputerVisionPlugin {
 
     }
 
-    String interpolateJoints(BodyJoints bodyJoints){
-
-        Map<String,XYZPointWithConfidence> pts = new HashMap<>();
-        pts.put("PE", getPelvis(bodyJoints));
-        pts.put("NN",bodyJoints.nose);
-        // Assuming Thorax is 1/3 of distance between shoulders and pelvis
-        XYZPointWithConfidence thorax = getSpinePoint(bodyJoints,1/3f);
-        // Assuming spine/middle back is 2/3 of distance between shoulders and pelvis
-        XYZPointWithConfidence spine = getSpinePoint(bodyJoints,2/3f);
-        pts.put("TH",thorax);
-        pts.put("SP",spine);
-        pts.put("HT",getHeadTop(bodyJoints));
-        final String[] bodyJointsString = {""};
-        pts.forEach((key,data)->{
-            bodyJointsString[0] = bodyJointsString[0].concat(String.format(key+":%f,%f,%f=",data.x,data.y,data.z));
-
-        });
-
-        return bodyJointsString[0];
 
 
 
 
 
-    }
-    XYZPointWithConfidence getPelvis(BodyJoints bodyJoints){
-        return getMiddleJoint(bodyJoints.leftHip,bodyJoints.rightHip);
-
-    }
-
-    XYZPointWithConfidence getJointBetweenPoints(XYZPointWithConfidence pt1,XYZPointWithConfidence pt2,float distance){
-        XYZPointWithConfidence midpt = new XYZPointWithConfidence();
-        midpt.x  = pt1.x + (pt2.x-pt1.x)*distance;
-        midpt.y  = pt1.y + (pt2.y-pt1.y)*distance;
-        midpt.z  = pt1.z + (pt2.z-pt1.z)*distance;
-
-
-        midpt.presence = min(pt1.presence,pt2.presence);
-        midpt.confidence = min(pt1.confidence,pt2.confidence);
-        return midpt;
-    }
-    XYZPointWithConfidence getMiddleJoint(XYZPointWithConfidence pt1,XYZPointWithConfidence pt2) {
-        XYZPointWithConfidence midpt = new XYZPointWithConfidence();
-        midpt.x  = (pt1.x + pt2.x)/2;
-        midpt.y  = (pt1.y + pt2.y)/2;
-        midpt.z  = (pt1.z + pt2.z)/2;
-
-        midpt.presence = min(pt1.presence,pt2.presence);
-        midpt.confidence = min(pt1.confidence,pt2.confidence);
-        return midpt;
-    }
-        XYZPointWithConfidence getSpinePoint(BodyJoints bodyJoints, float distanceFromShoulders){
-            XYZPointWithConfidence midShoulder = getMiddleJoint(bodyJoints.leftShoulder, bodyJoints.rightShoulder);
-            XYZPointWithConfidence pelvis = getPelvis(bodyJoints);
-            return getJointBetweenPoints(midShoulder,pelvis,distanceFromShoulders);
-    }
-
-    XYZPointWithConfidence getHeadTop(BodyJoints bodyJoints){
-        XYZPointWithConfidence middleEye = getMiddleJoint(bodyJoints.leftEye,bodyJoints.rightEye);
-        return getJointBetweenPoints(middleEye,bodyJoints.nose,2);
-    }
-
 
 
     @Override

mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java

@@ -83,7 +83,7 @@ public class MainActivity extends AppCompatActivity {
 
         findViewById(R.id.button_set_model).setVisibility(View.GONE);
         findViewById(R.id.button_toggle_landmarks).setVisibility(View.GONE);
-        findViewById(R.id.button_start_capture).setVisibility(View.GONE);
+        findViewById(R.id.button_capture_logging).setVisibility(View.GONE);
         setupLiveDemoUiComponents();
         plugin = new ComputerVisionPluginImpl();
 
@@ -116,7 +116,7 @@ public class MainActivity extends AppCompatActivity {
         Button startDetectionButton = findViewById(R.id.button_start_detection);
         Button toggleLandmarks = findViewById(R.id.button_toggle_landmarks);
         Button modelComplexity = findViewById(R.id.button_set_model);
-        Button startCapture = findViewById(R.id.button_start_capture);
+        Button startCapture = findViewById(R.id.button_capture_logging);
         FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
 
         startDetectionButton.setOnClickListener(
@@ -129,7 +129,7 @@ public class MainActivity extends AppCompatActivity {
                             startDetectionButton.setVisibility(View.GONE);
                             findViewById(R.id.button_set_model).setVisibility(View.VISIBLE);
                             findViewById(R.id.button_toggle_landmarks).setVisibility(View.VISIBLE);
-                            findViewById(R.id.button_start_capture).setVisibility(View.VISIBLE);
+                            findViewById(R.id.button_capture_logging).setVisibility(View.VISIBLE);
 
                             updateLandmarkButtonText();
                             updateModelComplexityButtonText();

mediapipe/examples/android/solutions/posetracking-lindera/src/main/res/layout/activity_main.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
     xmlns:tools="http://schemas.android.com/tools"
     android:layout_width="match_parent"
     android:layout_height="match_parent"
@@ -10,7 +11,8 @@
         style="?android:attr/buttonBarStyle"
         android:layout_width="match_parent"
         android:layout_height="wrap_content"
-        android:gravity="center">
+        android:gravity="center"
+        android:orientation="horizontal">
 
         <Button
             android:id="@+id/button_start_detection"
@@ -24,28 +26,32 @@
             style="?android:attr/buttonBarButtonStyle"
             android:layout_width="wrap_content"
             android:layout_height="wrap_content"
-            android:text="Show Landmarks" />
+            android:text="Landmarks" />
 
         <Button
             android:id="@+id/button_set_model"
             style="?android:attr/buttonBarButtonStyle"
             android:layout_width="wrap_content"
             android:layout_height="wrap_content"
-            android:text="Set Model" />
+            android:text="Model" />
 
         <Button
-            android:id="@+id/button_start_capture"
+            android:id="@+id/button_capture_logging"
             style="?android:attr/buttonBarButtonStyle"
             android:layout_width="wrap_content"
             android:layout_height="wrap_content"
             android:text="Start Capture" />
 
-        <TextView
-            android:id="@+id/fps_view"
-            android:layout_width="wrap_content"
-            android:layout_height="wrap_content" />
     </LinearLayout>
 
+    <FrameLayout
+        android:id="@+id/overlay"
+        style="?android:attr/buttonBarStyle"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:gravity="center"
+        android:orientation="horizontal">
+
         <FrameLayout
             android:id="@+id/preview_display_layout"
             android:layout_width="match_parent"
@@ -57,5 +63,20 @@
                 android:layout_height="wrap_content"
                 android:gravity="center"
                 android:text="@string/instruction" />
+
         </FrameLayout>
+
+        <TextView
+            android:id="@+id/fps_view"
+            android:layout_width="wrap_content"
+            android:layout_height="wrap_content"
+            android:textSize="30sp"
+            android:textStyle="bold"
+            android:layout_marginTop="20dp"
+            android:layout_marginLeft="20dp"
+            android:elevation="10dp"
+            android:gravity="center_horizontal|center_vertical"/>
+
+    </FrameLayout>
+
 </LinearLayout>

mediapipe/java/com/google/mediapipe/solutioncore/SolutionGlSurfaceViewRenderer.java

@@ -49,6 +49,9 @@ public class SolutionGlSurfaceViewRenderer<T extends ImageSolutionResult>
   public void setSolutionResultRenderer(ResultGlRenderer<T> resultGlRenderer) {
     this.resultGlRenderer = resultGlRenderer;
   }
+  public ResultGlRenderer<T> getSolutionResultRenderer(){
+    return this.resultGlRenderer;
+  }
 
   /**
    * Sets the next input {@link TextureFrame} and solution result to render.

mediapipe/java/com/google/mediapipe/solutions/lindera/BodyJoints.java

@@ -1,7 +1,7 @@
 package com.google.mediapipe.solutions.lindera;
 
 public class BodyJoints {
-    public XYZPointWithConfidence nose;
+    public XYZPointWithConfidence neckNose;
     
     public XYZPointWithConfidence leftEyeInner;
     public XYZPointWithConfidence leftEye;
@@ -52,16 +52,15 @@ public class BodyJoints {
     public XYZPointWithConfidence leftFoot;
     
     
-//    public XYZPointWithConfidence pelvis;
+    public XYZPointWithConfidence pelvis;
-//
+
-//    public XYZPointWithConfidence spine;
+    public XYZPointWithConfidence spine;
-//    public XYZPointWithConfidence thorax;
+    public XYZPointWithConfidence thorax;
-//    public XYZPointWithConfidence neckNose;
+    public XYZPointWithConfidence headTop;
-//    public XYZPointWithConfidence headTop;
 
 
     public BodyJoints() {
-         nose = new XYZPointWithConfidence();
+         neckNose = new XYZPointWithConfidence();
 
          leftEyeInner= new XYZPointWithConfidence();
          leftEye= new XYZPointWithConfidence();
@@ -111,5 +110,10 @@ public class BodyJoints {
          rightFoot= new XYZPointWithConfidence();
          leftFoot= new XYZPointWithConfidence();
 
+        pelvis = new XYZPointWithConfidence();
+         spine = new XYZPointWithConfidence();
+         thorax = new XYZPointWithConfidence();
+         headTop = new XYZPointWithConfidence();
+
     }
 }

mediapipe/java/com/google/mediapipe/solutions/lindera/Lindera.java

@@ -37,6 +37,7 @@ public class Lindera {
     private CameraInput.CameraFacing cameraFacing = CameraInput.CameraFacing.FRONT;
     private AppCompatActivity appCompatActivity;
     private  ViewGroup computerVisionContainerView;
+    private PoseTrackingResultGlRenderer solutionRenderer;
     
     public Lindera(ComputerVisionPlugin plugin){
         this.plugin = plugin;
@@ -45,6 +46,8 @@ public class Lindera {
     public void setLandmarksVisibility(boolean visible){
         this.poseTracking.options = PoseTrackingOptions.builder().withPoseTrackingOptions(this.poseTracking
                 .options).setLandmarkVisibility(visible).build();
+        solutionRenderer.setLandmarksVisibility(this.poseTracking.options.landmarkVisibility());
+        glSurfaceView.setSolutionResultRenderer(solutionRenderer);
     }
     public boolean getLandmarkVisibility(){
         return this.poseTracking.options.landmarkVisibility();
@@ -154,7 +157,9 @@ public class Lindera {
                 poseTracking.getGlContext(), 
                 poseTracking.getGlMajorVersion()
             );
-        glSurfaceView.setSolutionResultRenderer(new PoseTrackingResultGlRenderer());
+        solutionRenderer = new PoseTrackingResultGlRenderer();
+        solutionRenderer.setLandmarksVisibility(this.poseTracking.options.landmarkVisibility());
+        glSurfaceView.setSolutionResultRenderer(solutionRenderer);
         glSurfaceView.setRenderInputImage(true);
 
         setupEventListener();
@@ -193,7 +198,7 @@ public class Lindera {
     }
 
     private void landmarksToBodyJoints(ImmutableList<LandmarkProto.Landmark> landmarks , BodyJoints bodyJoints){
-        landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.NOSE), bodyJoints.nose);
+        landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.NOSE), bodyJoints.neckNose);
 
         landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE_INNER), bodyJoints.leftEyeInner);
         landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE), bodyJoints.leftEye);
@@ -242,6 +247,16 @@ public class Lindera {
 
         landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_FOOT), bodyJoints.rightFoot);
         landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_FOOT), bodyJoints.leftFoot);
+
+        // additional points
+        landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.PELVIS), bodyJoints.pelvis);
+        landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.SPINE), bodyJoints.spine);
+        landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.THORAX), bodyJoints.thorax);
+        landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.HEAD_TOP), bodyJoints.headTop);
+
+
+
+
     }
 
     private void startCamera() {

mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java

@@ -14,16 +14,21 @@
 
 package com.google.mediapipe.solutions.posetracking;
 
+import static java.lang.Math.min;
+
 import android.graphics.Bitmap;
+
 import com.google.auto.value.AutoBuilder;
 import com.google.common.collect.ImmutableList;
+import com.google.mediapipe.formats.proto.DetectionProto.Detection;
 import com.google.mediapipe.formats.proto.LandmarkProto;
 import com.google.mediapipe.framework.Packet;
 import com.google.mediapipe.framework.TextureFrame;
 import com.google.mediapipe.solutioncore.ImageSolutionResult;
-import com.google.mediapipe.formats.proto.DetectionProto.Detection;
-import java.util.List;
 
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 
 
 /**
@@ -35,6 +40,7 @@ public class PoseTrackingResult extends ImageSolutionResult {
     private final ImmutableList<Detection> multiPoseDetections;
     private final ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks;
 
+
     public static final int NOSE = 0;
     public static final int LEFT_EYE_INNER = 1;
     public static final int LEFT_EYE = 2;
@@ -69,6 +75,12 @@ public class PoseTrackingResult extends ImageSolutionResult {
     public static final int LEFT_FOOT = 31;
     public static final int RIGHT_FOOT = 32;
 
+    // Additional points not provided by MediaPipe
+    public static final int PELVIS = 33;
+    public static final int SPINE = 34;
+    public static final int THORAX = 35;
+    public static final int HEAD_TOP = 36;
+
 
     PoseTrackingResult(
             ImmutableList<Detection> multiPoseDetections, ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks, Packet imagePacket, long timestamp) {
@@ -87,19 +99,84 @@ public class PoseTrackingResult extends ImageSolutionResult {
         return multiPoseDetections;
     }
 
+    static LandmarkProto.Landmark getJointBetweenPoints(LandmarkProto.Landmark pt1, LandmarkProto.Landmark pt2, float distance) {
+        return LandmarkProto.Landmark.newBuilder()
+                .setX(pt1.getX() + (pt2.getX() - pt1.getX()) * distance)
+                .setY(pt1.getY() + (pt2.getY() - pt1.getY()) * distance)
+                .setZ(pt1.getZ() + (pt2.getZ() - pt1.getZ()) * distance)
+                .setPresence(min(pt1.getPresence(), pt2.getPresence()))
+                .setVisibility(min(pt1.getVisibility(), pt2.getVisibility())).build();
+    }
+
+    LandmarkProto.Landmark getPelvis(ImmutableList<LandmarkProto.Landmark> landmarks) {
+        // middle point b/w left hip and right hip
+        return getJointBetweenPoints(landmarks.get(LEFT_HIP), landmarks.get(RIGHT_HIP), 0.5f);
+    }
+
+    LandmarkProto.Landmark getSpinePoint(ImmutableList<LandmarkProto.Landmark> landmarks, float distanceFromShoulders) {
+        LandmarkProto.Landmark pelvis = getPelvis(landmarks);
+        // middle point b/w left shoulder and right shoulder
+        LandmarkProto.Landmark chest = getJointBetweenPoints(landmarks.get(LEFT_SHOULDER), landmarks.get(RIGHT_SHOULDER), 0.5f);
+
+        return getJointBetweenPoints(chest, pelvis, distanceFromShoulders);
+    }
+
+    LandmarkProto.Landmark getHeadTop(ImmutableList<LandmarkProto.Landmark> landmarks) {
+
+        float x = 0;
+        float y  = 0;
+        float z = 0;
+        final List<Integer> ptsIdx = Arrays.asList(LEFT_EAR, LEFT_EYE, RIGHT_EYE, RIGHT_EAR);
+        for (Integer i :ptsIdx){
+            LandmarkProto.Landmark landmark = landmarks.get(i);
+            x+=landmark.getX();
+            y+=landmark.getY();
+            z+=landmark.getZ();
+        }
+        x = x/ptsIdx.size();
+        y = y/ptsIdx.size();
+        z = z/ptsIdx.size();
+        LandmarkProto.Landmark midupper = LandmarkProto.Landmark.newBuilder().setX(x).setY(y).setZ(z).build();
+
+
+        LandmarkProto.Landmark midlower = getJointBetweenPoints(landmarks.get(MOUTH_LEFT), landmarks.get(MOUTH_RIGHT), 0.5f);
+        // 2 times the distance b/w nose and eyes
+        return getJointBetweenPoints(midlower, midupper, 2.5f);
+    }
+
+    ImmutableList<LandmarkProto.Landmark> getAdditionalLandmarksByInterpolation(ImmutableList<LandmarkProto.Landmark> originalLandmarks) {
+
+        if (originalLandmarks.isEmpty()) return originalLandmarks;
+        List<LandmarkProto.Landmark> landmarks = new ArrayList<>(originalLandmarks);
+        // pelvis
+        landmarks.add(getPelvis(originalLandmarks));
+        // spine assuming it is 2/3rd of distance b/w shoulders and pelvis
+        landmarks.add(getSpinePoint(originalLandmarks, 2 / 3f));
+        // thorax assuming it is 1/3rd of distance b/w shoulders and pelvis
+        landmarks.add(getSpinePoint(originalLandmarks, 1 / 3f));
+        // head top
+        landmarks.add(getHeadTop(originalLandmarks));
+
+
+        return ImmutableList.copyOf(landmarks);
+
+    }
 
     public ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks() {
-    return multiPoseLandmarks;
+        return getAdditionalLandmarksByInterpolation(multiPoseLandmarks);
     }
 
     public static Builder builder() {
         return new AutoBuilder_PoseTrackingResult_Builder();
     }
 
-  /** Builder for {@link PoseTrackingResult}. */
+    /**
+     * Builder for {@link PoseTrackingResult}.
+     */
     @AutoBuilder
     public abstract static class Builder {
         abstract Builder setMultiPoseDetections(List<Detection> value);
+
         abstract Builder setMultiPoseLandmarks(List<LandmarkProto.Landmark> value);
 
         abstract Builder setTimestamp(long value);

mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResultGlRenderer.java

@@ -16,7 +16,9 @@ package com.google.mediapipe.solutions.posetracking;
 
 import android.opengl.GLES20;
 
+import com.google.common.collect.ImmutableList;
 import com.google.mediapipe.formats.proto.DetectionProto.Detection;
+import com.google.mediapipe.formats.proto.LandmarkProto;
 import com.google.mediapipe.solutioncore.ResultGlRenderer;
 
 import java.nio.ByteBuffer;
@@ -52,7 +54,10 @@ public class PoseTrackingResultGlRenderer implements ResultGlRenderer<PoseTracki
   private int pointSizeHandle;
   private int projectionMatrixHandle;
   private int colorHandle;
-
+  private boolean areLandmarksVisible= true;
+  public void setLandmarksVisibility(boolean value){
+    areLandmarksVisible = value;
+  }
   private int loadShader(int type, String shaderCode) {
     int shader = GLES20.glCreateShader(type);
     GLES20.glShaderSource(shader, shaderCode);
@@ -79,17 +84,33 @@ public class PoseTrackingResultGlRenderer implements ResultGlRenderer<PoseTracki
    * **/
   @Override
   public void renderResult(PoseTrackingResult result, float[] projectionMatrix) {
-    if (result == null) {
+    if (result == null || !areLandmarksVisible) {
       return;
     }
 
     GLES20.glUseProgram(program);
     GLES20.glUniformMatrix4fv(projectionMatrixHandle, 1, false, projectionMatrix, 0);
-//    GLES20.glUniform1f(pointSizeHandle, KEYPOINT_SIZE);
+    GLES20.glUniform1f(pointSizeHandle, KEYPOINT_SIZE);
-//    int numDetectedFaces = result.multiPoseTrackings().size();
+    ImmutableList<LandmarkProto.Landmark> landmarks = result.multiPoseLandmarks();
-//    for (int i = 0; i < numDetectedFaces; ++i) {
+
-//      drawDetection(result.multiPoseTrackings().get(i));
+
-//    }
+      // Draw keypoints.
+      float[] points = new float[landmarks.size() * 2];
+      for (int i = 0; i < landmarks.size(); ++i) {
+        points[2 * i] = landmarks.get(i).getX();
+        points[2 * i + 1] = 1-landmarks.get(i).getY();
+      }
+      GLES20.glUniform4fv(colorHandle, 1, KEYPOINT_COLOR, 0);
+      FloatBuffer vertexBuffer =
+          ByteBuffer.allocateDirect(points.length * 4)
+              .order(ByteOrder.nativeOrder())
+              .asFloatBuffer()
+              .put(points);
+      vertexBuffer.position(0);
+      GLES20.glEnableVertexAttribArray(positionHandle);
+      GLES20.glVertexAttribPointer(positionHandle, 2, GLES20.GL_FLOAT, false, 0, vertexBuffer);
+      GLES20.glDrawArrays(GLES20.GL_POINTS, 0, landmarks.size());
+
   }
 
   /**
@@ -100,6 +121,8 @@ public class PoseTrackingResultGlRenderer implements ResultGlRenderer<PoseTracki
   public void release() {
     GLES20.glDeleteProgram(program);
   }
+
+
   /**
    * Not needed anymore, to be cleaned
    * */