fixed landmarks values from graph

mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackingsolutiongpu/BUILD

@@ -22,6 +22,7 @@ android_library(
     resource_files = glob(["res/**"]),
     visibility = ["//visibility:public"],
     deps = [
+        "//mediapipe/framework/formats:landmark_java_proto_lite",
         "//mediapipe/framework/formats:detection_java_proto_lite",
         "//mediapipe/framework/formats:location_data_java_proto_lite",
         "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",

mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackingsolutiongpu/MainActivity.java

@@ -30,6 +30,7 @@ import androidx.activity.result.ActivityResultLauncher;
 import androidx.activity.result.contract.ActivityResultContracts;
 import androidx.exifinterface.media.ExifInterface;
 // ContentResolver dependency
+import com.google.mediapipe.formats.proto.LandmarkProto;
 import com.google.mediapipe.solutioncore.CameraInput;
 import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
 import com.google.mediapipe.solutioncore.VideoInput;
@@ -201,7 +202,7 @@ public class MainActivity extends AppCompatActivity {
     // Connects MediaPipe Face Detection solution to the user-defined PoseTrackingResultImageView.
     poseTracking.setResultListener(
             poseTrackingResult -> {
-              logNoseTipKeypoint(poseTrackingResult, /*faceIndex=*/ 0, /*showPixelValues=*/ true);
+              logExampleKeypoint(poseTrackingResult, /*faceIndex=*/ 0, /*showPixelValues=*/ true);
 //              imageView.setPoseTrackingResult(poseTrackingResult);
 //              runOnUiThread(() -> imageView.update());
             });
@@ -271,7 +272,7 @@ public class MainActivity extends AppCompatActivity {
                     this,
                     PoseTrackingOptions.builder()
                             .setStaticImageMode(false)
-                            .setLandmarkVisibility(false)
+                            .setLandmarkVisibility(true)
                             .setModelSelection(0)
                             .build());
     poseTracking.setErrorListener(
@@ -293,7 +294,7 @@ public class MainActivity extends AppCompatActivity {
     glSurfaceView.setRenderInputImage(true);
     poseTracking.setResultListener(
             poseTrackingResult -> {
-              logNoseTipKeypoint(poseTrackingResult, /*faceIndex=*/ 0, /*showPixelValues=*/ false);
+              logExampleKeypoint(poseTrackingResult, /*faceIndex=*/ 0, /*showPixelValues=*/ false);
               glSurfaceView.setRenderData(poseTrackingResult);
               glSurfaceView.requestRender();
             });
@@ -339,33 +340,18 @@ public class MainActivity extends AppCompatActivity {
     }
   }
 
-  private void logNoseTipKeypoint(
+  private void logExampleKeypoint(
           PoseTrackingResult result, int faceIndex, boolean showPixelValues) {
     if (result.multiPoseTrackings().isEmpty()) {
       return;
     }
-//    RelativeKeypoint noseTip =
+    LandmarkProto.Landmark exampleLandmark = result.multiPoseLandmarks().get(0);
-//            result
+      Log.d(
-//                    .multiPoseTrackings()
+              TAG,
-//                    .get(faceIndex)
+              String.format(
-//                    .getLocationData()
+                      "Pose Landmark Landmark at index 0: x=%f, y=%f, z=%f",
-//                    .getRelativeKeypoints(FaceKeypoint.NOSE_TIP);
+                      exampleLandmark.getX() , exampleLandmark.getY(),exampleLandmark.getZ() ));
-    // For Bitmaps, show the pixel values. For texture inputs, show the normalized coordinates.
+
-    if (showPixelValues) {
+
-      int width = result.inputBitmap().getWidth();
-      int height = result.inputBitmap().getHeight();
-//      Log.i(
-//              TAG,
-//              String.format(
-//                      "MediaPipe Face Detection nose tip coordinates (pixel values): x=%f, y=%f",
-//                      noseTip.getX() * width, noseTip.getY() * height));
-    } else {
-//      Log.i(
-//              TAG,
-//              String.format(
-//                      "MediaPipe Face Detection nose tip normalized coordinates (value range: [0, 1]):"
-//                              + " x=%f, y=%f",
-//                      noseTip.getX(), noseTip.getY()));
-    }
   }
 }

mediapipe/graphs/pose_tracking/pose_tracking_gpu_image.pbtxt

@@ -5,12 +5,13 @@ input_stream: "IMAGE:input_video"
 
 # Output image with rendered results. (GpuBuffer)
 output_stream: "output_video"
-# Pose landmarks. (NormalizedLandmarkList)
-#output_stream: "pose_landmarks"
 
 output_stream: "IMAGE:throttled_input_video"
 output_stream: "DETECTION:pose_detection"
 output_stream: "output_video"
+# Pose landmarks. (NormalizedLandmarkList)
+output_stream: "pose_landmarks"
+
 
 # Generates side packet to enable segmentation.
 node {

mediapipe/java/com/google/mediapipe/solutions/posetracking/BUILD

@@ -39,6 +39,7 @@ android_library(
     visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework/formats:detection_java_proto_lite",
+        "//mediapipe/framework/formats:landmark_java_proto_lite",
         "//mediapipe/java/com/google/mediapipe/framework:android_framework",
         "//mediapipe/java/com/google/mediapipe/solutioncore:camera_input",
         "//mediapipe/java/com/google/mediapipe/solutioncore:solution_base",

mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTracking.java

@@ -19,6 +19,7 @@ import android.util.Log;
 
 import com.google.common.collect.ImmutableList;
 import com.google.mediapipe.formats.proto.DetectionProto;
+import com.google.mediapipe.formats.proto.LandmarkProto;
 import com.google.mediapipe.framework.MediaPipeException;
 import com.google.mediapipe.framework.Packet;
 import com.google.mediapipe.framework.PacketGetter;
@@ -31,6 +32,7 @@ import com.google.mediapipe.formats.proto.DetectionProto.Detection;
 import com.google.protobuf.InvalidProtocolBufferException;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import javax.annotation.Nullable;
 
@@ -48,10 +50,11 @@ public class PoseTracking extends ImageSolutionBase {
   private static final String FULL_RANGE_GRAPH_NAME = "face_detection_full_range_image.binarypb";
   private static final String IMAGE_INPUT_STREAM = "input_video";
   private static final ImmutableList<String> OUTPUT_STREAMS =
-      ImmutableList.of("pose_detection", "throttled_input_video","output_video");
+      ImmutableList.of("pose_detection", "throttled_input_video","output_video","pose_landmarks");
   private static final int DETECTIONS_INDEX = 0;
   private static final int INPUT_IMAGE_INDEX = 1;
   private static final int OUTPUT_IMAGE_INDEX = 2;
+  private static final int LANDMARKS_INDEX = 3;
   private final OutputHandler<PoseTrackingResult> outputHandler;
 
   /**
@@ -71,24 +74,40 @@ public class PoseTracking extends ImageSolutionBase {
               try {
                 byte[] bytes = PacketGetter.getProtoBytes(packet);
                 Detection det = Detection.parseFrom(bytes);
-                poseTrackingResultBuilder.setMultiPoseTrackings(
+                poseTrackingResultBuilder.setMultiPoseDetections(
                         ImmutableList.<Detection>of(det));
+
 //                Detection det = PacketGetter.getProto(packet, Detection.getDefaultInstance());
                 Log.v(TAG,"Packet not empty");
 
               }catch (InvalidProtocolBufferException e){
                 Log.e(TAG,e.getMessage());
-                poseTrackingResultBuilder.setMultiPoseTrackings(
+                poseTrackingResultBuilder.setMultiPoseDetections(
                         ImmutableList.<Detection>of());
 
               }
 
             }else {
-              poseTrackingResultBuilder.setMultiPoseTrackings(
+              poseTrackingResultBuilder.setMultiPoseDetections(
                       getProtoVector(packets.get(DETECTIONS_INDEX), Detection.parser()));
             }
+
+            Packet landmarksPacket = packets.get(LANDMARKS_INDEX);
+            if (landmarksPacket.isEmpty()){
+              poseTrackingResultBuilder.setMultiPoseLandmarks(ImmutableList.of());
+            }else {
+              byte[] landmarkBytes = PacketGetter.getProtoBytes(landmarksPacket);
+              LandmarkProto.LandmarkList landmarks = LandmarkProto.LandmarkList.parseFrom(landmarkBytes);
+//              List<LandmarkProto.Landmark> landmarks = getProtoVector(landmarksPacket, LandmarkProto.Landmark.parser());
+              poseTrackingResultBuilder.setMultiPoseLandmarks(landmarks.getLandmarkList());
+            }
+
+
+
           } catch (MediaPipeException e) {
             reportError("Error occurs while getting MediaPipe pose tracking results.", e);
+          } catch (InvalidProtocolBufferException e) {
+            e.printStackTrace();
           }
 
           int imageIndex = options.landmarkVisibility() ? OUTPUT_IMAGE_INDEX : INPUT_IMAGE_INDEX;

mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java

@@ -17,6 +17,7 @@ package com.google.mediapipe.solutions.posetracking;
 import android.graphics.Bitmap;
 import com.google.auto.value.AutoBuilder;
 import com.google.common.collect.ImmutableList;
+import com.google.mediapipe.formats.proto.LandmarkProto;
 import com.google.mediapipe.framework.Packet;
 import com.google.mediapipe.framework.TextureFrame;
 import com.google.mediapipe.solutioncore.ImageSolutionResult;
@@ -29,11 +30,13 @@ import java.util.List;
  * the corresponding input image.
  */
 public class PoseTrackingResult extends ImageSolutionResult {
-  private final ImmutableList<Detection> multiPoseTrackings;
+  private final ImmutableList<Detection> multiPoseDetections;
+  private final ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks;
 
   PoseTrackingResult(
-      ImmutableList<Detection> multiPoseTrackings, Packet imagePacket, long timestamp) {
+          ImmutableList<Detection> multiPoseDetections,ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks, Packet imagePacket, long timestamp) {
-    this.multiPoseTrackings = multiPoseTrackings;
+    this.multiPoseDetections = multiPoseDetections;
+    this.multiPoseLandmarks = multiPoseLandmarks;
     this.timestamp = timestamp;
     this.imagePacket = imagePacket;
   }
@@ -44,7 +47,12 @@ public class PoseTrackingResult extends ImageSolutionResult {
   // to [0.0, 1.0] by the image height). Each keypoint is composed of x and y, which are normalized
   // to [0.0, 1.0] by the image width and height respectively.
   public ImmutableList<Detection> multiPoseTrackings() {
-    return multiPoseTrackings;
+    return multiPoseDetections;
+  }
+
+
+  public ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks() {
+    return multiPoseLandmarks;
   }
 
   public static Builder builder() {
@@ -54,7 +62,8 @@ public class PoseTrackingResult extends ImageSolutionResult {
   /** Builder for {@link PoseTrackingResult}. */
   @AutoBuilder
   public abstract static class Builder {
-    abstract Builder setMultiPoseTrackings(List<Detection> value);
+    abstract Builder setMultiPoseDetections(List<Detection> value);
+    abstract Builder setMultiPoseLandmarks(List<LandmarkProto.Landmark> value);
 
     abstract Builder setTimestamp(long value);