From 9150dd57ccc01d1dfb6b903f897e1f64ad5b3991 Mon Sep 17 00:00:00 2001 From: Mautisim Munir Date: Mon, 26 Sep 2022 18:18:45 +0500 Subject: [PATCH] fixed landmarks values from graph --- .../apps/posetrackingsolutiongpu/BUILD | 1 + .../posetrackingsolutiongpu/MainActivity.java | 40 ++++++------------- .../pose_tracking_gpu_image.pbtxt | 5 ++- .../mediapipe/solutions/posetracking/BUILD | 1 + .../solutions/posetracking/PoseTracking.java | 27 +++++++++++-- .../posetracking/PoseTrackingResult.java | 19 ++++++--- 6 files changed, 55 insertions(+), 38 deletions(-) diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackingsolutiongpu/BUILD b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackingsolutiongpu/BUILD index c57e2a89e..0f513ddb8 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackingsolutiongpu/BUILD +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackingsolutiongpu/BUILD @@ -22,6 +22,7 @@ android_library( resource_files = glob(["res/**"]), visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework/formats:landmark_java_proto_lite", "//mediapipe/framework/formats:detection_java_proto_lite", "//mediapipe/framework/formats:location_data_java_proto_lite", "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper", diff --git a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackingsolutiongpu/MainActivity.java b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackingsolutiongpu/MainActivity.java index 32cb15bc6..64732be4a 100644 --- a/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackingsolutiongpu/MainActivity.java +++ b/mediapipe/examples/android/src/java/com/google/mediapipe/apps/posetrackingsolutiongpu/MainActivity.java @@ -30,6 +30,7 @@ import androidx.activity.result.ActivityResultLauncher; import androidx.activity.result.contract.ActivityResultContracts; import androidx.exifinterface.media.ExifInterface; // ContentResolver dependency +import com.google.mediapipe.formats.proto.LandmarkProto; import com.google.mediapipe.solutioncore.CameraInput; import com.google.mediapipe.solutioncore.SolutionGlSurfaceView; import com.google.mediapipe.solutioncore.VideoInput; @@ -201,7 +202,7 @@ public class MainActivity extends AppCompatActivity { // Connects MediaPipe Face Detection solution to the user-defined PoseTrackingResultImageView. poseTracking.setResultListener( poseTrackingResult -> { - logNoseTipKeypoint(poseTrackingResult, /*faceIndex=*/ 0, /*showPixelValues=*/ true); + logExampleKeypoint(poseTrackingResult, /*faceIndex=*/ 0, /*showPixelValues=*/ true); // imageView.setPoseTrackingResult(poseTrackingResult); // runOnUiThread(() -> imageView.update()); }); @@ -271,7 +272,7 @@ public class MainActivity extends AppCompatActivity { this, PoseTrackingOptions.builder() .setStaticImageMode(false) - .setLandmarkVisibility(false) + .setLandmarkVisibility(true) .setModelSelection(0) .build()); poseTracking.setErrorListener( @@ -293,7 +294,7 @@ public class MainActivity extends AppCompatActivity { glSurfaceView.setRenderInputImage(true); poseTracking.setResultListener( poseTrackingResult -> { - logNoseTipKeypoint(poseTrackingResult, /*faceIndex=*/ 0, /*showPixelValues=*/ false); + logExampleKeypoint(poseTrackingResult, /*faceIndex=*/ 0, /*showPixelValues=*/ false); glSurfaceView.setRenderData(poseTrackingResult); glSurfaceView.requestRender(); }); @@ -339,33 +340,18 @@ public class MainActivity extends AppCompatActivity { } } - private void logNoseTipKeypoint( + private void logExampleKeypoint( PoseTrackingResult result, int faceIndex, boolean showPixelValues) { if (result.multiPoseTrackings().isEmpty()) { return; } -// RelativeKeypoint noseTip = -// result -// .multiPoseTrackings() -// .get(faceIndex) -// .getLocationData() -// .getRelativeKeypoints(FaceKeypoint.NOSE_TIP); - // For Bitmaps, show the pixel values. For texture inputs, show the normalized coordinates. - if (showPixelValues) { - int width = result.inputBitmap().getWidth(); - int height = result.inputBitmap().getHeight(); -// Log.i( -// TAG, -// String.format( -// "MediaPipe Face Detection nose tip coordinates (pixel values): x=%f, y=%f", -// noseTip.getX() * width, noseTip.getY() * height)); - } else { -// Log.i( -// TAG, -// String.format( -// "MediaPipe Face Detection nose tip normalized coordinates (value range: [0, 1]):" -// + " x=%f, y=%f", -// noseTip.getX(), noseTip.getY())); - } + LandmarkProto.Landmark exampleLandmark = result.multiPoseLandmarks().get(0); + Log.d( + TAG, + String.format( + "Pose Landmark Landmark at index 0: x=%f, y=%f, z=%f", + exampleLandmark.getX() , exampleLandmark.getY(),exampleLandmark.getZ() )); + + } } diff --git a/mediapipe/graphs/pose_tracking/pose_tracking_gpu_image.pbtxt b/mediapipe/graphs/pose_tracking/pose_tracking_gpu_image.pbtxt index fa3ed97f5..9bd7c09be 100644 --- a/mediapipe/graphs/pose_tracking/pose_tracking_gpu_image.pbtxt +++ b/mediapipe/graphs/pose_tracking/pose_tracking_gpu_image.pbtxt @@ -5,12 +5,13 @@ input_stream: "IMAGE:input_video" # Output image with rendered results. (GpuBuffer) output_stream: "output_video" -# Pose landmarks. (NormalizedLandmarkList) -#output_stream: "pose_landmarks" output_stream: "IMAGE:throttled_input_video" output_stream: "DETECTION:pose_detection" output_stream: "output_video" +# Pose landmarks. (NormalizedLandmarkList) +output_stream: "pose_landmarks" + # Generates side packet to enable segmentation. node { diff --git a/mediapipe/java/com/google/mediapipe/solutions/posetracking/BUILD b/mediapipe/java/com/google/mediapipe/solutions/posetracking/BUILD index eb7199596..22908de58 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/posetracking/BUILD +++ b/mediapipe/java/com/google/mediapipe/solutions/posetracking/BUILD @@ -39,6 +39,7 @@ android_library( visibility = ["//visibility:public"], deps = [ "//mediapipe/framework/formats:detection_java_proto_lite", + "//mediapipe/framework/formats:landmark_java_proto_lite", "//mediapipe/java/com/google/mediapipe/framework:android_framework", "//mediapipe/java/com/google/mediapipe/solutioncore:camera_input", "//mediapipe/java/com/google/mediapipe/solutioncore:solution_base", diff --git a/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTracking.java b/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTracking.java index 9ec155a79..8f2a03b0b 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTracking.java +++ b/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTracking.java @@ -19,6 +19,7 @@ import android.util.Log; import com.google.common.collect.ImmutableList; import com.google.mediapipe.formats.proto.DetectionProto; +import com.google.mediapipe.formats.proto.LandmarkProto; import com.google.mediapipe.framework.MediaPipeException; import com.google.mediapipe.framework.Packet; import com.google.mediapipe.framework.PacketGetter; @@ -31,6 +32,7 @@ import com.google.mediapipe.formats.proto.DetectionProto.Detection; import com.google.protobuf.InvalidProtocolBufferException; import java.util.HashMap; +import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -48,10 +50,11 @@ public class PoseTracking extends ImageSolutionBase { private static final String FULL_RANGE_GRAPH_NAME = "face_detection_full_range_image.binarypb"; private static final String IMAGE_INPUT_STREAM = "input_video"; private static final ImmutableList OUTPUT_STREAMS = - ImmutableList.of("pose_detection", "throttled_input_video","output_video"); + ImmutableList.of("pose_detection", "throttled_input_video","output_video","pose_landmarks"); private static final int DETECTIONS_INDEX = 0; private static final int INPUT_IMAGE_INDEX = 1; private static final int OUTPUT_IMAGE_INDEX = 2; + private static final int LANDMARKS_INDEX = 3; private final OutputHandler outputHandler; /** @@ -71,24 +74,40 @@ public class PoseTracking extends ImageSolutionBase { try { byte[] bytes = PacketGetter.getProtoBytes(packet); Detection det = Detection.parseFrom(bytes); - poseTrackingResultBuilder.setMultiPoseTrackings( + poseTrackingResultBuilder.setMultiPoseDetections( ImmutableList.of(det)); + // Detection det = PacketGetter.getProto(packet, Detection.getDefaultInstance()); Log.v(TAG,"Packet not empty"); }catch (InvalidProtocolBufferException e){ Log.e(TAG,e.getMessage()); - poseTrackingResultBuilder.setMultiPoseTrackings( + poseTrackingResultBuilder.setMultiPoseDetections( ImmutableList.of()); } }else { - poseTrackingResultBuilder.setMultiPoseTrackings( + poseTrackingResultBuilder.setMultiPoseDetections( getProtoVector(packets.get(DETECTIONS_INDEX), Detection.parser())); } + + Packet landmarksPacket = packets.get(LANDMARKS_INDEX); + if (landmarksPacket.isEmpty()){ + poseTrackingResultBuilder.setMultiPoseLandmarks(ImmutableList.of()); + }else { + byte[] landmarkBytes = PacketGetter.getProtoBytes(landmarksPacket); + LandmarkProto.LandmarkList landmarks = LandmarkProto.LandmarkList.parseFrom(landmarkBytes); +// List landmarks = getProtoVector(landmarksPacket, LandmarkProto.Landmark.parser()); + poseTrackingResultBuilder.setMultiPoseLandmarks(landmarks.getLandmarkList()); + } + + + } catch (MediaPipeException e) { reportError("Error occurs while getting MediaPipe pose tracking results.", e); + } catch (InvalidProtocolBufferException e) { + e.printStackTrace(); } int imageIndex = options.landmarkVisibility() ? OUTPUT_IMAGE_INDEX : INPUT_IMAGE_INDEX; diff --git a/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java b/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java index b82989f73..06035e94b 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java +++ b/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java @@ -17,6 +17,7 @@ package com.google.mediapipe.solutions.posetracking; import android.graphics.Bitmap; import com.google.auto.value.AutoBuilder; import com.google.common.collect.ImmutableList; +import com.google.mediapipe.formats.proto.LandmarkProto; import com.google.mediapipe.framework.Packet; import com.google.mediapipe.framework.TextureFrame; import com.google.mediapipe.solutioncore.ImageSolutionResult; @@ -29,11 +30,13 @@ import java.util.List; * the corresponding input image. */ public class PoseTrackingResult extends ImageSolutionResult { - private final ImmutableList multiPoseTrackings; + private final ImmutableList multiPoseDetections; + private final ImmutableList multiPoseLandmarks; PoseTrackingResult( - ImmutableList multiPoseTrackings, Packet imagePacket, long timestamp) { - this.multiPoseTrackings = multiPoseTrackings; + ImmutableList multiPoseDetections,ImmutableList multiPoseLandmarks, Packet imagePacket, long timestamp) { + this.multiPoseDetections = multiPoseDetections; + this.multiPoseLandmarks = multiPoseLandmarks; this.timestamp = timestamp; this.imagePacket = imagePacket; } @@ -44,7 +47,12 @@ public class PoseTrackingResult extends ImageSolutionResult { // to [0.0, 1.0] by the image height). Each keypoint is composed of x and y, which are normalized // to [0.0, 1.0] by the image width and height respectively. public ImmutableList multiPoseTrackings() { - return multiPoseTrackings; + return multiPoseDetections; + } + + + public ImmutableList multiPoseLandmarks() { + return multiPoseLandmarks; } public static Builder builder() { @@ -54,7 +62,8 @@ public class PoseTrackingResult extends ImageSolutionResult { /** Builder for {@link PoseTrackingResult}. */ @AutoBuilder public abstract static class Builder { - abstract Builder setMultiPoseTrackings(List value); + abstract Builder setMultiPoseDetections(List value); + abstract Builder setMultiPoseLandmarks(List value); abstract Builder setTimestamp(long value);