From fa5da361d774027efad0497b54fa1fb22fd1f67a Mon Sep 17 00:00:00 2001 From: Mautisim Munir Date: Fri, 28 Oct 2022 14:01:10 +0500 Subject: [PATCH 1/4] changed timestamp to nanoseconds, added presence --- .../solutions/lindera/ComputerVisionPlugin.java | 2 +- .../google/mediapipe/solutions/lindera/Lindera.java | 12 +++++++----- .../solutions/lindera/XYZPointWithConfidence.java | 1 + 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/mediapipe/java/com/google/mediapipe/solutions/lindera/ComputerVisionPlugin.java b/mediapipe/java/com/google/mediapipe/solutions/lindera/ComputerVisionPlugin.java index db577f5d9..2d1dc0242 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/lindera/ComputerVisionPlugin.java +++ b/mediapipe/java/com/google/mediapipe/solutions/lindera/ComputerVisionPlugin.java @@ -1,5 +1,5 @@ package com.google.mediapipe.solutions.lindera; public interface ComputerVisionPlugin { - void bodyJoints(int timestamp, BodyJoints bodyJoints); + void bodyJoints(long timestamp, BodyJoints bodyJoints); } diff --git a/mediapipe/java/com/google/mediapipe/solutions/lindera/Lindera.java b/mediapipe/java/com/google/mediapipe/solutions/lindera/Lindera.java index 52f97d17c..0f8e749bb 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/lindera/Lindera.java +++ b/mediapipe/java/com/google/mediapipe/solutions/lindera/Lindera.java @@ -25,8 +25,10 @@ public class Lindera { public FpsHelper fpsHelper = new FpsHelper(); private PoseTracking poseTracking; - // TODO: Verify that this is the timestamp used in Actual Plugin - private int timeStamp = 0; + + private long getTimeStamp(){ + return System.nanoTime(); + } private CameraRotation cameraRotation = CameraRotation.AUTOMATIC; // Live camera demo UI and camera components. @@ -85,14 +87,13 @@ public class Lindera { glSurfaceView.setRenderData(poseTrackingResult); glSurfaceView.requestRender(); ImmutableList landmarks = poseTrackingResult.multiPoseLandmarks(); - timeStamp+=1; if (landmarks.isEmpty()) return; BodyJoints bodyJoints = new BodyJoints(); landmarksToBodyJoints(landmarks,bodyJoints); - plugin.bodyJoints(timeStamp, bodyJoints); + plugin.bodyJoints(getTimeStamp(), bodyJoints); }); } @@ -180,7 +181,6 @@ public class Lindera { if (poseTracking != null) { poseTracking.close(); } - timeStamp = 0; } private void landmarkToXYZPointWithConfidence(LandmarkProto.Landmark landmark,XYZPointWithConfidence bodyJoint){ @@ -188,6 +188,8 @@ public class Lindera { bodyJoint.y = landmark.getY(); bodyJoint.z = landmark.getZ(); bodyJoint.confidence = landmark.getVisibility(); + bodyJoint.presence = landmark.getPresence(); + } private void landmarksToBodyJoints(ImmutableList landmarks , BodyJoints bodyJoints){ diff --git a/mediapipe/java/com/google/mediapipe/solutions/lindera/XYZPointWithConfidence.java b/mediapipe/java/com/google/mediapipe/solutions/lindera/XYZPointWithConfidence.java index 576f8f489..ac368d60c 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/lindera/XYZPointWithConfidence.java +++ b/mediapipe/java/com/google/mediapipe/solutions/lindera/XYZPointWithConfidence.java @@ -5,6 +5,7 @@ public class XYZPointWithConfidence { public float y = 0; public float z = 0; public float confidence = 0; + public float presence = 0; From e2884bc21f39e1d76d46d3f9b32b0287fcc31317 Mon Sep 17 00:00:00 2001 From: Mautisim Munir Date: Fri, 28 Oct 2022 14:01:39 +0500 Subject: [PATCH 2/4] added interpolation implementation without visualization --- .../ComputerVisionPluginImpl.java | 70 ++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java index 88d1e0f45..cb5af4132 100644 --- a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java +++ b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java @@ -1,5 +1,7 @@ package com.google.mediapipe.examples.posetracking_lindera; +import static java.lang.Math.min; + import com.google.mediapipe.solutions.lindera.BodyJoints; import com.google.mediapipe.solutions.lindera.ComputerVisionPlugin; import com.google.mediapipe.solutions.lindera.XYZPointWithConfidence; @@ -9,8 +11,10 @@ import org.json.JSONException; import org.json.JSONObject; import java.lang.reflect.Field; +import java.util.HashMap; import java.util.LinkedList; import java.util.Locale; +import java.util.Map; public class ComputerVisionPluginImpl implements ComputerVisionPlugin { @@ -84,6 +88,7 @@ public class ComputerVisionPluginImpl implements ComputerVisionPlugin { } } + bodyJointsString = bodyJointsString.concat(interpolateJoints(bodyJoints)); // remove the last equal sign bodyJointsString = bodyJointsString.substring(0,bodyJointsString.length()-1); @@ -101,9 +106,72 @@ public class ComputerVisionPluginImpl implements ComputerVisionPlugin { } + String interpolateJoints(BodyJoints bodyJoints){ + + Map pts = new HashMap<>(); + pts.put("PE", getPelvis(bodyJoints)); + pts.put("NN",bodyJoints.nose); + // Assuming Thorax is 1/3 of distance between shoulders and pelvis + XYZPointWithConfidence thorax = getSpinePoint(bodyJoints,1/3f); + // Assuming spine/middle back is 2/3 of distance between shoulders and pelvis + XYZPointWithConfidence spine = getSpinePoint(bodyJoints,2/3f); + pts.put("TH",thorax); + pts.put("SP",spine); + pts.put("HT",getHeadTop(bodyJoints)); + final String[] bodyJointsString = {""}; + pts.forEach((key,data)->{ + bodyJointsString[0] = bodyJointsString[0].concat(String.format(key+":%f,%f,%f=",data.x,data.y,data.z)); + + }); + + return bodyJointsString[0]; + + + + + + } + XYZPointWithConfidence getPelvis(BodyJoints bodyJoints){ + return getMiddleJoint(bodyJoints.leftHip,bodyJoints.rightHip); + + } + + XYZPointWithConfidence getJointBetweenPoints(XYZPointWithConfidence pt1,XYZPointWithConfidence pt2,float distance){ + XYZPointWithConfidence midpt = new XYZPointWithConfidence(); + midpt.x = pt1.x + (pt2.x-pt1.x)*distance; + midpt.y = pt1.y + (pt2.y-pt1.y)*distance; + midpt.z = pt1.z + (pt2.z-pt1.z)*distance; + + + midpt.presence = min(pt1.presence,pt2.presence); + midpt.confidence = min(pt1.confidence,pt2.confidence); + return midpt; + } + XYZPointWithConfidence getMiddleJoint(XYZPointWithConfidence pt1,XYZPointWithConfidence pt2) { + XYZPointWithConfidence midpt = new XYZPointWithConfidence(); + midpt.x = (pt1.x + pt2.x)/2; + midpt.y = (pt1.y + pt2.y)/2; + midpt.z = (pt1.z + pt2.z)/2; + + midpt.presence = min(pt1.presence,pt2.presence); + midpt.confidence = min(pt1.confidence,pt2.confidence); + return midpt; + } + XYZPointWithConfidence getSpinePoint(BodyJoints bodyJoints, float distanceFromShoulders){ + XYZPointWithConfidence midShoulder = getMiddleJoint(bodyJoints.leftShoulder, bodyJoints.rightShoulder); + XYZPointWithConfidence pelvis = getPelvis(bodyJoints); + return getJointBetweenPoints(midShoulder,pelvis,distanceFromShoulders); + } + + XYZPointWithConfidence getHeadTop(BodyJoints bodyJoints){ + XYZPointWithConfidence middleEye = getMiddleJoint(bodyJoints.leftEye,bodyJoints.rightEye); + return getJointBetweenPoints(middleEye,bodyJoints.nose,2); + } + + @Override - public void bodyJoints(int timestamp, BodyJoints bodyJoints) { + public void bodyJoints(long timestamp, BodyJoints bodyJoints) { if (isLogging){ this.bodyJointsEventList.add(new BodyJointsEvent(timestamp,bodyJoints)); From be07542ef787bf4d7cbed1f2b31d84a00f69435e Mon Sep 17 00:00:00 2001 From: Mautisim Munir Date: Fri, 28 Oct 2022 19:51:05 +0500 Subject: [PATCH 3/4] added interpolation with visualization --- .../posetracking-lindera/src/main/BUILD | 1 + .../ComputerVisionPluginImpl.java | 58 ----- .../posetracking_lindera/MainActivity.java | 6 +- .../src/main/res/layout/activity_main.xml | 49 ++-- .../SolutionGlSurfaceViewRenderer.java | 3 + .../solutions/lindera/BodyJoints.java | 20 +- .../mediapipe/solutions/lindera/Lindera.java | 19 +- .../posetracking/PoseTrackingResult.java | 213 ++++++++++++------ .../PoseTrackingResultGlRenderer.java | 37 ++- 9 files changed, 246 insertions(+), 160 deletions(-) diff --git a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/BUILD b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/BUILD index 3bd9f445f..748ab157d 100644 --- a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/BUILD +++ b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/BUILD @@ -20,6 +20,7 @@ android_binary( name = "posetracking-lindera", srcs = glob(["**/*.java"]), custom_package = "com.google.mediapipe.examples.posetracking_lindera", + dex_shards = 10, manifest = "AndroidManifest.xml", manifest_values = { "applicationId": "com.google.mediapipe.examples.posetracking_lindera", diff --git a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java index cb5af4132..6ae829d53 100644 --- a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java +++ b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java @@ -88,7 +88,6 @@ public class ComputerVisionPluginImpl implements ComputerVisionPlugin { } } - bodyJointsString = bodyJointsString.concat(interpolateJoints(bodyJoints)); // remove the last equal sign bodyJointsString = bodyJointsString.substring(0,bodyJointsString.length()-1); @@ -106,68 +105,11 @@ public class ComputerVisionPluginImpl implements ComputerVisionPlugin { } - String interpolateJoints(BodyJoints bodyJoints){ - - Map pts = new HashMap<>(); - pts.put("PE", getPelvis(bodyJoints)); - pts.put("NN",bodyJoints.nose); - // Assuming Thorax is 1/3 of distance between shoulders and pelvis - XYZPointWithConfidence thorax = getSpinePoint(bodyJoints,1/3f); - // Assuming spine/middle back is 2/3 of distance between shoulders and pelvis - XYZPointWithConfidence spine = getSpinePoint(bodyJoints,2/3f); - pts.put("TH",thorax); - pts.put("SP",spine); - pts.put("HT",getHeadTop(bodyJoints)); - final String[] bodyJointsString = {""}; - pts.forEach((key,data)->{ - bodyJointsString[0] = bodyJointsString[0].concat(String.format(key+":%f,%f,%f=",data.x,data.y,data.z)); - - }); - - return bodyJointsString[0]; - } - XYZPointWithConfidence getPelvis(BodyJoints bodyJoints){ - return getMiddleJoint(bodyJoints.leftHip,bodyJoints.rightHip); - - } - - XYZPointWithConfidence getJointBetweenPoints(XYZPointWithConfidence pt1,XYZPointWithConfidence pt2,float distance){ - XYZPointWithConfidence midpt = new XYZPointWithConfidence(); - midpt.x = pt1.x + (pt2.x-pt1.x)*distance; - midpt.y = pt1.y + (pt2.y-pt1.y)*distance; - midpt.z = pt1.z + (pt2.z-pt1.z)*distance; - - - midpt.presence = min(pt1.presence,pt2.presence); - midpt.confidence = min(pt1.confidence,pt2.confidence); - return midpt; - } - XYZPointWithConfidence getMiddleJoint(XYZPointWithConfidence pt1,XYZPointWithConfidence pt2) { - XYZPointWithConfidence midpt = new XYZPointWithConfidence(); - midpt.x = (pt1.x + pt2.x)/2; - midpt.y = (pt1.y + pt2.y)/2; - midpt.z = (pt1.z + pt2.z)/2; - - midpt.presence = min(pt1.presence,pt2.presence); - midpt.confidence = min(pt1.confidence,pt2.confidence); - return midpt; - } - XYZPointWithConfidence getSpinePoint(BodyJoints bodyJoints, float distanceFromShoulders){ - XYZPointWithConfidence midShoulder = getMiddleJoint(bodyJoints.leftShoulder, bodyJoints.rightShoulder); - XYZPointWithConfidence pelvis = getPelvis(bodyJoints); - return getJointBetweenPoints(midShoulder,pelvis,distanceFromShoulders); - } - - XYZPointWithConfidence getHeadTop(BodyJoints bodyJoints){ - XYZPointWithConfidence middleEye = getMiddleJoint(bodyJoints.leftEye,bodyJoints.rightEye); - return getJointBetweenPoints(middleEye,bodyJoints.nose,2); - } - @Override diff --git a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java index a39a50013..3cd14ab9a 100644 --- a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java +++ b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java @@ -83,7 +83,7 @@ public class MainActivity extends AppCompatActivity { findViewById(R.id.button_set_model).setVisibility(View.GONE); findViewById(R.id.button_toggle_landmarks).setVisibility(View.GONE); - findViewById(R.id.button_start_capture).setVisibility(View.GONE); + findViewById(R.id.button_capture_logging).setVisibility(View.GONE); setupLiveDemoUiComponents(); plugin = new ComputerVisionPluginImpl(); @@ -116,7 +116,7 @@ public class MainActivity extends AppCompatActivity { Button startDetectionButton = findViewById(R.id.button_start_detection); Button toggleLandmarks = findViewById(R.id.button_toggle_landmarks); Button modelComplexity = findViewById(R.id.button_set_model); - Button startCapture = findViewById(R.id.button_start_capture); + Button startCapture = findViewById(R.id.button_capture_logging); FrameLayout frameLayout = findViewById(R.id.preview_display_layout); startDetectionButton.setOnClickListener( @@ -129,7 +129,7 @@ public class MainActivity extends AppCompatActivity { startDetectionButton.setVisibility(View.GONE); findViewById(R.id.button_set_model).setVisibility(View.VISIBLE); findViewById(R.id.button_toggle_landmarks).setVisibility(View.VISIBLE); - findViewById(R.id.button_start_capture).setVisibility(View.VISIBLE); + findViewById(R.id.button_capture_logging).setVisibility(View.VISIBLE); updateLandmarkButtonText(); updateModelComplexityButtonText(); diff --git a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/res/layout/activity_main.xml b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/res/layout/activity_main.xml index 84418b5b3..ba95a5142 100644 --- a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/res/layout/activity_main.xml +++ b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/res/layout/activity_main.xml @@ -1,5 +1,6 @@ + android:gravity="center" + android:orientation="horizontal">