diff --git a/mediapipe/examples/android/solutions/posetracking-camera/src/main/java/com/google/mediapipe/examples/posetracking_camera/MainActivity.java b/mediapipe/examples/android/solutions/posetracking-camera/src/main/java/com/google/mediapipe/examples/posetracking_camera/MainActivity.java index 7ac195d30..ad7b846b4 100644 --- a/mediapipe/examples/android/solutions/posetracking-camera/src/main/java/com/google/mediapipe/examples/posetracking_camera/MainActivity.java +++ b/mediapipe/examples/android/solutions/posetracking-camera/src/main/java/com/google/mediapipe/examples/posetracking_camera/MainActivity.java @@ -15,144 +15,142 @@ package com.google.mediapipe.examples.posetracking_camera; import android.content.Intent; -import android.graphics.Bitmap; -import android.graphics.Matrix; import android.os.Bundle; -import android.os.Looper; -import android.provider.MediaStore; -import androidx.appcompat.app.AppCompatActivity; import android.util.Log; import android.view.Surface; import android.view.View; import android.widget.Button; import android.widget.FrameLayout; -import android.widget.ImageView; import androidx.activity.result.ActivityResultLauncher; -import androidx.activity.result.contract.ActivityResultContracts; -import androidx.exifinterface.media.ExifInterface; -// ContentResolver dependency +import androidx.appcompat.app.AppCompatActivity; + import com.google.mediapipe.formats.proto.LandmarkProto; import com.google.mediapipe.solutioncore.CameraInput; import com.google.mediapipe.solutioncore.SolutionGlSurfaceView; -import com.google.mediapipe.solutioncore.VideoInput; import com.google.mediapipe.solutions.posetracking.PoseTracking; import com.google.mediapipe.solutions.posetracking.PoseTrackingOptions; import com.google.mediapipe.solutions.posetracking.PoseTrackingResult; import com.google.mediapipe.solutions.posetracking.PoseTrackingResultGlRenderer; -import java.io.IOException; -import java.io.InputStream; -/** Main activity of MediaPipe Face Detection app. */ +/** + * Main activity of MediaPipe Face Detection app. + */ public class MainActivity extends AppCompatActivity { - private static final String TAG = "MainActivity"; - private static final int rotation = Surface.ROTATION_0; - private PoseTracking poseTracking; + private static final String TAG = "MainActivity"; + private static final int rotation = Surface.ROTATION_0; + private PoseTracking poseTracking; + private ActivityResultLauncher videoGetter; + // Live camera demo UI and camera components. + private CameraInput cameraInput; - private ActivityResultLauncher videoGetter; - // Live camera demo UI and camera components. - private CameraInput cameraInput; + private SolutionGlSurfaceView glSurfaceView; - private SolutionGlSurfaceView glSurfaceView; + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + disableRedundantUI(); - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - setContentView(R.layout.activity_main); - disableRedundantUI(); + setupLiveDemoUiComponents(); - setupLiveDemoUiComponents(); - - } - - - /** Sets up the UI components for the live demo with camera input. */ - private void setupLiveDemoUiComponents() { - - Button startCameraButton = findViewById(R.id.button_start_camera); - startCameraButton.setOnClickListener( - v -> { - setupStreamingModePipeline(); - startCameraButton.setVisibility(View.GONE); - }); - } - /**Disables unecesary UI buttons*/ - private void disableRedundantUI(){ - findViewById(R.id.button_load_picture).setVisibility(View.GONE); - findViewById(R.id.button_load_video).setVisibility(View.GONE); - - } - - /** Sets up core workflow for streaming mode. */ - private void setupStreamingModePipeline() { - // Initializes a new MediaPipe Face Detection solution instance in the streaming mode. - poseTracking = - new PoseTracking( - this, - PoseTrackingOptions.builder() - .setStaticImageMode(false) - .setLandmarkVisibility(true) - .setModelComplexity(0) - .setSmoothLandmarks(true) - .build()); - poseTracking.setErrorListener( - (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message)); - cameraInput = new CameraInput(this); - - cameraInput.setNewFrameListener(textureFrame -> poseTracking.send(textureFrame)); - - - // Initializes a new Gl surface view with a user-defined PoseTrackingResultGlRenderer. - glSurfaceView = - new SolutionGlSurfaceView<>( - this, poseTracking.getGlContext(), poseTracking.getGlMajorVersion()); - glSurfaceView.setSolutionResultRenderer(new PoseTrackingResultGlRenderer()); - glSurfaceView.setRenderInputImage(true); - poseTracking.setResultListener( - poseTrackingResult -> { - logExampleKeypoint(poseTrackingResult); - glSurfaceView.setRenderData(poseTrackingResult); - glSurfaceView.requestRender(); - }); - - // The runnable to start camera after the gl surface view is attached. - // For video input source, videoInput.start() will be called when the video uri is available. - glSurfaceView.post(this::startCamera); - - // Updates the preview layout. - FrameLayout frameLayout = findViewById(R.id.preview_display_layout); - frameLayout.removeAllViewsInLayout(); - frameLayout.addView(glSurfaceView); - glSurfaceView.setVisibility(View.VISIBLE); - frameLayout.requestLayout(); - } - - private void startCamera() { - cameraInput.getConverter(poseTracking.getGlContext()).setRotation(rotation); - cameraInput.start( - this, - poseTracking.getGlContext(), - CameraInput.CameraFacing.FRONT, - glSurfaceView.getWidth(), - glSurfaceView.getHeight()); - } - - - private void logExampleKeypoint( - PoseTrackingResult result) { - if (result.multiPoseLandmarks().isEmpty()) { - return; } - LandmarkProto.Landmark exampleLandmark = result.multiPoseLandmarks().get(PoseTrackingResult.NOSE); - Log.i( - TAG, - String.format( - "Pose Landmark Landmark of Nose: x=%f, y=%f, z=%f", - exampleLandmark.getX() , exampleLandmark.getY(),exampleLandmark.getZ() )); - } + /** + * Sets up the UI components for the live demo with camera input. + */ + private void setupLiveDemoUiComponents() { + + Button startCameraButton = findViewById(R.id.button_start_camera); + startCameraButton.setOnClickListener( + v -> { + setupStreamingModePipeline(); + startCameraButton.setVisibility(View.GONE); + }); + } + + /** + * Disables unecesary UI buttons + */ + private void disableRedundantUI() { + findViewById(R.id.button_load_picture).setVisibility(View.GONE); + findViewById(R.id.button_load_video).setVisibility(View.GONE); + + } + + /** + * Sets up core workflow for streaming mode. + */ + private void setupStreamingModePipeline() { + // Initializes a new MediaPipe Face Detection solution instance in the streaming mode. + poseTracking = + new PoseTracking( + this, + PoseTrackingOptions.builder() + .setStaticImageMode(false) + .setLandmarkVisibility(true) + .setModelComplexity(0) + .setSmoothLandmarks(true) + .build()); + poseTracking.setErrorListener( + (message, e) -> Log.e(TAG, "MediaPipe Face Detection error:" + message)); + cameraInput = new CameraInput(this); + + cameraInput.setNewFrameListener(textureFrame -> poseTracking.send(textureFrame)); + + + // Initializes a new Gl surface view with a user-defined PoseTrackingResultGlRenderer. + glSurfaceView = + new SolutionGlSurfaceView<>( + this, poseTracking.getGlContext(), poseTracking.getGlMajorVersion()); + glSurfaceView.setSolutionResultRenderer(new PoseTrackingResultGlRenderer()); + glSurfaceView.setRenderInputImage(true); + poseTracking.setResultListener( + poseTrackingResult -> { + logExampleKeypoint(poseTrackingResult); + glSurfaceView.setRenderData(poseTrackingResult); + glSurfaceView.requestRender(); + }); + + // The runnable to start camera after the gl surface view is attached. + // For video input source, videoInput.start() will be called when the video uri is available. + glSurfaceView.post(this::startCamera); + + // Updates the preview layout. + FrameLayout frameLayout = findViewById(R.id.preview_display_layout); + frameLayout.removeAllViewsInLayout(); + frameLayout.addView(glSurfaceView); + glSurfaceView.setVisibility(View.VISIBLE); + frameLayout.requestLayout(); + } + + private void startCamera() { + cameraInput.getConverter(poseTracking.getGlContext()).setRotation(rotation); + cameraInput.start( + this, + poseTracking.getGlContext(), + CameraInput.CameraFacing.FRONT, + glSurfaceView.getWidth(), + glSurfaceView.getHeight()); + } + + + private void logExampleKeypoint( + PoseTrackingResult result) { + if (result.multiPoseLandmarks().isEmpty()) { + return; + } + LandmarkProto.Landmark exampleLandmark = result.multiPoseLandmarks().get(PoseTrackingResult.NOSE); + Log.i( + TAG, + String.format( + "Pose Landmark Landmark of Nose: x=%f, y=%f, z=%f", + exampleLandmark.getX(), exampleLandmark.getY(), exampleLandmark.getZ())); + + + } } diff --git a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java index bd85a70bd..eda241b4b 100644 --- a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java +++ b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java @@ -1,7 +1,18 @@ package com.google.mediapipe.examples.posetracking_lindera; +import android.util.Log; + +import com.google.mediapipe.solutions.posetracking.BodyJoints; import com.google.mediapipe.solutions.posetracking.ComputerVisionPlugin; +import com.google.mediapipe.solutions.posetracking.XYZPointWithConfidence; public class ComputerVisionPluginImpl implements ComputerVisionPlugin { + @Override + public void bodyJoints(int timestamp, BodyJoints bodyJoints) { + XYZPointWithConfidence nose = bodyJoints.nose; + Log.v("ComputerVisionPluginImpl", String.format( + + "Lindera BodyJoint of Nose: x=%f, y=%f, z=%f", nose.x, nose.y, nose.z)); + } } diff --git a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java index 93b048b9c..57d36af28 100644 --- a/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java +++ b/mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java @@ -14,75 +14,67 @@ package com.google.mediapipe.examples.posetracking_lindera; -import android.content.Intent; - import android.os.Bundle; - -import androidx.appcompat.app.AppCompatActivity; - -import android.os.Handler; -import android.os.Looper; -import android.view.Surface; import android.view.View; import android.widget.Button; import android.widget.FrameLayout; -import androidx.activity.result.ActivityResultLauncher; +import androidx.appcompat.app.AppCompatActivity; -import com.google.mediapipe.solutioncore.SolutionGlSurfaceView; import com.google.mediapipe.solutions.posetracking.ComputerVisionPlugin; import com.google.mediapipe.solutions.posetracking.Lindera; -import com.google.mediapipe.solutions.posetracking.PoseTrackingResultGlRenderer; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -/** Main activity of MediaPipe Face Detection app. */ +/** + * Main activity of MediaPipe Face Detection app. + */ public class MainActivity extends AppCompatActivity { - private static final String TAG = "MainActivity"; + private static final String TAG = "MainActivity"; - private Lindera lindera; - private ComputerVisionPlugin plugin; - // Live camera demo UI and camera components. + private Lindera lindera; + private ComputerVisionPlugin plugin; + // Live camera demo UI and camera components. - - @Override - protected void onCreate(Bundle savedInstanceState) { - super.onCreate(savedInstanceState); - setContentView(R.layout.activity_main); + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.activity_main); + disableRedundantUI(); - disableRedundantUI(); - - setupLiveDemoUiComponents(); - plugin = new ComputerVisionPluginImpl(); - lindera = new Lindera(plugin); - } + setupLiveDemoUiComponents(); + plugin = new ComputerVisionPluginImpl(); + lindera = new Lindera(plugin); + } - /** Sets up the UI components for the live demo with camera input. */ - private void setupLiveDemoUiComponents() { + /** + * Sets up the UI components for the live demo with camera input. + */ + private void setupLiveDemoUiComponents() { - Button startCameraButton = findViewById(R.id.button_start_camera); - FrameLayout frameLayout = findViewById(R.id.preview_display_layout); + Button startCameraButton = findViewById(R.id.button_start_camera); + FrameLayout frameLayout = findViewById(R.id.preview_display_layout); - startCameraButton.setOnClickListener( - v -> { - startCameraButton.setVisibility(View.GONE); + startCameraButton.setOnClickListener( + v -> { + startCameraButton.setVisibility(View.GONE); - lindera.initialize(frameLayout, MainActivity.this); + lindera.initialize(frameLayout, MainActivity.this); - }); - } - /**Disables unecesary UI buttons*/ - private void disableRedundantUI(){ - findViewById(R.id.button_load_picture).setVisibility(View.GONE); - findViewById(R.id.button_load_video).setVisibility(View.GONE); + }); + } - } + /** + * Disables unecesary UI buttons + */ + private void disableRedundantUI() { + findViewById(R.id.button_load_picture).setVisibility(View.GONE); + findViewById(R.id.button_load_video).setVisibility(View.GONE); + + } } diff --git a/mediapipe/java/com/google/mediapipe/solutions/posetracking/BUILD b/mediapipe/java/com/google/mediapipe/solutions/posetracking/BUILD index 579f75310..ef5493b8f 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/posetracking/BUILD +++ b/mediapipe/java/com/google/mediapipe/solutions/posetracking/BUILD @@ -117,7 +117,11 @@ android_library( android_library( name = "copperlabs-lindera-dummpy-interface", - srcs = ["ComputerVisionPlugin.java"], + srcs = [ + "BodyJoints.java", + "ComputerVisionPlugin.java", + "XYZPointWithConfidence.java", + ], manifest = "AndroidManifest.xml", visibility = ["//visibility:public"], deps = [], diff --git a/mediapipe/java/com/google/mediapipe/solutions/posetracking/ComputerVisionPlugin.java b/mediapipe/java/com/google/mediapipe/solutions/posetracking/ComputerVisionPlugin.java index 48d193adb..ed96377df 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/posetracking/ComputerVisionPlugin.java +++ b/mediapipe/java/com/google/mediapipe/solutions/posetracking/ComputerVisionPlugin.java @@ -1,4 +1,5 @@ package com.google.mediapipe.solutions.posetracking; public interface ComputerVisionPlugin { + void bodyJoints(int timestamp, BodyJoints bodyJoints); } diff --git a/mediapipe/java/com/google/mediapipe/solutions/posetracking/Lindera.java b/mediapipe/java/com/google/mediapipe/solutions/posetracking/Lindera.java index a69097868..1a3840dd1 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/posetracking/Lindera.java +++ b/mediapipe/java/com/google/mediapipe/solutions/posetracking/Lindera.java @@ -7,6 +7,8 @@ import android.view.ViewGroup; import androidx.appcompat.app.AppCompatActivity; +import com.google.common.collect.ImmutableList; +import com.google.mediapipe.formats.proto.LandmarkProto; import com.google.mediapipe.solutioncore.CameraInput; import com.google.mediapipe.solutioncore.SolutionGlSurfaceView; @@ -14,7 +16,8 @@ public class Lindera { private ComputerVisionPlugin plugin; private static final int rotation = Surface.ROTATION_0; private PoseTracking poseTracking; - + // TODO: Verify that this is the timestamp used in Actual Plugin + private int timeStamp = 0; // Live camera demo UI and camera components. private CameraInput cameraInput; private SolutionGlSurfaceView glSurfaceView; @@ -28,6 +31,8 @@ public class Lindera { setupStreamingModePipeline(computerVisionContainerView,appCompatActivity); } + + /** Sets up core workflow for streaming mode. */ private void setupStreamingModePipeline(ViewGroup computerVisionContainerView,AppCompatActivity appCompatActivity) { // Initializes a new MediaPipe Face Detection solution instance in the streaming mode. @@ -53,13 +58,8 @@ public class Lindera { appCompatActivity, poseTracking.getGlContext(), poseTracking.getGlMajorVersion()); glSurfaceView.setSolutionResultRenderer(new PoseTrackingResultGlRenderer()); glSurfaceView.setRenderInputImage(true); - poseTracking.setResultListener( - poseTrackingResult -> { -// logExampleKeypoint(poseTrackingResult); - glSurfaceView.setRenderData(poseTrackingResult); - glSurfaceView.requestRender(); - }); + setupEventListener(); // The runnable to start camera after the gl surface view is attached. // For video input source, videoInput.start() will be called when the video uri is available. glSurfaceView.post(()->{this.startCamera(appCompatActivity);}); @@ -81,4 +81,81 @@ public class Lindera { glSurfaceView.getHeight()); } + public void setupEventListener() { + poseTracking.setResultListener( + poseTrackingResult -> { + glSurfaceView.setRenderData(poseTrackingResult); + glSurfaceView.requestRender(); + ImmutableList landmarks = poseTrackingResult.multiPoseLandmarks(); + timeStamp+=1; + + if (landmarks.isEmpty()) return; + + BodyJoints bodyJoints = new BodyJoints(); + landmarksToBodyJoints(landmarks,bodyJoints); + + plugin.bodyJoints(timeStamp, bodyJoints); + }); + } + private void landmarkToXYZPointWithConfidence(LandmarkProto.Landmark landmark,XYZPointWithConfidence bodyJoint){ + bodyJoint.x = landmark.getX(); + bodyJoint.y = landmark.getY(); + bodyJoint.z = landmark.getZ(); + bodyJoint.confidence = landmark.getVisibility(); + } + private void landmarksToBodyJoints(ImmutableList landmarks , BodyJoints bodyJoints){ + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.NOSE), bodyJoints.nose); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE_INNER), bodyJoints.leftEyeInner); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE), bodyJoints.leftEye); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE_OUTER), bodyJoints.leftEyeOuter); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EYE_INNER), bodyJoints.rightEyeInner); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EYE), bodyJoints.rightEye); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EYE_OUTER), bodyJoints.rightEyeOuter); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EAR), bodyJoints.leftEar); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EAR), bodyJoints.rightEar); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.MOUTH_LEFT), bodyJoints.mouthLeft); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.MOUTH_RIGHT), bodyJoints.mouthRight); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_SHOULDER), bodyJoints.leftShoulder); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_SHOULDER), bodyJoints.rightShoulder); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_ELBOW), bodyJoints.leftElbow); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_ELBOW), bodyJoints.rightElbow); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_WRIST), bodyJoints.leftWrist); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_WRIST), bodyJoints.rightWrist); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_PINKY), bodyJoints.leftPinky); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_PINKY), bodyJoints.rightPinky); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_INDEX), bodyJoints.leftIndex); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_INDEX), bodyJoints.rightIndex); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_THUMB), bodyJoints.leftThumb); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_THUMB), bodyJoints.rightThumb); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_HIP), bodyJoints.leftHip); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_HIP), bodyJoints.rightHip); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_KNEE), bodyJoints.leftKnee); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_KNEE), bodyJoints.rightKnee); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_ANKLE), bodyJoints.rightAnkle); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_ANKLE), bodyJoints.leftAnkle); + + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_HEEL), bodyJoints.rightHeel); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_HEEL), bodyJoints.leftHeel); + + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_FOOT), bodyJoints.rightFoot); + landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_FOOT), bodyJoints.leftFoot); + } + + + + } diff --git a/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java b/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java index 1e173c3b5..46dc45f90 100644 --- a/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java +++ b/mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java @@ -49,7 +49,7 @@ public class PoseTrackingResult extends ImageSolutionResult { public static final int LEFT_SHOULDER = 11; public static final int RIGHT_SHOULDER = 12; public static final int LEFT_ELBOW = 13; - public static final int RIGH_ELBOW = 14; + public static final int RIGHT_ELBOW = 14; public static final int LEFT_WRIST = 15; public static final int RIGHT_WRIST = 16; public static final int LEFT_PINKY = 17;