cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

lindera callback working

Browse Source
This commit is contained in:
Mautisim Munir 2022-10-15 13:04:31 +05:00
parent 1c3fc550e9
commit 333c18569f
7 changed files with 250 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
mediapipe/examples/android/solutions/posetracking-camera/src/main/java/com/google/mediapipe/examples/posetracking_camera/MainActivity.java
View File

@ -15,43 +15,34 @@
package com.google.mediapipe.examples.posetracking_camera;
import android.content.Intent;
import android.graphics.Bitmap;
import android.graphics.Matrix;
import android.os.Bundle;
import android.os.Looper;
import android.provider.MediaStore;
import androidx.appcompat.app.AppCompatActivity;
import android.util.Log;
import android.view.Surface;
import android.view.View;
import android.widget.Button;
import android.widget.FrameLayout;
import android.widget.ImageView;
import androidx.activity.result.ActivityResultLauncher;
import androidx.activity.result.contract.ActivityResultContracts;
import androidx.exifinterface.media.ExifInterface;
// ContentResolver dependency
import androidx.appcompat.app.AppCompatActivity;
import com.google.mediapipe.formats.proto.LandmarkProto;
import com.google.mediapipe.solutioncore.CameraInput;
import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
import com.google.mediapipe.solutioncore.VideoInput;
import com.google.mediapipe.solutions.posetracking.PoseTracking;
import com.google.mediapipe.solutions.posetracking.PoseTrackingOptions;
import com.google.mediapipe.solutions.posetracking.PoseTrackingResult;
import com.google.mediapipe.solutions.posetracking.PoseTrackingResultGlRenderer;
import java.io.IOException;
import java.io.InputStream;
/** Main activity of MediaPipe Face Detection app. */
/**
* Main activity of MediaPipe Face Detection app.
*/
public class MainActivity extends AppCompatActivity {
private static final String TAG = "MainActivity";
private static final int rotation = Surface.ROTATION_0;
private PoseTracking poseTracking;
private ActivityResultLauncher<Intent> videoGetter;
// Live camera demo UI and camera components.
private CameraInput cameraInput;
@ -69,7 +60,9 @@ public class MainActivity extends AppCompatActivity {
}
/** Sets up the UI components for the live demo with camera input. */
/**
* Sets up the UI components for the live demo with camera input.
*/
private void setupLiveDemoUiComponents() {
Button startCameraButton = findViewById(R.id.button_start_camera);
@ -79,14 +72,19 @@ public class MainActivity extends AppCompatActivity {
startCameraButton.setVisibility(View.GONE);
});
}
/**Disables unecesary UI buttons*/
/**
* Disables unecesary UI buttons
*/
private void disableRedundantUI() {
findViewById(R.id.button_load_picture).setVisibility(View.GONE);
findViewById(R.id.button_load_video).setVisibility(View.GONE);
}
/** Sets up core workflow for streaming mode. */
/**
* Sets up core workflow for streaming mode.
*/
private void setupStreamingModePipeline() {
// Initializes a new MediaPipe Face Detection solution instance in the streaming mode.
poseTracking =

11
mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java
View File

@ -1,7 +1,18 @@
package com.google.mediapipe.examples.posetracking_lindera;
import android.util.Log;
import com.google.mediapipe.solutions.posetracking.BodyJoints;
import com.google.mediapipe.solutions.posetracking.ComputerVisionPlugin;
import com.google.mediapipe.solutions.posetracking.XYZPointWithConfidence;
public class ComputerVisionPluginImpl implements ComputerVisionPlugin {
@Override
public void bodyJoints(int timestamp, BodyJoints bodyJoints) {
XYZPointWithConfidence nose = bodyJoints.nose;
Log.v("ComputerVisionPluginImpl", String.format(
"Lindera BodyJoint of Nose: x=%f, y=%f, z=%f", nose.x, nose.y, nose.z));
}
}

30
mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java
View File

@ -14,31 +14,20 @@
package com.google.mediapipe.examples.posetracking_lindera;
import android.content.Intent;
import android.os.Bundle;
import androidx.appcompat.app.AppCompatActivity;
import android.os.Handler;
import android.os.Looper;
import android.view.Surface;
import android.view.View;
import android.widget.Button;
import android.widget.FrameLayout;
import androidx.activity.result.ActivityResultLauncher;
import androidx.appcompat.app.AppCompatActivity;
import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
import com.google.mediapipe.solutions.posetracking.ComputerVisionPlugin;
import com.google.mediapipe.solutions.posetracking.Lindera;
import com.google.mediapipe.solutions.posetracking.PoseTrackingResultGlRenderer;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/** Main activity of MediaPipe Face Detection app. */
/**
* Main activity of MediaPipe Face Detection app.
*/
public class MainActivity extends AppCompatActivity {
private static final String TAG = "MainActivity";
@ -47,14 +36,12 @@ public class MainActivity extends AppCompatActivity {
// Live camera demo UI and camera components.
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
disableRedundantUI();
setupLiveDemoUiComponents();
@ -63,7 +50,9 @@ public class MainActivity extends AppCompatActivity {
}
/** Sets up the UI components for the live demo with camera input. */
/**
* Sets up the UI components for the live demo with camera input.
*/
private void setupLiveDemoUiComponents() {
Button startCameraButton = findViewById(R.id.button_start_camera);
@ -77,7 +66,10 @@ public class MainActivity extends AppCompatActivity {
});
}
/**Disables unecesary UI buttons*/
/**
* Disables unecesary UI buttons
*/
private void disableRedundantUI() {
findViewById(R.id.button_load_picture).setVisibility(View.GONE);
findViewById(R.id.button_load_video).setVisibility(View.GONE);

6
mediapipe/java/com/google/mediapipe/solutions/posetracking/BUILD
View File

@ -117,7 +117,11 @@ android_library(
android_library(
name = "copperlabs-lindera-dummpy-interface",
srcs = ["ComputerVisionPlugin.java"],
srcs = [
"BodyJoints.java",
"ComputerVisionPlugin.java",
"XYZPointWithConfidence.java",
],
manifest = "AndroidManifest.xml",
visibility = ["//visibility:public"],
deps = [],

1
mediapipe/java/com/google/mediapipe/solutions/posetracking/ComputerVisionPlugin.java
View File

@ -1,4 +1,5 @@
package com.google.mediapipe.solutions.posetracking;
public interface ComputerVisionPlugin {
void bodyJoints(int timestamp, BodyJoints bodyJoints);
}

91
mediapipe/java/com/google/mediapipe/solutions/posetracking/Lindera.java
View File

@ -7,6 +7,8 @@ import android.view.ViewGroup;
import androidx.appcompat.app.AppCompatActivity;
import com.google.common.collect.ImmutableList;
import com.google.mediapipe.formats.proto.LandmarkProto;
import com.google.mediapipe.solutioncore.CameraInput;
import com.google.mediapipe.solutioncore.SolutionGlSurfaceView;
@ -14,7 +16,8 @@ public class Lindera {
private ComputerVisionPlugin plugin;
private static final int rotation = Surface.ROTATION_0;
private PoseTracking poseTracking;
// TODO: Verify that this is the timestamp used in Actual Plugin
private int timeStamp = 0;
// Live camera demo UI and camera components.
private CameraInput cameraInput;
private SolutionGlSurfaceView<PoseTrackingResult> glSurfaceView;
@ -28,6 +31,8 @@ public class Lindera {
setupStreamingModePipeline(computerVisionContainerView,appCompatActivity);
}
/** Sets up core workflow for streaming mode. */
private void setupStreamingModePipeline(ViewGroup computerVisionContainerView,AppCompatActivity appCompatActivity) {
// Initializes a new MediaPipe Face Detection solution instance in the streaming mode.
@ -53,13 +58,8 @@ public class Lindera {
appCompatActivity, poseTracking.getGlContext(), poseTracking.getGlMajorVersion());
glSurfaceView.setSolutionResultRenderer(new PoseTrackingResultGlRenderer());
glSurfaceView.setRenderInputImage(true);
poseTracking.setResultListener(
poseTrackingResult -> {
// logExampleKeypoint(poseTrackingResult);
glSurfaceView.setRenderData(poseTrackingResult);
glSurfaceView.requestRender();
});
setupEventListener();
// The runnable to start camera after the gl surface view is attached.
// For video input source, videoInput.start() will be called when the video uri is available.
glSurfaceView.post(()->{this.startCamera(appCompatActivity);});
@ -81,4 +81,81 @@ public class Lindera {
glSurfaceView.getHeight());
}
public void setupEventListener() {
poseTracking.setResultListener(
poseTrackingResult -> {
glSurfaceView.setRenderData(poseTrackingResult);
glSurfaceView.requestRender();
ImmutableList<LandmarkProto.Landmark> landmarks = poseTrackingResult.multiPoseLandmarks();
timeStamp+=1;
if (landmarks.isEmpty()) return;
BodyJoints bodyJoints = new BodyJoints();
landmarksToBodyJoints(landmarks,bodyJoints);
plugin.bodyJoints(timeStamp, bodyJoints);
});
}
private void landmarkToXYZPointWithConfidence(LandmarkProto.Landmark landmark,XYZPointWithConfidence bodyJoint){
bodyJoint.x = landmark.getX();
bodyJoint.y = landmark.getY();
bodyJoint.z = landmark.getZ();
bodyJoint.confidence = landmark.getVisibility();
}
private void landmarksToBodyJoints(ImmutableList<LandmarkProto.Landmark> landmarks , BodyJoints bodyJoints){
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.NOSE), bodyJoints.nose);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE_INNER), bodyJoints.leftEyeInner);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE), bodyJoints.leftEye);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE_OUTER), bodyJoints.leftEyeOuter);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EYE_INNER), bodyJoints.rightEyeInner);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EYE), bodyJoints.rightEye);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EYE_OUTER), bodyJoints.rightEyeOuter);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EAR), bodyJoints.leftEar);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_EAR), bodyJoints.rightEar);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.MOUTH_LEFT), bodyJoints.mouthLeft);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.MOUTH_RIGHT), bodyJoints.mouthRight);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_SHOULDER), bodyJoints.leftShoulder);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_SHOULDER), bodyJoints.rightShoulder);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_ELBOW), bodyJoints.leftElbow);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_ELBOW), bodyJoints.rightElbow);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_WRIST), bodyJoints.leftWrist);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_WRIST), bodyJoints.rightWrist);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_PINKY), bodyJoints.leftPinky);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_PINKY), bodyJoints.rightPinky);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_INDEX), bodyJoints.leftIndex);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_INDEX), bodyJoints.rightIndex);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_THUMB), bodyJoints.leftThumb);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_THUMB), bodyJoints.rightThumb);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_HIP), bodyJoints.leftHip);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_HIP), bodyJoints.rightHip);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_KNEE), bodyJoints.leftKnee);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_KNEE), bodyJoints.rightKnee);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_ANKLE), bodyJoints.rightAnkle);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_ANKLE), bodyJoints.leftAnkle);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_HEEL), bodyJoints.rightHeel);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_HEEL), bodyJoints.leftHeel);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_FOOT), bodyJoints.rightFoot);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_FOOT), bodyJoints.leftFoot);
}
}

2
mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java
View File

@ -49,7 +49,7 @@ public class PoseTrackingResult extends ImageSolutionResult {
public static final int LEFT_SHOULDER = 11;
public static final int RIGHT_SHOULDER = 12;
public static final int LEFT_ELBOW = 13;
public static final int RIGH_ELBOW = 14;
public static final int RIGHT_ELBOW = 14;
public static final int LEFT_WRIST = 15;
public static final int RIGHT_WRIST = 16;
public static final int LEFT_PINKY = 17;