cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'add-interpolated-landmarks' into merge-branch

Browse Source 
# Conflicts:
#	mediapipe/java/com/google/mediapipe/solutions/lindera/ComputerVisionPlugin.java
#	mediapipe/java/com/google/mediapipe/solutions/lindera/Lindera.java
This commit is contained in:
Mautisim Munir 2022-11-07 20:32:08 +05:00
commit 607b3954ce
10 changed files with 283 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mediapipe/examples/android/solutions/posetracking-lindera/src/main/BUILD
View File

@ -20,6 +20,7 @@ android_binary(
name = "posetracking-lindera",
srcs = glob(["**/*.java"]),
custom_package = "com.google.mediapipe.examples.posetracking_lindera",
dex_shards = 10,
manifest = "AndroidManifest.xml",
manifest_values = {
"applicationId": "com.google.mediapipe.examples.posetracking_lindera",

24
mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/ComputerVisionPluginImpl.java
View File

@ -1,5 +1,7 @@
package com.google.mediapipe.examples.posetracking_lindera;
import static java.lang.Math.min;
import com.google.mediapipe.solutions.lindera.BodyJoints;
import com.google.mediapipe.solutions.lindera.ComputerVisionPlugin;
import com.google.mediapipe.solutions.lindera.XYZPointWithConfidence;
@ -9,8 +11,10 @@ import org.json.JSONException;
import org.json.JSONObject;
import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Locale;
import java.util.Map;
public class ComputerVisionPluginImpl implements ComputerVisionPlugin {
@ -76,6 +80,18 @@ public class ComputerVisionPluginImpl implements ComputerVisionPlugin {
}
}
abbrev = abbrev.toUpperCase(Locale.ROOT);
// correct abbreviations here
switch (abbrev) {
case "P":
abbrev = "PE";
break;
case "T":
abbrev = "TH";
break;
case "S":
abbrev = "SP";
break;
}
XYZPointWithConfidence data = (XYZPointWithConfidence) field.get(bodyJoints);
assert data != null;
bodyJointsString = bodyJointsString.concat(String.format(abbrev+":%f,%f,%f=",data.x,data.y,data.z));
@ -102,8 +118,14 @@ public class ComputerVisionPluginImpl implements ComputerVisionPlugin {
}
@Override
public void bodyJoints(int timestamp, BodyJoints bodyJoints) {
public void bodyJoints(long timestamp, BodyJoints bodyJoints) {
if (isLogging){
this.bodyJointsEventList.add(new BodyJointsEvent(timestamp,bodyJoints));

6
mediapipe/examples/android/solutions/posetracking-lindera/src/main/java/com/google/mediapipe/examples/posetracking_lindera/MainActivity.java
View File

@ -83,7 +83,7 @@ public class MainActivity extends AppCompatActivity {
findViewById(R.id.button_set_model).setVisibility(View.GONE);
findViewById(R.id.button_toggle_landmarks).setVisibility(View.GONE);
findViewById(R.id.button_start_capture).setVisibility(View.GONE);
findViewById(R.id.button_capture_logging).setVisibility(View.GONE);
setupLiveDemoUiComponents();
plugin = new ComputerVisionPluginImpl();
@ -116,7 +116,7 @@ public class MainActivity extends AppCompatActivity {
Button startDetectionButton = findViewById(R.id.button_start_detection);
Button toggleLandmarks = findViewById(R.id.button_toggle_landmarks);
Button modelComplexity = findViewById(R.id.button_set_model);
Button startCapture = findViewById(R.id.button_start_capture);
Button startCapture = findViewById(R.id.button_capture_logging);
FrameLayout frameLayout = findViewById(R.id.preview_display_layout);
startDetectionButton.setOnClickListener(
@ -129,7 +129,7 @@ public class MainActivity extends AppCompatActivity {
startDetectionButton.setVisibility(View.GONE);
findViewById(R.id.button_set_model).setVisibility(View.VISIBLE);
findViewById(R.id.button_toggle_landmarks).setVisibility(View.VISIBLE);
findViewById(R.id.button_start_capture).setVisibility(View.VISIBLE);
findViewById(R.id.button_capture_logging).setVisibility(View.VISIBLE);
updateLandmarkButtonText();
updateModelComplexityButtonText();

49
mediapipe/examples/android/solutions/posetracking-lindera/src/main/res/layout/activity_main.xml
View File

@ -1,5 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
@ -10,7 +11,8 @@
style="?android:attr/buttonBarStyle"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:gravity="center">
android:gravity="center"
android:orientation="horizontal">
<Button
android:id="@+id/button_start_detection"
@ -24,38 +26,57 @@
style="?android:attr/buttonBarButtonStyle"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="Show Landmarks" />
android:text="Landmarks" />
<Button
android:id="@+id/button_set_model"
style="?android:attr/buttonBarButtonStyle"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="Set Model" />
android:text="Model" />
<Button
android:id="@+id/button_start_capture"
android:id="@+id/button_capture_logging"
style="?android:attr/buttonBarButtonStyle"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:text="Start Capture" />
<TextView
android:id="@+id/fps_view"
android:layout_width="wrap_content"
android:layout_height="wrap_content" />
</LinearLayout>
<FrameLayout
android:id="@+id/preview_display_layout"
android:id="@+id/overlay"
style="?android:attr/buttonBarStyle"
android:layout_width="match_parent"
android:layout_height="match_parent">
android:layout_height="match_parent"
android:gravity="center"
android:orientation="horizontal">
<FrameLayout
android:id="@+id/preview_display_layout"
android:layout_width="match_parent"
android:layout_height="match_parent">
<TextView
android:id="@+id/no_view"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:gravity="center"
android:text="@string/instruction" />
</FrameLayout>
<TextView
android:id="@+id/no_view"
android:layout_width="match_parent"
android:id="@+id/fps_view"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:gravity="center"
android:text="@string/instruction" />
android:textSize="30sp"
android:textStyle="bold"
android:layout_marginTop="20dp"
android:layout_marginLeft="20dp"
android:elevation="10dp"
android:gravity="center_horizontal|center_vertical"/>
</FrameLayout>
</LinearLayout>

3
mediapipe/java/com/google/mediapipe/solutioncore/SolutionGlSurfaceViewRenderer.java
View File

@ -49,6 +49,9 @@ public class SolutionGlSurfaceViewRenderer<T extends ImageSolutionResult>
public void setSolutionResultRenderer(ResultGlRenderer<T> resultGlRenderer) {
this.resultGlRenderer = resultGlRenderer;
}
public ResultGlRenderer<T> getSolutionResultRenderer(){
return this.resultGlRenderer;
}
/**
* Sets the next input {@link TextureFrame} and solution result to render.

20
mediapipe/java/com/google/mediapipe/solutions/lindera/BodyJoints.java
View File

@ -1,7 +1,7 @@
package com.google.mediapipe.solutions.lindera;
public class BodyJoints {
public XYZPointWithConfidence nose;
public XYZPointWithConfidence neckNose;
public XYZPointWithConfidence leftEyeInner;
public XYZPointWithConfidence leftEye;
@ -52,16 +52,15 @@ public class BodyJoints {
public XYZPointWithConfidence leftFoot;
// public XYZPointWithConfidence pelvis;
//
// public XYZPointWithConfidence spine;
// public XYZPointWithConfidence thorax;
// public XYZPointWithConfidence neckNose;
// public XYZPointWithConfidence headTop;
public XYZPointWithConfidence pelvis;
public XYZPointWithConfidence spine;
public XYZPointWithConfidence thorax;
public XYZPointWithConfidence headTop;
public BodyJoints() {
nose = new XYZPointWithConfidence();
neckNose = new XYZPointWithConfidence();
leftEyeInner= new XYZPointWithConfidence();
leftEye= new XYZPointWithConfidence();
@ -111,5 +110,10 @@ public class BodyJoints {
rightFoot= new XYZPointWithConfidence();
leftFoot= new XYZPointWithConfidence();
pelvis = new XYZPointWithConfidence();
spine = new XYZPointWithConfidence();
thorax = new XYZPointWithConfidence();
headTop = new XYZPointWithConfidence();
}
}

21
mediapipe/java/com/google/mediapipe/solutions/lindera/Lindera.java
View File

@ -33,6 +33,7 @@ public class Lindera {
private CameraInput.CameraFacing cameraFacing = CameraInput.CameraFacing.FRONT;
private AppCompatActivity appCompatActivity;
private ViewGroup computerVisionContainerView;
private PoseTrackingResultGlRenderer solutionRenderer;
public Lindera(ComputerVisionPlugin plugin){
this.plugin = plugin;
@ -41,6 +42,8 @@ public class Lindera {
public void setLandmarksVisibility(boolean visible){
this.poseTracking.options = PoseTrackingOptions.builder().withPoseTrackingOptions(this.poseTracking
.options).setLandmarkVisibility(visible).build();
solutionRenderer.setLandmarksVisibility(this.poseTracking.options.landmarkVisibility());
glSurfaceView.setSolutionResultRenderer(solutionRenderer);
}
public boolean getLandmarkVisibility(){
return this.poseTracking.options.landmarkVisibility();
@ -150,7 +153,9 @@ public class Lindera {
poseTracking.getGlContext(),
poseTracking.getGlMajorVersion()
);
glSurfaceView.setSolutionResultRenderer(new PoseTrackingResultGlRenderer());
solutionRenderer = new PoseTrackingResultGlRenderer();
solutionRenderer.setLandmarksVisibility(this.poseTracking.options.landmarkVisibility());
glSurfaceView.setSolutionResultRenderer(solutionRenderer);
glSurfaceView.setRenderInputImage(true);
setupEventListener();
@ -184,10 +189,12 @@ public class Lindera {
bodyJoint.y = landmark.getY();
bodyJoint.z = landmark.getZ();
bodyJoint.confidence = landmark.getVisibility();
bodyJoint.presence = landmark.getPresence();
}
private void landmarksToBodyJoints(ImmutableList<LandmarkProto.Landmark> landmarks , BodyJoints bodyJoints){
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.NOSE), bodyJoints.nose);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.NOSE), bodyJoints.neckNose);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE_INNER), bodyJoints.leftEyeInner);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_EYE), bodyJoints.leftEye);
@ -236,6 +243,16 @@ public class Lindera {
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.RIGHT_FOOT), bodyJoints.rightFoot);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.LEFT_FOOT), bodyJoints.leftFoot);
// additional points
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.PELVIS), bodyJoints.pelvis);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.SPINE), bodyJoints.spine);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.THORAX), bodyJoints.thorax);
landmarkToXYZPointWithConfidence(landmarks.get(PoseTrackingResult.HEAD_TOP), bodyJoints.headTop);
}
private void startCamera() {

1
mediapipe/java/com/google/mediapipe/solutions/lindera/XYZPointWithConfidence.java
View File

@ -5,6 +5,7 @@ public class XYZPointWithConfidence {
public float y = 0;
public float z = 0;
public float confidence = 0;
public float presence = 0;

218
mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResult.java
View File

@ -14,16 +14,21 @@
package com.google.mediapipe.solutions.posetracking;
import static java.lang.Math.min;
import android.graphics.Bitmap;
import com.google.auto.value.AutoBuilder;
import com.google.common.collect.ImmutableList;
import com.google.mediapipe.formats.proto.DetectionProto.Detection;
import com.google.mediapipe.formats.proto.LandmarkProto;
import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.framework.TextureFrame;
import com.google.mediapipe.solutioncore.ImageSolutionResult;
import com.google.mediapipe.formats.proto.DetectionProto.Detection;
import java.util.List;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
/**
@ -32,80 +37,157 @@ import java.util.List;
* the corresponding input image.
*/
public class PoseTrackingResult extends ImageSolutionResult {
private final ImmutableList<Detection> multiPoseDetections;
private final ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks;
public static final int NOSE = 0;
public static final int LEFT_EYE_INNER = 1;
public static final int LEFT_EYE = 2;
public static final int LEFT_EYE_OUTER = 3;
public static final int RIGHT_EYE_INNER = 4;
public static final int RIGHT_EYE = 5;
public static final int RIGHT_EYE_OUTER = 6;
public static final int LEFT_EAR = 7;
public static final int RIGHT_EAR = 8;
public static final int MOUTH_LEFT = 9;
public static final int MOUTH_RIGHT = 10;
public static final int LEFT_SHOULDER = 11;
public static final int RIGHT_SHOULDER = 12;
public static final int LEFT_ELBOW = 13;
public static final int RIGHT_ELBOW = 14;
public static final int LEFT_WRIST = 15;
public static final int RIGHT_WRIST = 16;
public static final int LEFT_PINKY = 17;
public static final int RIGHT_PINKY = 18;
public static final int LEFT_INDEX = 19;
public static final int RIGHT_INDEX = 20;
public static final int LEFT_THUMB = 21;
public static final int RIGHT_THUMB = 22;
public static final int LEFT_HIP = 23;
public static final int RIGHT_HIP = 24;
public static final int LEFT_KNEE = 25;
public static final int RIGHT_KNEE = 26;
public static final int LEFT_ANKLE = 27;
public static final int RIGHT_ANKLE = 28;
public static final int LEFT_HEEL = 29;
public static final int RIGHT_HEEL = 30;
public static final int LEFT_FOOT = 31;
public static final int RIGHT_FOOT = 32;
private final ImmutableList<Detection> multiPoseDetections;
private final ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks;
PoseTrackingResult(
ImmutableList<Detection> multiPoseDetections,ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks, Packet imagePacket, long timestamp) {
this.multiPoseDetections = multiPoseDetections;
this.multiPoseLandmarks = multiPoseLandmarks;
this.timestamp = timestamp;
this.imagePacket = imagePacket;
}
public static final int NOSE = 0;
public static final int LEFT_EYE_INNER = 1;
public static final int LEFT_EYE = 2;
public static final int LEFT_EYE_OUTER = 3;
public static final int RIGHT_EYE_INNER = 4;
public static final int RIGHT_EYE = 5;
public static final int RIGHT_EYE_OUTER = 6;
public static final int LEFT_EAR = 7;
public static final int RIGHT_EAR = 8;
public static final int MOUTH_LEFT = 9;
public static final int MOUTH_RIGHT = 10;
public static final int LEFT_SHOULDER = 11;
public static final int RIGHT_SHOULDER = 12;
public static final int LEFT_ELBOW = 13;
public static final int RIGHT_ELBOW = 14;
public static final int LEFT_WRIST = 15;
public static final int RIGHT_WRIST = 16;
public static final int LEFT_PINKY = 17;
public static final int RIGHT_PINKY = 18;
public static final int LEFT_INDEX = 19;
public static final int RIGHT_INDEX = 20;
public static final int LEFT_THUMB = 21;
public static final int RIGHT_THUMB = 22;
public static final int LEFT_HIP = 23;
public static final int RIGHT_HIP = 24;
public static final int LEFT_KNEE = 25;
public static final int RIGHT_KNEE = 26;
public static final int LEFT_ANKLE = 27;
public static final int RIGHT_ANKLE = 28;
public static final int LEFT_HEEL = 29;
public static final int RIGHT_HEEL = 30;
public static final int LEFT_FOOT = 31;
public static final int RIGHT_FOOT = 32;
// Collection of detected faces, where each face is represented as a detection proto message that
// contains a bounding box and 6 {@link FaceKeypoint}s. The bounding box is composed of xmin and
// width (both normalized to [0.0, 1.0] by the image width) and ymin and height (both normalized
// to [0.0, 1.0] by the image height). Each keypoint is composed of x and y, which are normalized
// to [0.0, 1.0] by the image width and height respectively.
public ImmutableList<Detection> multiPoseTrackings() {
return multiPoseDetections;
}
// Additional points not provided by MediaPipe
public static final int PELVIS = 33;
public static final int SPINE = 34;
public static final int THORAX = 35;
public static final int HEAD_TOP = 36;
public ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks() {
return multiPoseLandmarks;
}
PoseTrackingResult(
ImmutableList<Detection> multiPoseDetections, ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks, Packet imagePacket, long timestamp) {
this.multiPoseDetections = multiPoseDetections;
this.multiPoseLandmarks = multiPoseLandmarks;
this.timestamp = timestamp;
this.imagePacket = imagePacket;
}
public static Builder builder() {
return new AutoBuilder_PoseTrackingResult_Builder();
}
// Collection of detected faces, where each face is represented as a detection proto message that
// contains a bounding box and 6 {@link FaceKeypoint}s. The bounding box is composed of xmin and
// width (both normalized to [0.0, 1.0] by the image width) and ymin and height (both normalized
// to [0.0, 1.0] by the image height). Each keypoint is composed of x and y, which are normalized
// to [0.0, 1.0] by the image width and height respectively.
public ImmutableList<Detection> multiPoseTrackings() {
return multiPoseDetections;
}
/** Builder for {@link PoseTrackingResult}. */
@AutoBuilder
public abstract static class Builder {
abstract Builder setMultiPoseDetections(List<Detection> value);
abstract Builder setMultiPoseLandmarks(List<LandmarkProto.Landmark> value);
static LandmarkProto.Landmark getJointBetweenPoints(LandmarkProto.Landmark pt1, LandmarkProto.Landmark pt2, float distance) {
return LandmarkProto.Landmark.newBuilder()
.setX(pt1.getX() + (pt2.getX() - pt1.getX()) * distance)
.setY(pt1.getY() + (pt2.getY() - pt1.getY()) * distance)
.setZ(pt1.getZ() + (pt2.getZ() - pt1.getZ()) * distance)
.setPresence(min(pt1.getPresence(), pt2.getPresence()))
.setVisibility(min(pt1.getVisibility(), pt2.getVisibility())).build();
}
abstract Builder setTimestamp(long value);
LandmarkProto.Landmark getPelvis(ImmutableList<LandmarkProto.Landmark> landmarks) {
// middle point b/w left hip and right hip
return getJointBetweenPoints(landmarks.get(LEFT_HIP), landmarks.get(RIGHT_HIP), 0.5f);
}
abstract Builder setImagePacket(Packet value);
LandmarkProto.Landmark getSpinePoint(ImmutableList<LandmarkProto.Landmark> landmarks, float distanceFromShoulders) {
LandmarkProto.Landmark pelvis = getPelvis(landmarks);
// middle point b/w left shoulder and right shoulder
LandmarkProto.Landmark chest = getJointBetweenPoints(landmarks.get(LEFT_SHOULDER), landmarks.get(RIGHT_SHOULDER), 0.5f);
abstract PoseTrackingResult build();
}
return getJointBetweenPoints(chest, pelvis, distanceFromShoulders);
}
LandmarkProto.Landmark getHeadTop(ImmutableList<LandmarkProto.Landmark> landmarks) {
float x = 0;
float y = 0;
float z = 0;
float confidence = 1;
float presence = 1;
final List<Integer> ptsIdx = Arrays.asList(LEFT_EAR, LEFT_EYE, RIGHT_EYE, RIGHT_EAR);
for (Integer i :ptsIdx){
LandmarkProto.Landmark landmark = landmarks.get(i);
x+=landmark.getX();
y+=landmark.getY();
z+=landmark.getZ();
presence = min(presence,landmark.getPresence());
confidence = min(confidence,landmark.getVisibility());
}
x = x/ptsIdx.size();
y = y/ptsIdx.size();
z = z/ptsIdx.size();
LandmarkProto.Landmark midupper = LandmarkProto.Landmark.newBuilder().setX(x).setY(y).setZ(z)
.setVisibility(confidence).setPresence(presence).build();
LandmarkProto.Landmark midlower = getJointBetweenPoints(landmarks.get(MOUTH_LEFT), landmarks.get(MOUTH_RIGHT), 0.5f);
// 2 times the distance b/w nose and eyes
return getJointBetweenPoints(midlower, midupper, 2.5f);
}
ImmutableList<LandmarkProto.Landmark> getAdditionalLandmarksByInterpolation(ImmutableList<LandmarkProto.Landmark> originalLandmarks) {
if (originalLandmarks.isEmpty()) return originalLandmarks;
List<LandmarkProto.Landmark> landmarks = new ArrayList<>(originalLandmarks);
// pelvis
landmarks.add(getPelvis(originalLandmarks));
// spine assuming it is 2/3rd of distance b/w shoulders and pelvis
landmarks.add(getSpinePoint(originalLandmarks, 2 / 3f));
// thorax assuming it is 1/3rd of distance b/w shoulders and pelvis
landmarks.add(getSpinePoint(originalLandmarks, 1 / 3f));
// head top
landmarks.add(getHeadTop(originalLandmarks));
return ImmutableList.copyOf(landmarks);
}
public ImmutableList<LandmarkProto.Landmark> multiPoseLandmarks() {
return getAdditionalLandmarksByInterpolation(multiPoseLandmarks);
}
public static Builder builder() {
return new AutoBuilder_PoseTrackingResult_Builder();
}
/**
* Builder for {@link PoseTrackingResult}.
*/
@AutoBuilder
public abstract static class Builder {
abstract Builder setMultiPoseDetections(List<Detection> value);
abstract Builder setMultiPoseLandmarks(List<LandmarkProto.Landmark> value);
abstract Builder setTimestamp(long value);
abstract Builder setImagePacket(Packet value);
abstract PoseTrackingResult build();
}
}

44
mediapipe/java/com/google/mediapipe/solutions/posetracking/PoseTrackingResultGlRenderer.java
View File

@ -16,13 +16,17 @@ package com.google.mediapipe.solutions.posetracking;
import android.opengl.GLES20;
import com.google.common.collect.ImmutableList;
import com.google.mediapipe.formats.proto.DetectionProto.Detection;
import com.google.mediapipe.formats.proto.LandmarkProto;
import com.google.mediapipe.solutioncore.ResultGlRenderer;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/** A custom implementation of {@link ResultGlRenderer} to render {@link PoseTrackingResult}. */
@ -52,7 +56,10 @@ public class PoseTrackingResultGlRenderer implements ResultGlRenderer<PoseTracki
private int pointSizeHandle;
private int projectionMatrixHandle;
private int colorHandle;
private boolean areLandmarksVisible= true;
public void setLandmarksVisibility(boolean value){
areLandmarksVisible = value;
}
private int loadShader(int type, String shaderCode) {
int shader = GLES20.glCreateShader(type);
GLES20.glShaderSource(shader, shaderCode);
@ -79,17 +86,36 @@ public class PoseTrackingResultGlRenderer implements ResultGlRenderer<PoseTracki
* **/
@Override
public void renderResult(PoseTrackingResult result, float[] projectionMatrix) {
if (result == null) {
if (result == null || !areLandmarksVisible) {
return;
}
GLES20.glUseProgram(program);
GLES20.glUniformMatrix4fv(projectionMatrixHandle, 1, false, projectionMatrix, 0);
// GLES20.glUniform1f(pointSizeHandle, KEYPOINT_SIZE);
// int numDetectedFaces = result.multiPoseTrackings().size();
// for (int i = 0; i < numDetectedFaces; ++i) {
// drawDetection(result.multiPoseTrackings().get(i));
// }
GLES20.glUniform1f(pointSizeHandle, KEYPOINT_SIZE);
ImmutableList<LandmarkProto.Landmark> originalLandmarks = result.multiPoseLandmarks();
List<LandmarkProto.Landmark> landmarks = originalLandmarks.stream().filter((landmark -> {
return landmark.getVisibility() > 0.25 || landmark.getPresence()>0.25;
})).collect(Collectors.toList());
// Draw keypoints.
float[] points = new float[landmarks.size() * 2];
for (int i = 0; i < landmarks.size(); ++i) {
points[2 * i] = landmarks.get(i).getX();
points[2 * i + 1] = 1-landmarks.get(i).getY();
}
GLES20.glUniform4fv(colorHandle, 1, KEYPOINT_COLOR, 0);
FloatBuffer vertexBuffer =
ByteBuffer.allocateDirect(points.length * 4)
.order(ByteOrder.nativeOrder())
.asFloatBuffer()
.put(points);
vertexBuffer.position(0);
GLES20.glEnableVertexAttribArray(positionHandle);
GLES20.glVertexAttribPointer(positionHandle, 2, GLES20.GL_FLOAT, false, 0, vertexBuffer);
GLES20.glDrawArrays(GLES20.GL_POINTS, 0, landmarks.size());
}
/**
@ -100,6 +126,8 @@ public class PoseTrackingResultGlRenderer implements ResultGlRenderer<PoseTracki
public void release() {
GLES20.glDeleteProgram(program);
}
/**
* Not needed anymore, to be cleaned
* */