From 299aa03302d66d1ed449eaf10e01702b633538ac Mon Sep 17 00:00:00 2001 From: Jiuqiang Tang Date: Thu, 15 Dec 2022 09:20:22 -0800 Subject: [PATCH] Internal change PiperOrigin-RevId: 495613573 --- .../audioclassifier/AudioClassifier.java | 2 + .../audio/audioembedder/AudioEmbedder.java | 2 + .../com/google/mediapipe/tasks/core/BUILD | 12 +++ .../google/mediapipe/tasks/core/TaskInfo.java | 12 ++- .../mediapipe/tasks/core/TaskRunner.java | 29 +++++- .../core/logging/TasksStatsDummyLogger.java | 78 +++++++++++++++ .../tasks/core/logging/TasksStatsLogger.java | 98 +++++++++++++++++++ .../text/textclassifier/TextClassifier.java | 1 + .../tasks/text/textembedder/TextEmbedder.java | 1 + .../gesturerecognizer/GestureRecognizer.java | 2 + .../vision/handlandmarker/HandLandmarker.java | 2 + .../imageclassifier/ImageClassifier.java | 2 + .../vision/imageembedder/ImageEmbedder.java | 2 + .../vision/objectdetector/ObjectDetector.java | 2 + 14 files changed, 239 insertions(+), 6 deletions(-) create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/core/logging/TasksStatsDummyLogger.java create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/core/logging/TasksStatsLogger.java diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioclassifier/AudioClassifier.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioclassifier/AudioClassifier.java index d78685fe3..4e5cd7655 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioclassifier/AudioClassifier.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioclassifier/AudioClassifier.java @@ -203,6 +203,8 @@ public final class AudioClassifier extends BaseAudioTaskApi { TaskRunner.create( context, TaskInfo.builder() + .setTaskName(AudioClassifier.class.getSimpleName()) + .setTaskRunningModeName(options.runningMode().name()) .setTaskGraphName(TASK_GRAPH_NAME) .setInputStreams(INPUT_STREAMS) .setOutputStreams(OUTPUT_STREAMS) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioembedder/AudioEmbedder.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioembedder/AudioEmbedder.java index 4bc505d84..077f28ca2 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioembedder/AudioEmbedder.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/audio/audioembedder/AudioEmbedder.java @@ -200,6 +200,8 @@ public final class AudioEmbedder extends BaseAudioTaskApi { TaskRunner.create( context, TaskInfo.builder() + .setTaskName(AudioEmbedder.class.getSimpleName()) + .setTaskRunningModeName(options.runningMode().name()) .setTaskGraphName(TASK_GRAPH_NAME) .setInputStreams(INPUT_STREAMS) .setOutputStreams(OUTPUT_STREAMS) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD index 31f885267..3eb28d38b 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/BUILD @@ -22,6 +22,7 @@ android_library( ], manifest = "AndroidManifest.xml", deps = [ + ":logging", "//mediapipe/calculators/core:flow_limiter_calculator_java_proto_lite", "//mediapipe/calculators/tensor:inference_calculator_java_proto_lite", "//mediapipe/framework:calculator_java_proto_lite", @@ -37,6 +38,17 @@ android_library( ], ) +android_library( + name = "logging", + srcs = glob( + ["logging/*.java"], + ), + deps = [ + "//third_party:autovalue", + "@maven//:com_google_guava_guava", + ], +) + load("//mediapipe/tasks/java/com/google/mediapipe/tasks:mediapipe_tasks_aar.bzl", "mediapipe_tasks_core_aar") mediapipe_tasks_core_aar( diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskInfo.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskInfo.java index 12f8be8ba..310f5739c 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskInfo.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskInfo.java @@ -32,6 +32,12 @@ public abstract class TaskInfo { /** Builder for {@link TaskInfo}. */ @AutoValue.Builder public abstract static class Builder { + /** Sets the MediaPipe task name. */ + public abstract Builder setTaskName(String value); + + /** Sets the MediaPipe task running mode name. */ + public abstract Builder setTaskRunningModeName(String value); + /** Sets the MediaPipe task graph name. */ public abstract Builder setTaskGraphName(String value); @@ -71,6 +77,10 @@ public abstract class TaskInfo { } } + abstract String taskName(); + + abstract String taskRunningModeName(); + abstract String taskGraphName(); abstract T taskOptions(); @@ -82,7 +92,7 @@ public abstract class TaskInfo { abstract Boolean enableFlowLimiting(); public static Builder builder() { - return new AutoValue_TaskInfo.Builder(); + return new AutoValue_TaskInfo.Builder().setTaskName("").setTaskRunningModeName(""); } /* Returns a list of the output stream names without the stream tags. */ diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskRunner.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskRunner.java index e6fc91cf6..1a128c538 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskRunner.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/TaskRunner.java @@ -21,6 +21,8 @@ import com.google.mediapipe.framework.AndroidPacketCreator; import com.google.mediapipe.framework.Graph; import com.google.mediapipe.framework.MediaPipeException; import com.google.mediapipe.framework.Packet; +import com.google.mediapipe.tasks.core.logging.TasksStatsLogger; +import com.google.mediapipe.tasks.core.logging.TasksStatsDummyLogger; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -34,6 +36,7 @@ public class TaskRunner implements AutoCloseable { private final Graph graph; private final ModelResourcesCache modelResourcesCache; private final AndroidPacketCreator packetCreator; + private final TasksStatsLogger statsLogger; private long lastSeenTimestamp = Long.MIN_VALUE; private ErrorListener errorListener; @@ -51,6 +54,8 @@ public class TaskRunner implements AutoCloseable { Context context, TaskInfo taskInfo, OutputHandler outputHandler) { + TasksStatsLogger statsLogger = + TasksStatsDummyLogger.create(context, taskInfo.taskName(), taskInfo.taskRunningModeName()); AndroidAssetUtil.initializeNativeAssetManager(context); Graph mediapipeGraph = new Graph(); mediapipeGraph.loadBinaryGraph(taskInfo.generateGraphConfig()); @@ -58,12 +63,15 @@ public class TaskRunner implements AutoCloseable { mediapipeGraph.setServiceObject(new ModelResourcesCacheService(), graphModelResourcesCache); mediapipeGraph.addMultiStreamCallback( taskInfo.outputStreamNames(), - outputHandler::run, - /*observeTimestampBounds=*/ outputHandler.handleTimestampBoundChanges()); + packets -> { + outputHandler.run(packets); + statsLogger.recordInvocationEnd(packets.get(0).getTimestamp()); + }, + /* observeTimestampBounds= */ outputHandler.handleTimestampBoundChanges()); mediapipeGraph.startRunningGraph(); // Waits until all calculators are opened and the graph is fully started. mediapipeGraph.waitUntilGraphIdle(); - return new TaskRunner(mediapipeGraph, graphModelResourcesCache, outputHandler); + return new TaskRunner(mediapipeGraph, graphModelResourcesCache, outputHandler, statsLogger); } /** @@ -91,7 +99,10 @@ public class TaskRunner implements AutoCloseable { * @param inputs a map contains (input stream {@link String}, data {@link Packet}) pairs. */ public synchronized TaskResult process(Map inputs) { - addPackets(inputs, generateSyntheticTimestamp()); + long syntheticInputTimestamp = generateSyntheticTimestamp(); + // TODO: Support recording GPU input arrival. + statsLogger.recordCpuInputArrival(syntheticInputTimestamp); + addPackets(inputs, syntheticInputTimestamp); graph.waitUntilGraphIdle(); lastSeenTimestamp = outputHandler.getLatestOutputTimestamp(); return outputHandler.retrieveCachedTaskResult(); @@ -112,6 +123,7 @@ public class TaskRunner implements AutoCloseable { */ public synchronized TaskResult process(Map inputs, long inputTimestamp) { validateInputTimstamp(inputTimestamp); + statsLogger.recordCpuInputArrival(inputTimestamp); addPackets(inputs, inputTimestamp); graph.waitUntilGraphIdle(); return outputHandler.retrieveCachedTaskResult(); @@ -132,6 +144,7 @@ public class TaskRunner implements AutoCloseable { */ public synchronized void send(Map inputs, long inputTimestamp) { validateInputTimstamp(inputTimestamp); + statsLogger.recordCpuInputArrival(inputTimestamp); addPackets(inputs, inputTimestamp); } @@ -145,6 +158,7 @@ public class TaskRunner implements AutoCloseable { graphStarted.set(false); graph.closeAllPacketSources(); graph.waitUntilGraphDone(); + statsLogger.logSessionEnd(); } catch (MediaPipeException e) { reportError(e); } @@ -154,6 +168,7 @@ public class TaskRunner implements AutoCloseable { // Waits until all calculators are opened and the graph is fully restarted. graph.waitUntilGraphIdle(); graphStarted.set(true); + statsLogger.logSessionStart(); } catch (MediaPipeException e) { reportError(e); } @@ -169,6 +184,7 @@ public class TaskRunner implements AutoCloseable { graphStarted.set(false); graph.closeAllPacketSources(); graph.waitUntilGraphDone(); + statsLogger.logSessionEnd(); if (modelResourcesCache != null) { modelResourcesCache.release(); } @@ -247,12 +263,15 @@ public class TaskRunner implements AutoCloseable { private TaskRunner( Graph graph, ModelResourcesCache modelResourcesCache, - OutputHandler outputHandler) { + OutputHandler outputHandler, + TasksStatsLogger statsLogger) { this.outputHandler = outputHandler; this.graph = graph; this.modelResourcesCache = modelResourcesCache; this.packetCreator = new AndroidPacketCreator(graph); + this.statsLogger = statsLogger; graphStarted.set(true); + this.statsLogger.logSessionStart(); } /** Reports error. */ diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/logging/TasksStatsDummyLogger.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/logging/TasksStatsDummyLogger.java new file mode 100644 index 000000000..c10b5d224 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/logging/TasksStatsDummyLogger.java @@ -0,0 +1,78 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.core.logging; + +import android.content.Context; + +/** A dummy MediaPipe Tasks stats logger that has all methods as no-ops. */ +public class TasksStatsDummyLogger implements TasksStatsLogger { + + /** + * Creates the MediaPipe Tasks stats dummy logger. + * + * @param context a {@link Context}. + * @param taskNameStr the task api name. + * @param taskRunningModeStr the task running mode string representation. + */ + public static TasksStatsDummyLogger create( + Context context, String taskNameStr, String taskRunningModeStr) { + return new TasksStatsDummyLogger(); + } + + private TasksStatsDummyLogger() {} + + /** Logs the start of a MediaPipe Tasks API session. */ + @Override + public void logSessionStart() {} + + /** + * Records MediaPipe Tasks API receiving CPU input data. + * + * @param packetTimestamp the input packet timestamp that acts as the identifier of the api + * invocation. + */ + @Override + public void recordCpuInputArrival(long packetTimestamp) {} + + /** + * Records MediaPipe Tasks API receiving GPU input data. + * + * @param packetTimestamp the input packet timestamp that acts as the identifier of the api + * invocation. + */ + @Override + public void recordGpuInputArrival(long packetTimestamp) {} + + /** + * Records the end of a Mediapipe Tasks API invocation. + * + * @param packetTimestamp the output packet timestamp that acts as the identifier of the api + * invocation. + */ + @Override + public void recordInvocationEnd(long packetTimestamp) {} + + /** Logs the MediaPipe Tasks API periodic invocation report. */ + @Override + public void logInvocationReport(StatsSnapshot stats) {} + + /** Logs the Tasks API session end event. */ + @Override + public void logSessionEnd() {} + + /** Logs the MediaPipe Tasks API initialization error. */ + @Override + public void logInitError() {} +} diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/core/logging/TasksStatsLogger.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/logging/TasksStatsLogger.java new file mode 100644 index 000000000..c726e7d0d --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/core/logging/TasksStatsLogger.java @@ -0,0 +1,98 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.core.logging; + +import com.google.auto.value.AutoValue; + +/** The stats logger interface that defines what MediaPipe Tasks events to log. */ +public interface TasksStatsLogger { + /** Task stats snapshot. */ + @AutoValue + abstract static class StatsSnapshot { + static StatsSnapshot create( + int cpuInputCount, + int gpuInputCount, + int finishedCount, + int droppedCount, + long totalLatencyMs, + long peakLatencyMs, + long elapsedTimeMs) { + return new AutoValue_TasksStatsLogger_StatsSnapshot( + cpuInputCount, + gpuInputCount, + finishedCount, + droppedCount, + totalLatencyMs, + peakLatencyMs, + elapsedTimeMs); + } + + static StatsSnapshot createDefault() { + return new AutoValue_TasksStatsLogger_StatsSnapshot(0, 0, 0, 0, 0, 0, 0); + } + + abstract int cpuInputCount(); + + abstract int gpuInputCount(); + + abstract int finishedCount(); + + abstract int droppedCount(); + + abstract long totalLatencyMs(); + + abstract long peakLatencyMs(); + + abstract long elapsedTimeMs(); + } + + /** Logs the start of a MediaPipe Tasks API session. */ + public void logSessionStart(); + + /** + * Records MediaPipe Tasks API receiving CPU input data. + * + * @param packetTimestamp the input packet timestamp that acts as the identifier of the api + * invocation. + */ + public void recordCpuInputArrival(long packetTimestamp); + + /** + * Records MediaPipe Tasks API receiving GPU input data. + * + * @param packetTimestamp the input packet timestamp that acts as the identifier of the api + * invocation. + */ + public void recordGpuInputArrival(long packetTimestamp); + + /** + * Records the end of a Mediapipe Tasks API invocation. + * + * @param packetTimestamp the output packet timestamp that acts as the identifier of the api + * invocation. + */ + public void recordInvocationEnd(long packetTimestamp); + + /** Logs the MediaPipe Tasks API periodic invocation report. */ + public void logInvocationReport(StatsSnapshot stats); + + /** Logs the Tasks API session end event. */ + public void logSessionEnd(); + + /** Logs the MediaPipe Tasks API initialization error. */ + public void logInitError(); + + // TODO: Logs more error types. +} diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/text/textclassifier/TextClassifier.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/text/textclassifier/TextClassifier.java index 0ea91a9f8..edb78a191 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/text/textclassifier/TextClassifier.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/text/textclassifier/TextClassifier.java @@ -169,6 +169,7 @@ public final class TextClassifier implements AutoCloseable { TaskRunner.create( context, TaskInfo.builder() + .setTaskName(TextClassifier.class.getSimpleName()) .setTaskGraphName(TASK_GRAPH_NAME) .setInputStreams(INPUT_STREAMS) .setOutputStreams(OUTPUT_STREAMS) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/text/textembedder/TextEmbedder.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/text/textembedder/TextEmbedder.java index 9b464d0e8..28f351d4b 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/text/textembedder/TextEmbedder.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/text/textembedder/TextEmbedder.java @@ -159,6 +159,7 @@ public final class TextEmbedder implements AutoCloseable { TaskRunner.create( context, TaskInfo.builder() + .setTaskName(TextEmbedder.class.getSimpleName()) .setTaskGraphName(TASK_GRAPH_NAME) .setInputStreams(INPUT_STREAMS) .setOutputStreams(OUTPUT_STREAMS) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java index e9e74a067..a933d2f65 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizer.java @@ -194,6 +194,8 @@ public final class GestureRecognizer extends BaseVisionTaskApi { TaskRunner.create( context, TaskInfo.builder() + .setTaskName(GestureRecognizer.class.getSimpleName()) + .setTaskRunningModeName(recognizerOptions.runningMode().name()) .setTaskGraphName(TASK_GRAPH_NAME) .setInputStreams(INPUT_STREAMS) .setOutputStreams(OUTPUT_STREAMS) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarker.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarker.java index a9270d347..1d08ab928 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarker.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarker.java @@ -183,6 +183,8 @@ public final class HandLandmarker extends BaseVisionTaskApi { TaskRunner.create( context, TaskInfo.builder() + .setTaskName(HandLandmarker.class.getSimpleName()) + .setTaskRunningModeName(landmarkerOptions.runningMode().name()) .setTaskGraphName(TASK_GRAPH_NAME) .setInputStreams(INPUT_STREAMS) .setOutputStreams(OUTPUT_STREAMS) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imageclassifier/ImageClassifier.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imageclassifier/ImageClassifier.java index 8990f46fd..38482797c 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imageclassifier/ImageClassifier.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imageclassifier/ImageClassifier.java @@ -197,6 +197,8 @@ public final class ImageClassifier extends BaseVisionTaskApi { TaskRunner.create( context, TaskInfo.builder() + .setTaskName(ImageClassifier.class.getSimpleName()) + .setTaskRunningModeName(options.runningMode().name()) .setTaskGraphName(TASK_GRAPH_NAME) .setInputStreams(INPUT_STREAMS) .setOutputStreams(OUTPUT_STREAMS) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imageembedder/ImageEmbedder.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imageembedder/ImageEmbedder.java index af053d860..488927257 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imageembedder/ImageEmbedder.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imageembedder/ImageEmbedder.java @@ -180,6 +180,8 @@ public final class ImageEmbedder extends BaseVisionTaskApi { TaskRunner.create( context, TaskInfo.builder() + .setTaskName(ImageEmbedder.class.getSimpleName()) + .setTaskRunningModeName(options.runningMode().name()) .setTaskGraphName(TASK_GRAPH_NAME) .setInputStreams(INPUT_STREAMS) .setOutputStreams(OUTPUT_STREAMS) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetector.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetector.java index 769b9137f..d706189ee 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetector.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetector.java @@ -190,6 +190,8 @@ public final class ObjectDetector extends BaseVisionTaskApi { TaskRunner.create( context, TaskInfo.builder() + .setTaskName(ObjectDetector.class.getSimpleName()) + .setTaskRunningModeName(detectorOptions.runningMode().name()) .setTaskGraphName(TASK_GRAPH_NAME) .setInputStreams(INPUT_STREAMS) .setOutputStreams(OUTPUT_STREAMS)