diff --git a/mediapipe/tasks/python/benchmark/__init__.py b/mediapipe/tasks/python/benchmark/__init__.py new file mode 100644 index 000000000..3fbc1e8a7 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/mediapipe/tasks/python/benchmark/vision/__init__.py b/mediapipe/tasks/python/benchmark/vision/__init__.py new file mode 100644 index 000000000..3fbc1e8a7 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md new file mode 100644 index 000000000..0c3e23432 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md @@ -0,0 +1,34 @@ +# MediaPipe Image Classifier Benchmark + +## Download the repository + +First, clone this Git repo. + +Run this script to install the required dependencies and download the TFLite models: + +``` +cd mediapipe/tasks/python/benchmark/vision/image_classifier +sh setup.sh +``` + +## Run the benchmark +``` +python3 image_classifier_benchmark.py +``` +* You can optionally specify the `model` parameter to set the TensorFlow Lite + model to be used: + * The default value is `classifier.tflite` + * TensorFlow Lite image classification models **with metadata** + * Models from [TensorFlow Hub](https://tfhub.dev/tensorflow/collections/lite/task-library/image-classifier/1) + * Models from [MediaPipe Models](https://developers.google.com/mediapipe/solutions/vision/image_classifier/index#models) + * Models trained with [MediaPipe Model Maker](https://developers.google.com/mediapipe/solutions/customization/image_classifier) are supported. +* You can optionally specify the `iterations` parameter to limit the number of + iterations for benchmarking: + * Supported value: A positive integer. + * Default value: `100` +* Example usage: + ``` + python3 image_classifier_benchmark.py \ + --model classifier.tflite \ + --iterations 200 + ``` diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py new file mode 100644 index 000000000..c6a80da11 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -0,0 +1,74 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Benchmark for the image classifier task.""" +import argparse +import time +import numpy as np +import mediapipe as mp +from mediapipe.tasks import python +from mediapipe.tasks.python import vision + +_IMAGE_FILE = 'burger.jpg' + + +def run(model: str, n_iterations: int, delegate: python.BaseOptions.Delegate): + """Run asynchronous inference on images and benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + """ + inference_times = [] + + # Initialize the image classifier + base_options = python.BaseOptions(model_asset_path=model, delegate=delegate) + options = vision.ImageClassifierOptions( + base_options=base_options, running_mode=vision.RunningMode.IMAGE, + max_results=1) + classifier = vision.ImageClassifier.create_from_options(options) + mp_image = mp.Image.create_from_file(_IMAGE_FILE) + + for _ in range(n_iterations): + start_time_ns = time.time_ns() + classifier.classify(mp_image) + end_time_ns = time.time_ns() + # Convert to milliseconds + inference_times.append((end_time_ns - start_time_ns) / 1_000_000) + + classifier.close() + return np.percentile(inference_times, 95) + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + '--model', help='Path to image classification model.', required=True) + parser.add_argument( + '--iterations', help='Number of iterations for benchmarking.', type=int, + default=100) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.CPU) + print(f"95th Percentile Inference Time on CPU: {cpu_time:.6f} milliseconds") + + # Run benchmark on GPU + gpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.GPU) + print(f"95th Percentile Inference Time on GPU: {gpu_time:.6f} milliseconds") + + +if __name__ == '__main__': + main() diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/setup.sh b/mediapipe/tasks/python/benchmark/vision/image_classifier/setup.sh new file mode 100644 index 000000000..79e35f447 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/setup.sh @@ -0,0 +1,6 @@ +# Install Python dependencies. +python3 -m pip install pip --upgrade +python3 -m pip install mediapipe + +wget -O classifier.tflite -q https://storage.googleapis.com/mediapipe-models/image_classifier/efficientnet_lite0/float32/1/efficientnet_lite0.tflite +wget -O burger.jpg https://storage.googleapis.com/mediapipe-assets/burger.jpg