From 35f2f36733f6f80618a4ff4b8c57e43ee1fb4e9d Mon Sep 17 00:00:00 2001 From: Kinar Date: Sat, 11 Nov 2023 03:25:36 -0800 Subject: [PATCH 1/6] Added image classifier benchmark --- mediapipe/tasks/python/benchmark/__init__.py | 13 ++++ .../tasks/python/benchmark/vision/__init__.py | 13 ++++ .../vision/image_classifier/README.md | 34 +++++++++ .../image_classifier_benchmark.py | 74 +++++++++++++++++++ .../vision/image_classifier/setup.sh | 6 ++ 5 files changed, 140 insertions(+) create mode 100644 mediapipe/tasks/python/benchmark/__init__.py create mode 100644 mediapipe/tasks/python/benchmark/vision/__init__.py create mode 100644 mediapipe/tasks/python/benchmark/vision/image_classifier/README.md create mode 100644 mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py create mode 100644 mediapipe/tasks/python/benchmark/vision/image_classifier/setup.sh diff --git a/mediapipe/tasks/python/benchmark/__init__.py b/mediapipe/tasks/python/benchmark/__init__.py new file mode 100644 index 000000000..3fbc1e8a7 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/mediapipe/tasks/python/benchmark/vision/__init__.py b/mediapipe/tasks/python/benchmark/vision/__init__.py new file mode 100644 index 000000000..3fbc1e8a7 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md new file mode 100644 index 000000000..0c3e23432 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md @@ -0,0 +1,34 @@ +# MediaPipe Image Classifier Benchmark + +## Download the repository + +First, clone this Git repo. + +Run this script to install the required dependencies and download the TFLite models: + +``` +cd mediapipe/tasks/python/benchmark/vision/image_classifier +sh setup.sh +``` + +## Run the benchmark +``` +python3 image_classifier_benchmark.py +``` +* You can optionally specify the `model` parameter to set the TensorFlow Lite + model to be used: + * The default value is `classifier.tflite` + * TensorFlow Lite image classification models **with metadata** + * Models from [TensorFlow Hub](https://tfhub.dev/tensorflow/collections/lite/task-library/image-classifier/1) + * Models from [MediaPipe Models](https://developers.google.com/mediapipe/solutions/vision/image_classifier/index#models) + * Models trained with [MediaPipe Model Maker](https://developers.google.com/mediapipe/solutions/customization/image_classifier) are supported. +* You can optionally specify the `iterations` parameter to limit the number of + iterations for benchmarking: + * Supported value: A positive integer. + * Default value: `100` +* Example usage: + ``` + python3 image_classifier_benchmark.py \ + --model classifier.tflite \ + --iterations 200 + ``` diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py new file mode 100644 index 000000000..c6a80da11 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -0,0 +1,74 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Benchmark for the image classifier task.""" +import argparse +import time +import numpy as np +import mediapipe as mp +from mediapipe.tasks import python +from mediapipe.tasks.python import vision + +_IMAGE_FILE = 'burger.jpg' + + +def run(model: str, n_iterations: int, delegate: python.BaseOptions.Delegate): + """Run asynchronous inference on images and benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + """ + inference_times = [] + + # Initialize the image classifier + base_options = python.BaseOptions(model_asset_path=model, delegate=delegate) + options = vision.ImageClassifierOptions( + base_options=base_options, running_mode=vision.RunningMode.IMAGE, + max_results=1) + classifier = vision.ImageClassifier.create_from_options(options) + mp_image = mp.Image.create_from_file(_IMAGE_FILE) + + for _ in range(n_iterations): + start_time_ns = time.time_ns() + classifier.classify(mp_image) + end_time_ns = time.time_ns() + # Convert to milliseconds + inference_times.append((end_time_ns - start_time_ns) / 1_000_000) + + classifier.close() + return np.percentile(inference_times, 95) + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + '--model', help='Path to image classification model.', required=True) + parser.add_argument( + '--iterations', help='Number of iterations for benchmarking.', type=int, + default=100) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.CPU) + print(f"95th Percentile Inference Time on CPU: {cpu_time:.6f} milliseconds") + + # Run benchmark on GPU + gpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.GPU) + print(f"95th Percentile Inference Time on GPU: {gpu_time:.6f} milliseconds") + + +if __name__ == '__main__': + main() diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/setup.sh b/mediapipe/tasks/python/benchmark/vision/image_classifier/setup.sh new file mode 100644 index 000000000..79e35f447 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/setup.sh @@ -0,0 +1,6 @@ +# Install Python dependencies. +python3 -m pip install pip --upgrade +python3 -m pip install mediapipe + +wget -O classifier.tflite -q https://storage.googleapis.com/mediapipe-models/image_classifier/efficientnet_lite0/float32/1/efficientnet_lite0.tflite +wget -O burger.jpg https://storage.googleapis.com/mediapipe-assets/burger.jpg From 021c7edde74c0096372091d526db8dcf9c48bb14 Mon Sep 17 00:00:00 2001 From: Kinar Date: Sat, 11 Nov 2023 03:32:45 -0800 Subject: [PATCH 2/6] Updated README and script --- .../tasks/python/benchmark/vision/image_classifier/README.md | 2 +- .../vision/image_classifier/image_classifier_benchmark.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md index 0c3e23432..d7db3559d 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md @@ -7,7 +7,7 @@ First, clone this Git repo. Run this script to install the required dependencies and download the TFLite models: ``` -cd mediapipe/tasks/python/benchmark/vision/image_classifier +cd mediapipe/mediapipe/tasks/python/benchmark/vision/image_classifier sh setup.sh ``` diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py index c6a80da11..3fc7c8a2e 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -55,7 +55,8 @@ def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - '--model', help='Path to image classification model.', required=True) + '--model', help='Path to image classification model.', required=True, + default='classifier.tflite') parser.add_argument( '--iterations', help='Number of iterations for benchmarking.', type=int, default=100) From 99c8b9ee3c79ab5d5b367811bd30a80807283b3b Mon Sep 17 00:00:00 2001 From: Kinar Date: Sat, 11 Nov 2023 03:34:26 -0800 Subject: [PATCH 3/6] Updated copyright --- mediapipe/tasks/python/benchmark/__init__.py | 2 +- mediapipe/tasks/python/benchmark/vision/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/python/benchmark/__init__.py b/mediapipe/tasks/python/benchmark/__init__.py index 3fbc1e8a7..2eb077987 100644 --- a/mediapipe/tasks/python/benchmark/__init__.py +++ b/mediapipe/tasks/python/benchmark/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. +# Copyright 2023 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/mediapipe/tasks/python/benchmark/vision/__init__.py b/mediapipe/tasks/python/benchmark/vision/__init__.py index 3fbc1e8a7..2eb077987 100644 --- a/mediapipe/tasks/python/benchmark/vision/__init__.py +++ b/mediapipe/tasks/python/benchmark/vision/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. +# Copyright 2023 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 38737849e6407fccc771d88f34bbaf6a5d5f5a16 Mon Sep 17 00:00:00 2001 From: Kinar Date: Sat, 11 Nov 2023 03:34:57 -0800 Subject: [PATCH 4/6] Updated copyright --- .../vision/image_classifier/image_classifier_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py index 3fc7c8a2e..502441879 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. +# Copyright 2023 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From f8add5ad420798ff0ae023afed5bc0a8b8abed83 Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 13 Nov 2023 21:17:28 -0800 Subject: [PATCH 5/6] Documented the return value and added percentile to argparser --- .../image_classifier_benchmark.py | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py index 502441879..d505a8fe8 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -1,17 +1,3 @@ -# Copyright 2023 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Benchmark for the image classifier task.""" import argparse import time import numpy as np @@ -22,16 +8,21 @@ from mediapipe.tasks.python import vision _IMAGE_FILE = 'burger.jpg' -def run(model: str, n_iterations: int, delegate: python.BaseOptions.Delegate): +def run(model: str, n_iterations: int, delegate: python.BaseOptions.Delegate, + percentile: float): """Run asynchronous inference on images and benchmark. Args: model: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + Returns: + The n-th percentile of the inference times. """ inference_times = [] - + # Initialize the image classifier base_options = python.BaseOptions(model_asset_path=model, delegate=delegate) options = vision.ImageClassifierOptions( @@ -48,7 +39,7 @@ def run(model: str, n_iterations: int, delegate: python.BaseOptions.Delegate): inference_times.append((end_time_ns - start_time_ns) / 1_000_000) classifier.close() - return np.percentile(inference_times, 95) + return np.percentile(inference_times, percentile) def main(): @@ -60,15 +51,22 @@ def main(): parser.add_argument( '--iterations', help='Number of iterations for benchmarking.', type=int, default=100) + parser.add_argument( + '--percentile', help='Percentile for benchmarking statistics.', + type=float, default=95.0) args = parser.parse_args() # Run benchmark on CPU - cpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.CPU) - print(f"95th Percentile Inference Time on CPU: {cpu_time:.6f} milliseconds") + cpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.CPU, + args.percentile) + print(f"{args.percentile}th Percentile Inference Time on CPU: " + f"{cpu_time:.6f} milliseconds") # Run benchmark on GPU - gpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.GPU) - print(f"95th Percentile Inference Time on GPU: {gpu_time:.6f} milliseconds") + gpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.GPU, + args.percentile) + print(f"{args.percentile}th Percentile Inference Time on GPU: " + f"{gpu_time:.6f} milliseconds") if __name__ == '__main__': From 252cca72e77c13e5f284404a6b4c2b52e08369ca Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 13 Nov 2023 21:44:27 -0800 Subject: [PATCH 6/6] Allowed a default value for the model argument --- .../vision/image_classifier/image_classifier_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py index d505a8fe8..65a629bab 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -46,7 +46,7 @@ def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - '--model', help='Path to image classification model.', required=True, + '--model', help='Path to image classification model.', required=False, default='classifier.tflite') parser.add_argument( '--iterations', help='Number of iterations for benchmarking.', type=int,