From dd6663b818214dc3efd0949990af2053753cf73b Mon Sep 17 00:00:00 2001 From: Anh Date: Thu, 23 Nov 2023 17:54:17 +0700 Subject: [PATCH 01/20] add new tasks for MobileNetEdgeTPUv2 model --- .github/cloudbuild/android-test.yaml | 2 +- flutter/assets/tasks.pbtxt | 74 ++++++++++++++++++- .../integration_test/expected_accuracy.dart | 9 +++ .../integration_test/expected_throughput.dart | 38 ++++++++++ flutter/integration_test/first_test.dart | 9 ++- flutter/lib/app_constants.dart | 8 +- flutter/lib/benchmark/info.dart | 5 ++ .../cpp/backend_coreml/coreml_settings.pbtxt | 19 +++++ mobile_back_apple/dev-utils/Makefile | 56 +++++++++++++- mobile_back_apple/models/icv2.py | 52 +++++++++++++ .../settings/tflite_settings_pixel6.pbtxt | 19 +++++ .../tflite_settings_android.pbtxt | 42 +++++++++++ .../tflite_settings_apple_iphone11.pbtxt | 22 ++++++ .../tflite_settings_apple_iphone12.pbtxt | 22 ++++++ .../tflite_settings_apple_iphoneX.pbtxt | 22 ++++++ .../tflite_settings_apple_main.pbtxt | 20 +++++ .../tflite_settings_windows.pbtxt | 24 ++++++ 17 files changed, 431 insertions(+), 12 deletions(-) create mode 100644 mobile_back_apple/models/icv2.py diff --git a/.github/cloudbuild/android-test.yaml b/.github/cloudbuild/android-test.yaml index e438252e8..c55ffab9e 100644 --- a/.github/cloudbuild/android-test.yaml +++ b/.github/cloudbuild/android-test.yaml @@ -27,7 +27,7 @@ steps: --type instrumentation \ --app output/apks/$_APK_TEST_MAIN_NAME \ --test output/apks/$_APK_TEST_HELPER_NAME \ - --timeout 25m \ + --timeout 30m \ --num-flaky-test-attempts 2 \ --device $_DEVICE_SELECTOR diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt index c96506ce2..5a113cad9 100644 --- a/flutter/assets/tasks.pbtxt +++ b/flutter/assets/tasks.pbtxt @@ -77,7 +77,7 @@ task { task { id: "image_segmentation_v2" - name: "v2.0 Image Segmentation" + name: "Image Segmentation v2" min_query_count: 1024 min_duration: 60 max_duration: 600 @@ -215,3 +215,75 @@ task { num_classes: 1000 } } + +task { + id: "image_classification_v2" + name: "Image Classification v2" + min_query_count: 1024 + min_duration: 60 + max_throughput: 1000 + max_accuracy: 1.0 + scenario: "SingleStream" + datasets { + type: IMAGENET + full { + name: "Imagenet classification validation set" + input_path: "local:///mlperf_datasets/imagenet/img" + groundtruth_path: "local:///mlperf_datasets/imagenet/imagenet_val_full.txt" + } + lite { + name: "Open images subset for classification" + input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet.zip" + groundtruth_path: "" + } + tiny { + name: "Imagenet dataset for integration test" + input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet_tiny.zip" + groundtruth_path: "https://github.com/mlcommons/mobile_models/raw/main/v3_0/assets/imagenet_tiny-groundtruth.txt" + } + } + model { + id: "MobileNetEdgeTPUv2-L" + name: "MobileNetEdgeTPUv2-L" + offset: 1 + image_width: 224 + image_height: 224 + num_classes: 1000 + } +} + +task { + id: "image_classification_offline_v2" + name: "Image Classification v2 (Offline)" + min_query_count: 24576 + min_duration: 0 + max_throughput: 2000 + max_accuracy: 1.0 + scenario: "Offline" + datasets { + type: IMAGENET + full { + name: "Imagenet classification validation set" + input_path: "local:///mlperf_datasets/imagenet/img" + groundtruth_path: "local:///mlperf_datasets/imagenet/imagenet_val_full.txt" + } + lite { + name: "Open images subset for classification" + input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet.zip" + groundtruth_path: "" + } + tiny { + name: "Imagenet dataset for integration test" + input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet_tiny.zip" + groundtruth_path: "https://github.com/mlcommons/mobile_models/raw/main/v3_0/assets/imagenet_tiny-groundtruth.txt" + } + } + model { + id: "MobileNetEdgeTPUv2-L" + name: "MobileNetEdgeTPUv2-L" + offset: 1 + image_width: 224 + image_height: 224 + num_classes: 1000 + } +} diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart index 7b2589d8c..c67fe8328 100644 --- a/flutter/integration_test/expected_accuracy.dart +++ b/flutter/integration_test/expected_accuracy.dart @@ -20,6 +20,13 @@ const Map _imageClassification = { 'cpu&gpu&ane': Interval(min: 1.00, max: 1.00) }; +const Map _imageClassificationV2 = { + 'cpu': Interval(min: 0.69, max: 0.71), + 'npu': Interval(min: 0.69, max: 0.71), + 'tpu': Interval(min: 0.69, max: 0.71), + 'ane': Interval(min: 0.69, max: 0.71), +}; + const Map _objectDetection = { 'cpu': Interval(min: 0.31, max: 0.32), 'npu': Interval(min: 0.28, max: 0.31), @@ -58,9 +65,11 @@ const Map _superResolution = { const benchmarkExpectedAccuracy = { BenchmarkId.imageClassification: _imageClassification, + BenchmarkId.imageClassificationV2: _imageClassificationV2, BenchmarkId.objectDetection: _objectDetection, BenchmarkId.imageSegmentationV2: _imageSegmentation, BenchmarkId.naturalLanguageProcessing: _naturalLanguageProcessing, BenchmarkId.superResolution: _superResolution, BenchmarkId.imageClassificationOffline: _imageClassification, + BenchmarkId.imageClassificationOfflineV2: _imageClassificationV2, }; diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 0c943e5f4..a0beb9ac9 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -42,6 +42,24 @@ const Map> _imageClassification = { }, }; +// TODO (anhappdev): update the expected value for _imageClassificationV2 after gathering some statistics +const Map> _imageClassificationV2 = { + _kTFLiteBackend: { + _kCloudBuildX23: Interval(min: 2, max: 11), + _kCloudBuildX28: Interval(min: 2, max: 12), + _kRyzen5600: Interval(min: 1, max: 37), + _kPixel5: Interval(min: 60, max: 75), + _kIphoneOnGitHubAction: Interval(min: 1, max: 7), + _kIphoneOnMacbookM1: Interval(min: 10, max: 27), + }, + _kCoreMLBackend: { + _kIphoneOnGitHubAction: Interval(min: 1, max: 7), + }, + _kPixelBackend: { + _kPixel6: Interval(min: 500, max: 600), + }, +}; + const Map> _objectDetection = { _kTFLiteBackend: { _kCloudBuildX23: Interval(min: 4, max: 7), @@ -111,6 +129,24 @@ const Map> _imageClassificationOffline = { }, }; +// TODO (anhappdev): update the expected value for _imageClassificationOfflineV2 after gathering some statistics +const Map> _imageClassificationOfflineV2 = { + _kTFLiteBackend: { + _kCloudBuildX23: Interval(min: 4, max: 14), + _kCloudBuildX28: Interval(min: 3, max: 14), + _kRyzen5600: Interval(min: 20, max: 60), + _kPixel5: Interval(min: 60, max: 180), + _kIphoneOnGitHubAction: Interval(min: 1, max: 15), + _kIphoneOnMacbookM1: Interval(min: 10, max: 45), + }, + _kCoreMLBackend: { + _kIphoneOnGitHubAction: Interval(min: 1, max: 15), + }, + _kPixelBackend: { + _kPixel6: Interval(min: 700, max: 1700), + }, +}; + // TODO (anhappdev): update min throughput for _superResolution after we gather some statistic const Map> _superResolution = { _kTFLiteBackend: { @@ -131,9 +167,11 @@ const Map> _superResolution = { const benchmarkExpectedThroughput = { BenchmarkId.imageClassification: _imageClassification, + BenchmarkId.imageClassificationV2: _imageClassificationV2, BenchmarkId.objectDetection: _objectDetection, BenchmarkId.imageSegmentationV2: _imageSegmentation, BenchmarkId.naturalLanguageProcessing: _naturalLanguageProcessing, BenchmarkId.superResolution: _superResolution, BenchmarkId.imageClassificationOffline: _imageClassificationOffline, + BenchmarkId.imageClassificationOfflineV2: _imageClassificationOfflineV2, }; diff --git a/flutter/integration_test/first_test.dart b/flutter/integration_test/first_test.dart index afa115e8e..b822a7f4e 100644 --- a/flutter/integration_test/first_test.dart +++ b/flutter/integration_test/first_test.dart @@ -1,5 +1,6 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:integration_test/integration_test.dart'; +import 'package:mlperfbench/app_constants.dart'; import 'package:mlperfbench/benchmark/run_mode.dart'; import 'package:mlperfbench/firebase/firebase_manager.dart'; import 'package:mlperfbench/store.dart'; @@ -31,7 +32,7 @@ void main() { if (enablePerfTest) { prefs[StoreConstants.testMinDuration] = 15; prefs[StoreConstants.testMinQueryCount] = 64; - prefs[StoreConstants.testCooldownDuration] = 10; + prefs[StoreConstants.testCooldownDuration] = 2; } SharedPreferences.setMockInitialValues(prefs); @@ -56,10 +57,10 @@ void main() { } void checkTasks(ExtendedResult extendedResult) { - final length = extendedResult.results.length; - const expectedTasksCount = 6; + final tasksCount = extendedResult.results.length; + final expectedTasksCount = BenchmarkId.allIds.length; - expect(length, expectedTasksCount, reason: 'tasks count does not match'); + expect(tasksCount, expectedTasksCount, reason: 'tasks count does not match'); for (final benchmarkResult in extendedResult.results) { print('checking ${benchmarkResult.benchmarkId}'); diff --git a/flutter/lib/app_constants.dart b/flutter/lib/app_constants.dart index cff63e92b..93727572c 100644 --- a/flutter/lib/app_constants.dart +++ b/flutter/lib/app_constants.dart @@ -78,16 +78,20 @@ class BenchmarkId { static const objectDetection = 'object_detection'; static const imageSegmentationV2 = 'image_segmentation_v2'; static const naturalLanguageProcessing = 'natural_language_processing'; - static const imageClassificationOffline = 'image_classification_offline'; static const superResolution = 'super_resolution'; + static const imageClassificationOffline = 'image_classification_offline'; + static const imageClassificationV2 = 'image_classification_v2'; + static const imageClassificationOfflineV2 = 'image_classification_offline_v2'; static const allIds = [ imageClassification, objectDetection, imageSegmentationV2, naturalLanguageProcessing, - imageClassificationOffline, superResolution, + imageClassificationOffline, + imageClassificationV2, + imageClassificationOfflineV2, ]; } diff --git a/flutter/lib/benchmark/info.dart b/flutter/lib/benchmark/info.dart index 13589386e..6e97c5b88 100644 --- a/flutter/lib/benchmark/info.dart +++ b/flutter/lib/benchmark/info.dart @@ -82,19 +82,24 @@ class BenchmarkInfo { final _benchmarkIcons = { BenchmarkId.imageClassification: AppIcons.imageClassification, + BenchmarkId.imageClassificationV2: AppIcons.imageClassification, BenchmarkId.objectDetection: AppIcons.objectDetection, BenchmarkId.imageSegmentationV2: AppIcons.imageSegmentation, BenchmarkId.naturalLanguageProcessing: AppIcons.languageProcessing, BenchmarkId.superResolution: AppIcons.superResolution, BenchmarkId.imageClassificationOffline: AppIcons.imageClassificationOffline, + BenchmarkId.imageClassificationOfflineV2: AppIcons.imageClassificationOffline, }; final _benchmarkIconsWhite = { BenchmarkId.imageClassification: AppIcons.imageClassificationWhite, + BenchmarkId.imageClassificationV2: AppIcons.imageClassificationWhite, BenchmarkId.objectDetection: AppIcons.objectDetectionWhite, BenchmarkId.imageSegmentationV2: AppIcons.imageSegmentationWhite, BenchmarkId.naturalLanguageProcessing: AppIcons.languageProcessingWhite, BenchmarkId.superResolution: AppIcons.superResolutionWhite, BenchmarkId.imageClassificationOffline: AppIcons.imageClassificationOfflineWhite, + BenchmarkId.imageClassificationOfflineV2: + AppIcons.imageClassificationOfflineWhite, }; diff --git a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt index 27415ca59..040b09868 100644 --- a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt +++ b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt @@ -58,6 +58,25 @@ benchmark_setting { delegate_selected: "CPU & GPU & ANE" } +benchmark_setting { + benchmark_id: "image_classification_v2" + accelerator: "ane" + accelerator_desc: "Apple Neural Engine" + framework: "Core ML" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" + model_checksum: "875730de6fa07333051ca0f283fb23fd" +} + +benchmark_setting { + benchmark_id: "image_classification_offline_v2" + accelerator: "ane" + accelerator_desc: "Apple Neural Engine" + framework: "Core ML" + batch_size: 32 + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" + model_checksum: "875730de6fa07333051ca0f283fb23fd" +} + benchmark_setting { benchmark_id: "object_detection" framework: "Core ML" diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile index 1fcb9f60e..5648e04a5 100644 --- a/mobile_back_apple/dev-utils/Makefile +++ b/mobile_back_apple/dev-utils/Makefile @@ -8,10 +8,10 @@ REPO_ROOT_DIR="/Users/anh/dev/mlcommons/mobile_app_open" #REPO_ROOT_DIR="/Users/freedom/work/mobile_app_open" -tflite: tflite-build tflite-run-ic -coreml: coreml-build coreml-run-ic -tflite-all: tflite-build tflite-run-ic tflite-run-ic-offline tflite-run-od tflite-run-is tflite-run-lu tflite-run-sr -coreml-all: coreml-build coreml-run-ic coreml-run-ic-offline coreml-run-od coreml-run-is coreml-run-lu coreml-run-sr +tflite: tflite-build tflite-run-ic-v2 tflite-run-ic-offline-v2 +coreml: coreml-build coreml-run-ic-v2 coreml-run-ic-offline-v2 +tflite-all: tflite-build tflite-run-ic tflite-run-ic-offline tflite-run-od tflite-run-is tflite-run-lu tflite-run-sr tflite-run-ic-v2 tflite-run-ic-offline-v2 +coreml-all: coreml-build coreml-run-ic coreml-run-ic-offline coreml-run-od coreml-run-is coreml-run-lu coreml-run-sr coreml-run-ic-v2 coreml-run-ic-offline-v2 backend=WITH_APPLE=1 WITH_TFLITE=0 app: @@ -52,6 +52,30 @@ tflite-run-ic-offline: --max_duration_ms=600000 \ --min_duration_ms=100 +tflite-run-ic-v2: + cd ${REPO_ROOT_DIR} && \ + bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification_v2 \ + --mode=PerformanceOnly \ + --output_dir="${REPO_ROOT_DIR}/output" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/mobilenet-edgetpu-v2-L_fp32.tflite" \ + --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ + --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ + --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ + --offset=1 + +tflite-run-ic-offline-v2: + cd ${REPO_ROOT_DIR} && \ + bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification_offline_v2 \ + --mode=PerformanceOnly \ + --scenario=Offline \ + --batch_size=32 \ + --output_dir="${REPO_ROOT_DIR}/output" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/mobilenet-edgetpu-v2-L_fp32.tflite" \ + --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ + --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ + --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ + --offset=1 + tflite-run-od: cd ${REPO_ROOT_DIR} && \ bazel-bin/flutter/cpp/binary/main EXTERNAL object_detection \ @@ -123,6 +147,30 @@ coreml-run-ic-offline: --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ --offset=1 +coreml-run-ic-v2: + cd ${REPO_ROOT_DIR} && \ + bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification \ + --mode=PerformanceOnly \ + --output_dir="${REPO_ROOT_DIR}/output" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel" \ + --lib_path="bazel-bin/mobile_back_apple/cpp/backend_coreml/libcoremlbackend.so" \ + --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ + --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ + --offset=1 + +coreml-run-ic-offline-v2: + cd ${REPO_ROOT_DIR} && \ + bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification_offline \ + --mode=PerformanceOnly \ + --scenario=Offline \ + --batch_size=32 \ + --output_dir="${REPO_ROOT_DIR}/output" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel" \ + --lib_path="bazel-bin/mobile_back_apple/cpp/backend_coreml/libcoremlbackend.so" \ + --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ + --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ + --offset=1 + coreml-run-od: cd ${REPO_ROOT_DIR} && \ bazel-bin/flutter/cpp/binary/main EXTERNAL object_detection \ diff --git a/mobile_back_apple/models/icv2.py b/mobile_back_apple/models/icv2.py new file mode 100644 index 000000000..61ff905e9 --- /dev/null +++ b/mobile_back_apple/models/icv2.py @@ -0,0 +1,52 @@ +# !/usr/bin/env python3 +# coding: utf-8 + +# Copyright 2023 The MLPerf Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import coremltools as ct +import tensorflow as tf +import tensorflow_hub as hub + + +def main(): + print("Retrieve model from TFHub...") + keras_layer = hub.KerasLayer('https://tfhub.dev/google/edgetpu/vision/mobilenet-edgetpu-v2/l/1') + keras_model = tf.keras.Sequential([keras_layer]) + keras_model.build([None, 224, 224, 3]) + input_tensor = tf.ones((4, 224, 224, 3)) + output_tensor = keras_model(input_tensor) + print("output_tensor.shape:", output_tensor.shape) + + model = ct.convert( + keras_model, + convert_to="neuralnetwork", + inputs=[ct.TensorType(shape=(1, 224, 224, 3))], + ) + model.short_description = "MobilenetEdgeTPUv2 from https://tfhub.dev/google/edgetpu/vision/mobilenet-edgetpu-v2/l/1" + + spec = model.get_spec() + for n in (1, 1001): + spec.description.output[0].type.multiArrayType.shape.append(n) + + ct.utils.rename_feature(spec, "keras_layer_input", "images") + ct.utils.rename_feature(spec, "Identity", "Softmax") + print(spec.description) + + export_fpath = '../dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel' + ct.models.MLModel(spec).save(export_fpath) + + +if __name__ == "__main__": + main() diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index 08ae1686e..8fae651f3 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -30,6 +30,15 @@ benchmark_setting { delegate_selected: "NNAPI" } +benchmark_setting { + benchmark_id: "image_classification_v2" + accelerator: "tpu" + accelerator_desc: "Google Edge TPU" + framework: "TFLite NNAPI" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" +} + benchmark_setting { benchmark_id: "image_classification_offline" framework: "TFLite" @@ -52,6 +61,16 @@ benchmark_setting { delegate_selected: "NNAPI" } +benchmark_setting { + benchmark_id: "image_classification_offline_v2" + accelerator: "tpu" + accelerator_desc: "Google Edge TPU" + framework: "TFLite NNAPI" + batch_size: 16 + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" +} + benchmark_setting { benchmark_id: "object_detection" framework: "TFLite" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt index 4a2e521bc..3bbe1ae17 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt @@ -31,6 +31,26 @@ benchmark_setting { delegate_selected: "NNAPI" } +benchmark_setting { + benchmark_id: "image_classification_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "NNAPI" + accelerator_name: "npu" + accelerator_desc: "NPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + } + delegate_selected: "NNAPI" +} + benchmark_setting { benchmark_id: "image_classification_offline" framework: "TFLite" @@ -53,6 +73,28 @@ benchmark_setting { delegate_selected: "NNAPI" } +benchmark_setting { + benchmark_id: "image_classification_offline_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "NNAPI" + accelerator_name: "npu" + accelerator_desc: "NPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + batch_size: 2 + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + batch_size: 2 + } + delegate_selected: "NNAPI" +} + benchmark_setting { benchmark_id: "object_detection" framework: "TFLite" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt index c13d1907f..8669e7fb4 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt @@ -24,6 +24,28 @@ benchmark_setting { delegate_selected: "Core ML" } +benchmark_setting { + benchmark_id: "image_classification_offline_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "Core ML" + accelerator_name: "ane" + accelerator_desc: "ANE" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + batch_size: 64 + } + delegate_choice: { + delegate_name: "Metal" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + batch_size: 64 + } + delegate_selected: "Core ML" +} + common_setting { id: "shards_num" name: "Number of threads for inference" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt index 0530938d8..44b482330 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt @@ -24,6 +24,28 @@ benchmark_setting { delegate_selected: "Core ML" } +benchmark_setting { + benchmark_id: "image_classification_offline_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "Core ML" + accelerator_name: "ane" + accelerator_desc: "ANE" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + batch_size: 8 + } + delegate_choice: { + delegate_name: "Metal" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + batch_size: 8 + } + delegate_selected: "Core ML" +} + common_setting { id: "shards_num" name: "Number of threads for inference" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt index bfee2a31b..f90255d22 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt @@ -24,6 +24,28 @@ benchmark_setting { delegate_selected: "Core ML" } +benchmark_setting { + benchmark_id: "image_classification_offline_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "Core ML" + accelerator_name: "ane" + accelerator_desc: "ANE" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + batch_size: 32 + } + delegate_choice: { + delegate_name: "Metal" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + batch_size: 32 + } + delegate_selected: "Core ML" +} + common_setting { id: "shards_num" name: "Number of threads for inference" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt index c97ccbca4..42cd0e2d1 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt @@ -31,6 +31,26 @@ benchmark_setting { delegate_selected: "Core ML" } +benchmark_setting { + benchmark_id: "image_classification_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "Core ML" + accelerator_name: "ane" + accelerator_desc: "ANE" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + } + delegate_choice: { + delegate_name: "Metal" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + } + delegate_selected: "Core ML" +} + benchmark_setting { benchmark_id: "object_detection" framework: "TFLite" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt index 0bac45970..300b7d178 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt @@ -38,6 +38,30 @@ benchmark_setting { delegate_selected: "CPU" } +benchmark_setting { + benchmark_id: "image_classification_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "CPU" + accelerator_name: "cpu" + accelerator_desc: "CPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + } +} + +benchmark_setting { + benchmark_id: "image_classification_offline_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "CPU" + accelerator_name: "cpu" + accelerator_desc: "CPU" + batch_size: 2 + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" + model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + } +} benchmark_setting { benchmark_id: "object_detection" framework: "TFLite" From ee0fc545f65637406ee0785c2ff3cba124714c28 Mon Sep 17 00:00:00 2001 From: Anh Date: Sat, 25 Nov 2023 21:48:46 +0700 Subject: [PATCH 02/20] Set max_duration for new tasks --- flutter/assets/tasks.pbtxt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt index 5a113cad9..01029bfd1 100644 --- a/flutter/assets/tasks.pbtxt +++ b/flutter/assets/tasks.pbtxt @@ -221,6 +221,7 @@ task { name: "Image Classification v2" min_query_count: 1024 min_duration: 60 + max_duration: 600 max_throughput: 1000 max_accuracy: 1.0 scenario: "SingleStream" @@ -257,6 +258,7 @@ task { name: "Image Classification v2 (Offline)" min_query_count: 24576 min_duration: 0 + max_duration: 0 max_throughput: 2000 max_accuracy: 1.0 scenario: "Offline" From e8eb505998f11125ce5aae60d1ea4ba56a2843ab Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 2 Jan 2024 22:52:40 +0700 Subject: [PATCH 03/20] Update backend settings --- .../cpp/backend_coreml/coreml_settings.pbtxt | 56 ++++++++++++++++--- .../settings/tflite_settings_pixel6.pbtxt | 53 +++++++++++++----- .../tflite_settings_windows.pbtxt | 1 + 3 files changed, 86 insertions(+), 24 deletions(-) diff --git a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt index 040b09868..7e8389600 100644 --- a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt +++ b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt @@ -60,21 +60,59 @@ benchmark_setting { benchmark_setting { benchmark_id: "image_classification_v2" - accelerator: "ane" - accelerator_desc: "Apple Neural Engine" framework: "Core ML" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + delegate_choice: { + delegate_name: "CPU & GPU & ANE" + accelerator_name: "cpu&gpu&ane" + accelerator_desc: "All compute units" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" + model_checksum: "875730de6fa07333051ca0f283fb23fd" + } + delegate_choice: { + delegate_name: "CPU & GPU" + accelerator_name: "cpu&gpu" + accelerator_desc: "CPU and GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" + model_checksum: "875730de6fa07333051ca0f283fb23fd" + } + delegate_choice: { + delegate_name: "CPU & ANE" + accelerator_name: "cpu&ane" + accelerator_desc: "CPU and Neural Engine" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" + model_checksum: "875730de6fa07333051ca0f283fb23fd" + } + delegate_selected: "CPU & GPU & ANE" } benchmark_setting { benchmark_id: "image_classification_offline_v2" - accelerator: "ane" - accelerator_desc: "Apple Neural Engine" framework: "Core ML" - batch_size: 32 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + delegate_choice: { + delegate_name: "CPU & GPU & ANE" + accelerator_name: "cpu&gpu&ane" + accelerator_desc: "All compute units" + batch_size: 32 + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" + model_checksum: "875730de6fa07333051ca0f283fb23fd" + } + delegate_choice: { + delegate_name: "CPU & GPU" + accelerator_name: "cpu&gpu" + accelerator_desc: "CPU and GPU" + batch_size: 32 + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" + model_checksum: "875730de6fa07333051ca0f283fb23fd" + } + delegate_choice: { + delegate_name: "CPU & ANE" + accelerator_name: "cpu&ane" + accelerator_desc: "CPU and Neural Engine" + batch_size: 32 + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" + model_checksum: "875730de6fa07333051ca0f283fb23fd" + } + delegate_selected: "CPU & GPU & ANE" } benchmark_setting { diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index 8fae651f3..1022ac06f 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -30,15 +30,6 @@ benchmark_setting { delegate_selected: "NNAPI" } -benchmark_setting { - benchmark_id: "image_classification_v2" - accelerator: "tpu" - accelerator_desc: "Google Edge TPU" - framework: "TFLite NNAPI" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" -} - benchmark_setting { benchmark_id: "image_classification_offline" framework: "TFLite" @@ -61,14 +52,46 @@ benchmark_setting { delegate_selected: "NNAPI" } +benchmark_setting { + benchmark_id: "image_classification_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "NNAPI" + accelerator_name: "tpu" + accelerator_desc: "Google EdgeTPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + } + delegate_selected: "NNAPI" +} + benchmark_setting { benchmark_id: "image_classification_offline_v2" - accelerator: "tpu" - accelerator_desc: "Google Edge TPU" - framework: "TFLite NNAPI" - batch_size: 16 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + framework: "TFLite" + delegate_choice: { + delegate_name: "NNAPI" + accelerator_name: "tpu" + accelerator_desc: "Google EdgeTPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + batch_size: 16 + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" + model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + batch_size: 2 + } + delegate_selected: "NNAPI" } benchmark_setting { diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt index 300b7d178..5aafba83a 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt @@ -62,6 +62,7 @@ benchmark_setting { model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" } } + benchmark_setting { benchmark_id: "object_detection" framework: "TFLite" From 6561eb18870688cde1c745904db890bcd37a77a1 Mon Sep 17 00:00:00 2001 From: Anh Date: Wed, 3 Jan 2024 19:27:18 +0700 Subject: [PATCH 04/20] Update expected_accuracy.dart --- .../integration_test/expected_accuracy.dart | 74 +++++++++++-------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart index c67fe8328..9b815d529 100644 --- a/flutter/integration_test/expected_accuracy.dart +++ b/flutter/integration_test/expected_accuracy.dart @@ -11,56 +11,66 @@ key: OR | - ane -> iOS TFLite or Core ML - cpu&gpu&ane -> IOS Core ML */ +const _kCpu = 'cpu'; +const _kNpu = 'npu'; +const _kTpu = 'tpu'; +const _kAne = 'ane'; +const _kCpuGpuAne = 'cpu&gpu&ane'; +const _kAneTflite = 'ane|TFLite'; +const _kAneCoreml = 'ane|Core ML'; +const _kGpuTflite = 'gpu|TFLite'; +const _kGpuCoreml = 'gpu|Core ML'; const Map _imageClassification = { - 'cpu': Interval(min: 1.00, max: 1.00), - 'npu': Interval(min: 0.89, max: 0.91), - 'tpu': Interval(min: 0.89, max: 0.91), - 'ane': Interval(min: 1.00, max: 1.00), - 'cpu&gpu&ane': Interval(min: 1.00, max: 1.00) + _kCpu: Interval(min: 1.00, max: 1.00), + _kNpu: Interval(min: 0.89, max: 0.91), + _kTpu: Interval(min: 0.89, max: 0.91), + _kAne: Interval(min: 1.00, max: 1.00), + _kCpuGpuAne: Interval(min: 1.00, max: 1.00) }; const Map _imageClassificationV2 = { - 'cpu': Interval(min: 0.69, max: 0.71), - 'npu': Interval(min: 0.69, max: 0.71), - 'tpu': Interval(min: 0.69, max: 0.71), - 'ane': Interval(min: 0.69, max: 0.71), + _kCpu: Interval(min: 0.69, max: 0.71), + _kNpu: Interval(min: 0.69, max: 0.71), + _kTpu: Interval(min: 0.69, max: 0.71), + _kAne: Interval(min: 0.69, max: 0.71), + _kCpuGpuAne: Interval(min: 0.69, max: 0.71), }; const Map _objectDetection = { - 'cpu': Interval(min: 0.31, max: 0.32), - 'npu': Interval(min: 0.28, max: 0.31), - 'tpu': Interval(min: 0.36, max: 0.38), - 'ane|TFLite': Interval(min: 0.31, max: 0.34), - 'ane|Core ML': Interval(min: 0.45, max: 0.46), - 'cpu&gpu&ane': Interval(min: 0.45, max: 0.46) + _kCpu: Interval(min: 0.31, max: 0.32), + _kNpu: Interval(min: 0.28, max: 0.31), + _kTpu: Interval(min: 0.36, max: 0.38), + _kAneTflite: Interval(min: 0.31, max: 0.34), + _kAneCoreml: Interval(min: 0.45, max: 0.46), + _kCpuGpuAne: Interval(min: 0.45, max: 0.46) }; const Map _imageSegmentation = { - 'cpu': Interval(min: 0.38, max: 0.40), - 'npu': Interval(min: 0.33, max: 0.34), - 'tpu': Interval(min: 0.33, max: 0.34), - 'ane|TFLite': Interval(min: 0.38, max: 0.40), - 'ane|Core ML': Interval(min: 0.38, max: 0.40), - 'cpu&gpu&ane': Interval(min: 0.38, max: 0.40) + _kCpu: Interval(min: 0.38, max: 0.40), + _kNpu: Interval(min: 0.33, max: 0.34), + _kTpu: Interval(min: 0.33, max: 0.34), + _kAneTflite: Interval(min: 0.38, max: 0.40), + _kAneCoreml: Interval(min: 0.38, max: 0.40), + _kCpuGpuAne: Interval(min: 0.38, max: 0.40) }; const Map _naturalLanguageProcessing = { - 'cpu': Interval(min: 1.00, max: 1.00), - 'tpu': Interval(min: 1.00, max: 1.00), - 'gpu|TFLite': Interval(min: 1.00, max: 1.00), + _kCpu: Interval(min: 1.00, max: 1.00), + _kTpu: Interval(min: 1.00, max: 1.00), + _kGpuTflite: Interval(min: 1.00, max: 1.00), // 1.00 in simulator, 0.80 on iphone 12 mini - 'gpu|Core ML': Interval(min: 0.80, max: 1.00), - 'cpu&gpu&ane': Interval(min: 0.80, max: 1.00) + _kGpuCoreml: Interval(min: 0.80, max: 1.00), + _kCpuGpuAne: Interval(min: 0.80, max: 1.00) }; const Map _superResolution = { - 'cpu': Interval(min: 0.32, max: 0.35), - 'npu': Interval(min: 0.32, max: 0.35), - 'tpu': Interval(min: 0.32, max: 0.35), - 'ane|TFLite': Interval(min: 0.32, max: 0.35), - 'ane|Core ML': Interval(min: 0.32, max: 0.35), - 'cpu&gpu&ane': Interval(min: 0.32, max: 0.35) + _kCpu: Interval(min: 0.32, max: 0.35), + _kNpu: Interval(min: 0.32, max: 0.35), + _kTpu: Interval(min: 0.32, max: 0.35), + _kAneTflite: Interval(min: 0.32, max: 0.35), + _kAneCoreml: Interval(min: 0.32, max: 0.35), + _kCpuGpuAne: Interval(min: 0.32, max: 0.35) }; const benchmarkExpectedAccuracy = { From 07d6e87654e770031418ffead38ad09b68d3efd1 Mon Sep 17 00:00:00 2001 From: Anh Date: Wed, 3 Jan 2024 19:43:36 +0700 Subject: [PATCH 05/20] Update integration test --- flutter/integration_test/utils.dart | 30 ++++++++++++------- .../tflite_settings_windows.pbtxt | 2 ++ 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/flutter/integration_test/utils.dart b/flutter/integration_test/utils.dart index d78c9faa8..079c87f3d 100644 --- a/flutter/integration_test/utils.dart +++ b/flutter/integration_test/utils.dart @@ -34,23 +34,31 @@ Future validateSettings(WidgetTester tester) async { final state = tester.state(find.byType(MaterialApp)); final benchmarkState = state.context.read(); for (var benchmark in benchmarkState.benchmarks) { - expect(benchmark.selectedDelegate.batchSize, greaterThanOrEqualTo(0)); - expect(benchmark.selectedDelegate.modelPath.isNotEmpty, isTrue); - expect(benchmark.selectedDelegate.modelChecksum.isNotEmpty, isTrue); - expect(benchmark.selectedDelegate.acceleratorName.isNotEmpty, isTrue); - expect(benchmark.selectedDelegate.acceleratorDesc.isNotEmpty, isTrue); - expect(benchmark.benchmarkSettings.framework.isNotEmpty, isTrue); + expect(benchmark.selectedDelegate.batchSize, greaterThanOrEqualTo(0), + reason: 'batchSize must >= 0'); + expect(benchmark.selectedDelegate.modelPath.isNotEmpty, isTrue, + reason: 'modelPath cannot be empty'); + expect(benchmark.selectedDelegate.modelChecksum.isNotEmpty, isTrue, + reason: 'modelChecksum cannot be empty'); + expect(benchmark.selectedDelegate.acceleratorName.isNotEmpty, isTrue, + reason: 'acceleratorName cannot be empty'); + expect(benchmark.selectedDelegate.acceleratorDesc.isNotEmpty, isTrue, + reason: 'acceleratorDesc cannot be empty'); + expect(benchmark.benchmarkSettings.framework.isNotEmpty, isTrue, + reason: 'framework cannot be empty'); expect(benchmark.selectedDelegate.delegateName, - equals(benchmark.benchmarkSettings.delegateSelected)); + equals(benchmark.benchmarkSettings.delegateSelected), + reason: 'delegateSelected must be the same as delegateName'); final selected = benchmark.benchmarkSettings.delegateSelected; final choices = benchmark.benchmarkSettings.delegateChoice .map((e) => e.delegateName) .toList(); - expect(choices.isNotEmpty, isTrue); - final reason = - 'delegate_selected=$selected must be one of delegate_choice=$choices'; - expect(choices.contains(selected), isTrue, reason: reason); + expect(choices.isNotEmpty, isTrue, + reason: 'There must be at least one delegate choice'); + expect(choices.contains(selected), isTrue, + reason: + 'delegate_selected=$selected must be one of delegate_choice=$choices'); } } diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt index 5aafba83a..2eae34450 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt @@ -48,6 +48,7 @@ benchmark_setting { model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" } + delegate_selected: "CPU" } benchmark_setting { @@ -61,6 +62,7 @@ benchmark_setting { model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" } + delegate_selected: "CPU" } benchmark_setting { From 045024cb798c84d09021043e0cf533bd337949e6 Mon Sep 17 00:00:00 2001 From: Anh Date: Wed, 7 Feb 2024 11:44:44 +0700 Subject: [PATCH 06/20] Fix Apple dev Makefile --- .bazelrc | 2 +- mobile_back_apple/dev-utils/Makefile | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.bazelrc b/.bazelrc index 0406131b7..0913da99a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -21,7 +21,7 @@ build:android --copt=-w build:ios --copt=-w build:windows --copt=/W0 -# Build in C++ 14 mode. +# Build in C++ 17 mode. build:android --cxxopt=-std=c++17 build:android --host_cxxopt=-std=c++17 build:ios --cxxopt=-std=c++17 diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile index 5648e04a5..1015847c5 100644 --- a/mobile_back_apple/dev-utils/Makefile +++ b/mobile_back_apple/dev-utils/Makefile @@ -8,6 +8,9 @@ REPO_ROOT_DIR="/Users/anh/dev/mlcommons/mobile_app_open" #REPO_ROOT_DIR="/Users/freedom/work/mobile_app_open" +export ANDROID_NDK_VERSION=25 +export ANDROID_NDK_API_LEVEL=33 + tflite: tflite-build tflite-run-ic-v2 tflite-run-ic-offline-v2 coreml: coreml-build coreml-run-ic-v2 coreml-run-ic-offline-v2 tflite-all: tflite-build tflite-run-ic tflite-run-ic-offline tflite-run-od tflite-run-is tflite-run-lu tflite-run-sr tflite-run-ic-v2 tflite-run-ic-offline-v2 @@ -22,7 +25,7 @@ app: tflite-build: cd ${REPO_ROOT_DIR} && \ - bazel build -c opt --cxxopt=-std=c++14 --host_cxxopt=-std=c++14 //flutter/cpp/binary:main //mobile_back_tflite:tflitebackend + bazel build -c opt --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //flutter/cpp/binary:main //mobile_back_tflite:tflitebackend tflite-run-ic: cd ${REPO_ROOT_DIR} && \ @@ -120,7 +123,7 @@ tflite-run-sr: coreml-build: cd ${REPO_ROOT_DIR} && \ - bazel build -c opt --cxxopt=-fobjc-arc --cxxopt=-xobjective-c++ --cxxopt=-std=c++14 --host_cxxopt=-std=c++14 \ + bazel build -c opt --cxxopt=-fobjc-arc --cxxopt=-xobjective-c++ --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 \ //flutter/cpp/binary:main //mobile_back_apple:coremlbackend coreml-run-ic: From ec8a550d552892e489ebe78c8a41bdd7d891ba52 Mon Sep 17 00:00:00 2001 From: Anh Date: Thu, 8 Feb 2024 15:31:34 +0700 Subject: [PATCH 07/20] Use GitHub vars for --num-flaky-test-attempts flag --- .github/workflows/android-build-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index 0ac420f01..9f300854d 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -204,7 +204,7 @@ jobs: --app $GCLOUD_BUCKET_PATH/test-main-tflite.apk \ --test $GCLOUD_BUCKET_PATH/test-helper-tflite.apk \ --timeout 30m \ - --num-flaky-test-attempts 2 \ + --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ --device model=redfin,version=30,locale=en,orientation=portrait \ --client-details=buildNumber=${{ github.run_number }} @@ -229,7 +229,7 @@ jobs: --app $GCLOUD_BUCKET_PATH/test-main-pixel.apk \ --test $GCLOUD_BUCKET_PATH/test-helper-pixel.apk \ --timeout 30m \ - --num-flaky-test-attempts 2 \ + --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ --device model=oriole,version=32,locale=en,orientation=portrait \ --client-details=buildNumber=${{ github.run_number }} @@ -254,7 +254,7 @@ jobs: --app $GCLOUD_BUCKET_PATH/test-main-qti.apk \ --test $GCLOUD_BUCKET_PATH/test-helper-qti.apk \ --timeout 30m \ - --num-flaky-test-attempts 2 \ + --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ --device model=b0q,version=33,locale=en,orientation=portrait \ --client-details=buildNumber=${{ github.run_number }} @@ -279,6 +279,6 @@ jobs: --app $GCLOUD_BUCKET_PATH/test-main-mtk.apk \ --test $GCLOUD_BUCKET_PATH/test-helper-mtk.apk \ --timeout 30m \ - --num-flaky-test-attempts 2 \ + --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ --device model=OP515BL1,version=33,locale=en,orientation=portrait \ --client-details=buildNumber=${{ github.run_number }} From d4aa2976b25aa29cab408cc280663344f50626b2 Mon Sep 17 00:00:00 2001 From: Anh Date: Thu, 8 Feb 2024 15:35:47 +0700 Subject: [PATCH 08/20] Add icon for new tasks --- flutter/lib/ui/icons.dart | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/flutter/lib/ui/icons.dart b/flutter/lib/ui/icons.dart index edc1dcc80..c09f493fc 100644 --- a/flutter/lib/ui/icons.dart +++ b/flutter/lib/ui/icons.dart @@ -59,21 +59,27 @@ class AppIcons { class BenchmarkIcons { static final darkSet = { BenchmarkId.imageClassification: AppIcons.imageClassification, + BenchmarkId.imageClassificationV2: AppIcons.imageClassification, BenchmarkId.objectDetection: AppIcons.objectDetection, BenchmarkId.imageSegmentationV2: AppIcons.imageSegmentation, BenchmarkId.naturalLanguageProcessing: AppIcons.languageProcessing, BenchmarkId.superResolution: AppIcons.superResolution, BenchmarkId.imageClassificationOffline: AppIcons.imageClassificationOffline, + BenchmarkId.imageClassificationOfflineV2: + AppIcons.imageClassificationOffline, }; static final lightSet = { BenchmarkId.imageClassification: AppIcons.imageClassificationWhite, + BenchmarkId.imageClassificationV2: AppIcons.imageClassificationWhite, BenchmarkId.objectDetection: AppIcons.objectDetectionWhite, BenchmarkId.imageSegmentationV2: AppIcons.imageSegmentationWhite, BenchmarkId.naturalLanguageProcessing: AppIcons.languageProcessingWhite, BenchmarkId.superResolution: AppIcons.superResolutionWhite, BenchmarkId.imageClassificationOffline: AppIcons.imageClassificationOfflineWhite, + BenchmarkId.imageClassificationOfflineV2: + AppIcons.imageClassificationOfflineWhite, }; static Widget getDarkIcon(String benchmarkId) => From 5c2717d6c99786d140f4865630419c30b98f1602 Mon Sep 17 00:00:00 2001 From: Nathan Wasson Date: Tue, 13 Feb 2024 10:55:06 -0600 Subject: [PATCH 09/20] Update S3 provider in android-build-test.yml (#855) * Update S3 provider in android-build-test.yml * Update env var name --------- Co-authored-by: Anh --- .github/workflows/android-build-test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index 0ac420f01..8ced2cd29 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -86,10 +86,10 @@ jobs: python3 -m pip install awscli - name: Download Samsung libraries env: - AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }} run: | - aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \ + aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \ cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/samsung_libs_v3.1_20231116.zip /tmp/ && \ unzip /tmp/samsung_libs_v3.1_20231116.zip -d /tmp/samsung_libs_v3.1_20231116 && \ rm /tmp/samsung_libs_v3.1_20231116.zip && \ @@ -97,12 +97,12 @@ jobs: mv /tmp/samsung_libs_v3.1_20231116/* mobile_back_samsung/samsung/lib/internal/ - name: Download QTI libraries env: - AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }} run: | - aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \ + aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \ cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/qaisw-2.13.4.230831213719_60417-linux.zip /tmp/ && \ - unzip -P ${AWS_SECRET_ACCESS_KEY} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \ + unzip -P ${{ secrets.READONLY_AWS_KEY }} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \ rm /tmp/qaisw-2.13.4.230831213719_60417-linux.zip && \ mv /tmp/qaisw-2.13.4.230831213719_60417-linux/qaisw-2.13.4.230831213719_60417 mobile_back_qti/ - name: Cache bazel From c128476b9e07dcbd718da85f9ecf7f85ef83c46c Mon Sep 17 00:00:00 2001 From: Koan-Sin Tan Date: Tue, 27 Feb 2024 14:57:15 +0800 Subject: [PATCH 10/20] For Mobilenet V4 large (#821) * hacks for model small * small -> medium * medium -> large * for quantized models * make float model work again fixed a stupid bug * fix pixel and neuron settings * change preprocessing and label offset * Adding changes for calibration dataset preprocessing * hacks * clean up * fix pixel backend settings * fix pixel settings & icons for classificaton v2 * fix lint issue * Revert "Adding changes for calibration dataset preprocessing" This reverts commit c2593337b3cca1bedd1c2533890b5adba21314a5. New model doesn't need these * revert imagenet width and height we'll use #853 to handle different width and height * fix offline for Pixel backend * Undo update for Windows build * update MobilenetV4 settings for TFLite, Neuron, and Pixel * update windows setting for Mobilenet V4 * Use MobileNetV4 model in Apple devices * Fix tflite_settings_windows.pbtxt * Update expected_accuracy * Update expected_accuracy and expected_throughput for Apple devices * Use fp32 model for Windows * Update expected_throughput for Windows * Update expected_throughput for Android * Update S3 provider in android-build-test.yml (#855) * Update S3 provider in android-build-test.yml * Update env var name --------- Co-authored-by: Anh * Reduce batch_size for image_classification_offline_v2 in Pixel 6 --------- Co-authored-by: Mohit Mundhra Co-authored-by: Anh Co-authored-by: Nathan Wasson --- .github/workflows/android-build-test.yml | 14 ++--- flutter/assets/tasks.pbtxt | 12 ++-- .../integration_test/expected_accuracy.dart | 16 +++--- .../integration_test/expected_throughput.dart | 30 +++++----- .../cpp/backend_coreml/coreml_settings.pbtxt | 24 ++++---- mobile_back_apple/dev-utils/Makefile | 24 +++++--- mobile_back_apple/models/icv2.py | 28 +++++----- .../settings/tflite_settings_pixel6.pbtxt | 33 ++++++----- .../tflite_settings_android.pbtxt | 16 +++--- .../tflite_settings_apple_iphone11.pbtxt | 8 +-- .../tflite_settings_apple_iphone12.pbtxt | 8 +-- .../tflite_settings_apple_iphoneX.pbtxt | 8 +-- .../tflite_settings_apple_main.pbtxt | 8 +-- .../tflite_settings_windows.pbtxt | 8 +-- .../tflite_settings_mtk.pbtxt | 55 +++++++++++++++++++ 15 files changed, 176 insertions(+), 116 deletions(-) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index 9f300854d..ccf84647c 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -86,10 +86,10 @@ jobs: python3 -m pip install awscli - name: Download Samsung libraries env: - AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }} run: | - aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \ + aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \ cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/samsung_libs_v3.1_20231116.zip /tmp/ && \ unzip /tmp/samsung_libs_v3.1_20231116.zip -d /tmp/samsung_libs_v3.1_20231116 && \ rm /tmp/samsung_libs_v3.1_20231116.zip && \ @@ -97,12 +97,12 @@ jobs: mv /tmp/samsung_libs_v3.1_20231116/* mobile_back_samsung/samsung/lib/internal/ - name: Download QTI libraries env: - AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }} run: | - aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \ + aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \ cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/qaisw-2.13.4.230831213719_60417-linux.zip /tmp/ && \ - unzip -P ${AWS_SECRET_ACCESS_KEY} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \ + unzip -P ${{ secrets.READONLY_AWS_KEY }} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \ rm /tmp/qaisw-2.13.4.230831213719_60417-linux.zip && \ mv /tmp/qaisw-2.13.4.230831213719_60417-linux/qaisw-2.13.4.230831213719_60417 mobile_back_qti/ - name: Cache bazel diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt index 01029bfd1..7795e39fd 100644 --- a/flutter/assets/tasks.pbtxt +++ b/flutter/assets/tasks.pbtxt @@ -246,9 +246,9 @@ task { model { id: "MobileNetEdgeTPUv2-L" name: "MobileNetEdgeTPUv2-L" - offset: 1 - image_width: 224 - image_height: 224 + offset: 0 + image_width: 384 + image_height: 384 num_classes: 1000 } } @@ -283,9 +283,9 @@ task { model { id: "MobileNetEdgeTPUv2-L" name: "MobileNetEdgeTPUv2-L" - offset: 1 - image_width: 224 - image_height: 224 + offset: 0 + image_width: 384 + image_height: 384 num_classes: 1000 } } diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart index 880f07fe4..dc50aee8b 100644 --- a/flutter/integration_test/expected_accuracy.dart +++ b/flutter/integration_test/expected_accuracy.dart @@ -28,14 +28,14 @@ const Map _imageClassification = { }; const Map _imageClassificationV2 = { - 'cpu': Interval(min: 0.69, max: 0.71), - 'npu': Interval(min: 0.69, max: 0.71), - 'tpu': Interval(min: 0.69, max: 0.71), - 'ane': Interval(min: 0.69, max: 0.71), - 'cpu&gpu&ane': Interval(min: 0.69, max: 0.71), - 'snpe_dsp': Interval(min: 0.69, max: 0.71), - 'psnpe_dsp': Interval(min: 0.69, max: 0.71), - 'neuron-mdla': Interval(min: 0.69, max: 0.71), + 'cpu': Interval(min: 0.88, max: 0.91), + 'npu': Interval(min: 0.88, max: 0.91), + 'tpu': Interval(min: 0.88, max: 0.91), + 'ane': Interval(min: 0.79, max: 0.91), + 'cpu&gpu&ane': Interval(min: 0.79, max: 0.91), + 'snpe_dsp': Interval(min: 0.88, max: 0.91), + 'psnpe_dsp': Interval(min: 0.88, max: 0.91), + 'neuron-mdla': Interval(min: 0.79, max: 0.81), }; const Map _objectDetection = { diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 29c215a53..85c7a3c59 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -55,24 +55,24 @@ const Map> _imageClassification = { // TODO (anhappdev): update the expected value for _imageClassificationV2 after gathering some statistics const Map> _imageClassificationV2 = { _kTFLiteBackend: { - _kCloudBuildX23: Interval(min: 2, max: 11), - _kCloudBuildX28: Interval(min: 2, max: 12), + _kCloudBuildX23: Interval(min: 1, max: 9), + _kCloudBuildX28: Interval(min: 1, max: 9), _kRyzen5600: Interval(min: 1, max: 37), - _kPixel5: Interval(min: 60, max: 75), - _kIphoneOnGitHubAction: Interval(min: 1, max: 7), + _kPixel5: Interval(min: 20, max: 75), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7), _kIphoneOnMacbookM1: Interval(min: 10, max: 27), }, _kCoreMLBackend: { - _kIphoneOnGitHubAction: Interval(min: 1, max: 7), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7), }, _kPixelBackend: { - _kPixel6: Interval(min: 500, max: 600), + _kPixel6: Interval(min: 100, max: 600), }, _kQtiBackend: { _kS22Ultra: Interval(min: 1900, max: 2200), }, _kMediatekBackend: { - _kDN2103: Interval(min: 30, max: 50), + _kDN2103: Interval(min: 5, max: 50), }, }; @@ -89,7 +89,7 @@ const Map> _objectDetection = { _kIphoneOnGitHubAction: Interval(min: 0.5, max: 4), }, _kPixelBackend: { - _kPixel6: Interval(min: 250, max: 450), + _kPixel6: Interval(min: 200, max: 450), }, _kQtiBackend: { _kS22Ultra: Interval(min: 800, max: 1400), @@ -195,24 +195,24 @@ const Map> _imageClassificationOffline = { // TODO (anhappdev): update the expected value for _imageClassificationOfflineV2 after gathering some statistics const Map> _imageClassificationOfflineV2 = { _kTFLiteBackend: { - _kCloudBuildX23: Interval(min: 4, max: 14), - _kCloudBuildX28: Interval(min: 3, max: 14), + _kCloudBuildX23: Interval(min: 1, max: 9), + _kCloudBuildX28: Interval(min: 1, max: 9), _kRyzen5600: Interval(min: 20, max: 60), - _kPixel5: Interval(min: 60, max: 180), - _kIphoneOnGitHubAction: Interval(min: 1, max: 15), + _kPixel5: Interval(min: 20, max: 180), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), _kIphoneOnMacbookM1: Interval(min: 10, max: 45), }, _kCoreMLBackend: { - _kIphoneOnGitHubAction: Interval(min: 1, max: 15), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), }, _kPixelBackend: { - _kPixel6: Interval(min: 700, max: 1700), + _kPixel6: Interval(min: 100, max: 1700), }, _kQtiBackend: { _kS22Ultra: Interval(min: 2600, max: 3000), }, _kMediatekBackend: { - _kDN2103: Interval(min: 75, max: 110), + _kDN2103: Interval(min: 5, max: 90), }, }; diff --git a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt index 7e8389600..758d675cc 100644 --- a/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt +++ b/mobile_back_apple/cpp/backend_coreml/coreml_settings.pbtxt @@ -65,22 +65,22 @@ benchmark_setting { delegate_name: "CPU & GPU & ANE" accelerator_name: "cpu&gpu&ane" accelerator_desc: "All compute units" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & GPU" accelerator_name: "cpu&gpu" accelerator_desc: "CPU and GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & ANE" accelerator_name: "cpu&ane" accelerator_desc: "CPU and Neural Engine" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_selected: "CPU & GPU & ANE" } @@ -93,24 +93,24 @@ benchmark_setting { accelerator_name: "cpu&gpu&ane" accelerator_desc: "All compute units" batch_size: 32 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & GPU" accelerator_name: "cpu&gpu" accelerator_desc: "CPU and GPU" batch_size: 32 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_choice: { delegate_name: "CPU & ANE" accelerator_name: "cpu&ane" accelerator_desc: "CPU and Neural Engine" batch_size: 32 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/CoreML/MobilenetEdgeTPUv2.mlmodel" - model_checksum: "875730de6fa07333051ca0f283fb23fd" + model_path: "https://github.com/mlcommons/mobile_models/releases/download/v4.0-coreml/MobilenetV4_Large.mlmodel" + model_checksum: "fbfba70596f74010852f22fa04721202" } delegate_selected: "CPU & GPU & ANE" } diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile index 1015847c5..863d4909b 100644 --- a/mobile_back_apple/dev-utils/Makefile +++ b/mobile_back_apple/dev-utils/Makefile @@ -60,11 +60,13 @@ tflite-run-ic-v2: bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification_v2 \ --mode=PerformanceOnly \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/mobilenet-edgetpu-v2-L_fp32.tflite" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-fp32.tflite" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 tflite-run-ic-offline-v2: cd ${REPO_ROOT_DIR} && \ @@ -73,11 +75,13 @@ tflite-run-ic-offline-v2: --scenario=Offline \ --batch_size=32 \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/mobilenet-edgetpu-v2-L_fp32.tflite" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-fp32.tflite" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 tflite-run-od: cd ${REPO_ROOT_DIR} && \ @@ -155,11 +159,13 @@ coreml-run-ic-v2: bazel-bin/flutter/cpp/binary/main EXTERNAL image_classification \ --mode=PerformanceOnly \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_apple/cpp/backend_coreml/libcoremlbackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 coreml-run-ic-offline-v2: cd ${REPO_ROOT_DIR} && \ @@ -168,11 +174,13 @@ coreml-run-ic-offline-v2: --scenario=Offline \ --batch_size=32 \ --output_dir="${REPO_ROOT_DIR}/output" \ - --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel" \ + --model_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel" \ + --image_width=384 \ + --image_height=384 \ --lib_path="bazel-bin/mobile_back_apple/cpp/backend_coreml/libcoremlbackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/images" \ --groundtruth_file="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/imagenet/imagenet_val.txt" \ - --offset=1 + --offset=0 coreml-run-od: cd ${REPO_ROOT_DIR} && \ diff --git a/mobile_back_apple/models/icv2.py b/mobile_back_apple/models/icv2.py index 61ff905e9..4af313776 100644 --- a/mobile_back_apple/models/icv2.py +++ b/mobile_back_apple/models/icv2.py @@ -16,36 +16,34 @@ # limitations under the License. import coremltools as ct -import tensorflow as tf -import tensorflow_hub as hub def main(): - print("Retrieve model from TFHub...") - keras_layer = hub.KerasLayer('https://tfhub.dev/google/edgetpu/vision/mobilenet-edgetpu-v2/l/1') - keras_model = tf.keras.Sequential([keras_layer]) - keras_model.build([None, 224, 224, 3]) - input_tensor = tf.ones((4, 224, 224, 3)) - output_tensor = keras_model(input_tensor) - print("output_tensor.shape:", output_tensor.shape) - + """ + Download the MobileNetV4-Conv-Large-fp32 from https://github.com/mlcommons/mobile_open/releases + Tested with tensorflow==2.15.0 and coremltools==7.1 + """ + saved_model_dir = '../dev-resources/mobilenet_v4/MobileNetV4-Conv-Large-saved-model' + export_fpath = '../dev-resources/mobilenet_v4/MobilenetV4_Large.mlmodel' + print("Converting model...") model = ct.convert( - keras_model, + saved_model_dir, + source="tensorflow", convert_to="neuralnetwork", - inputs=[ct.TensorType(shape=(1, 224, 224, 3))], + inputs=[ct.TensorType(shape=(1, 384, 384, 3))], ) - model.short_description = "MobilenetEdgeTPUv2 from https://tfhub.dev/google/edgetpu/vision/mobilenet-edgetpu-v2/l/1" + model.short_description = "MobileNetV4-Conv-Large-fp32 from https://github.com/mlcommons/mobile_open" spec = model.get_spec() for n in (1, 1001): spec.description.output[0].type.multiArrayType.shape.append(n) - ct.utils.rename_feature(spec, "keras_layer_input", "images") + ct.utils.rename_feature(spec, "inputs", "images") ct.utils.rename_feature(spec, "Identity", "Softmax") print(spec.description) - export_fpath = '../dev-resources/mobilenet_edgetpu/MobilenetEdgeTPUv2.mlmodel' ct.models.MLModel(spec).save(export_fpath) + print("Done! Core ML model exported to:", export_fpath) if __name__ == "__main__": diff --git a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt index 1022ac06f..6f07fee5f 100644 --- a/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt +++ b/mobile_back_pixel/cpp/backend_tflite/settings/tflite_settings_pixel6.pbtxt @@ -31,43 +31,43 @@ benchmark_setting { } benchmark_setting { - benchmark_id: "image_classification_offline" + benchmark_id: "image_classification_v2" framework: "TFLite" delegate_choice: { delegate_name: "NNAPI" accelerator_name: "tpu" accelerator_desc: "Google EdgeTPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_uint8.tflite" - model_checksum: "008dfcb1c1962fedbeef1b998d4c84f2" - batch_size: 64 + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/main/v1_1/tflite/mobilenet_edgetpu_224_1.0_float.tflite" - model_checksum: "66bb4eba50987221608f8487ed405794" - batch_size: 2 + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "NNAPI" } benchmark_setting { - benchmark_id: "image_classification_v2" + benchmark_id: "image_classification_offline" framework: "TFLite" delegate_choice: { delegate_name: "NNAPI" accelerator_name: "tpu" accelerator_desc: "Google EdgeTPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/tflite/mobilenet_edgetpu_224_1.0_uint8.tflite" + model_checksum: "008dfcb1c1962fedbeef1b998d4c84f2" + batch_size: 64 } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_models/raw/main/v1_1/tflite/mobilenet_edgetpu_224_1.0_float.tflite" + model_checksum: "66bb4eba50987221608f8487ed405794" + batch_size: 2 } delegate_selected: "NNAPI" } @@ -79,16 +79,16 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "tpu" accelerator_desc: "Google EdgeTPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" batch_size: 16 + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 2 } delegate_selected: "NNAPI" @@ -112,7 +112,6 @@ benchmark_setting { model_checksum: "566ceb72a4c7c8926fe4ac8eededb5bf" } delegate_selected: "NNAPI" - } benchmark_setting { diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt index 3bbe1ae17..7f0b3b8a6 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_android.pbtxt @@ -38,15 +38,15 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "NNAPI" } @@ -80,16 +80,16 @@ benchmark_setting { delegate_name: "NNAPI" accelerator_name: "npu" accelerator_desc: "NPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_int8.tflite" - model_checksum: "60f651765fa0d52b468f2e1c15c47f91" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" batch_size: 2 } delegate_choice: { delegate_name: "GPU" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 2 } delegate_selected: "NNAPI" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt index 8669e7fb4..4b60e43c8 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone11.pbtxt @@ -31,16 +31,16 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 64 } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 64 } delegate_selected: "Core ML" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt index 44b482330..2b77ca063 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphone12.pbtxt @@ -31,16 +31,16 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 8 } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 8 } delegate_selected: "Core ML" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt index f90255d22..6285f4eeb 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_iphoneX.pbtxt @@ -31,16 +31,16 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 32 } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" batch_size: 32 } delegate_selected: "Core ML" diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt index 42cd0e2d1..355aa3d42 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_apple_main.pbtxt @@ -38,15 +38,15 @@ benchmark_setting { delegate_name: "Core ML" accelerator_name: "ane" accelerator_desc: "ANE" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_choice: { delegate_name: "Metal" accelerator_name: "gpu" accelerator_desc: "GPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "Core ML" } diff --git a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt index 2eae34450..6bfad7fd5 100644 --- a/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/backend_settings/tflite_settings_windows.pbtxt @@ -45,8 +45,8 @@ benchmark_setting { delegate_name: "CPU" accelerator_name: "cpu" accelerator_desc: "CPU" - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "CPU" } @@ -59,8 +59,8 @@ benchmark_setting { accelerator_name: "cpu" accelerator_desc: "CPU" batch_size: 2 - model_path: "https://github.com/mlcommons/mobile_models/raw/anh/add-MNEdgeTPU-V2-L/v3_1/tflite/mobilenet-edgetpu-v2-L_fp32.tflite" - model_checksum: "4b7ce0a82f8eaa542404d2ab2ee4e575" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" } delegate_selected: "CPU" } diff --git a/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt b/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt index 118e8609a..eddd47301 100644 --- a/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt +++ b/mobile_back_tflite/cpp/backend_tflite/neuron/backend_settings/tflite_settings_mtk.pbtxt @@ -35,6 +35,33 @@ benchmark_setting { delegate_selected: "Neuron" } +benchmark_setting { + benchmark_id: "image_classification_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "Neuron" + accelerator_name: "neuron-mdla" + accelerator_desc: "MediaTek NN accelerator via the Neuron Delegate" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + } + delegate_choice: { + delegate_name: "NNAPI" + accelerator_name: "npu" + accelerator_desc: "NPU" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" + } + delegate_selected: "Neuron" +} + benchmark_setting { benchmark_id: "image_classification_offline" framework: "TFLite" @@ -63,6 +90,34 @@ benchmark_setting { delegate_selected: "Neuron" } +benchmark_setting { + benchmark_id: "image_classification_offline_v2" + framework: "TFLite" + delegate_choice: { + delegate_name: "Neuron" + accelerator_name: "neuron-mdla" + accelerator_desc: "MediaTek NN accelerator via the Neuron Delegate" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + batch_size: 256 + } + delegate_choice: { + delegate_name: "NNAPI" + accelerator_name: "npu" + accelerator_desc: "NPU" + model_path: "https://github.com/mlcommons/mobile_open/raw/main/vision/mobilenetV4/MobileNetV4-Conv-Large-int8-ptq.tflite" + model_checksum: "590a7a88640a18d28b16b6f571cdfc93" + } + delegate_choice: { + delegate_name: "GPU" + accelerator_name: "gpu" + accelerator_desc: "GPU" + model_path: "https://github.com/mlcommons/mobile_open/releases/download/model_upload/MobileNetV4-Conv-Large-fp32.tflite" + model_checksum: "b57cc2a027607c3b36873a15ace84acb" + } + delegate_selected: "Neuron" +} + benchmark_setting { benchmark_id: "object_detection" framework: "TFLite" From 89991f580766d86ce666a46b9a98d81d769f40a9 Mon Sep 17 00:00:00 2001 From: Anh Date: Thu, 29 Feb 2024 15:42:25 +0700 Subject: [PATCH 11/20] fix: performance drop caused by UI animation (#857) * Remove InfiniteProgressCircle * Use GitHub vars for --num-flaky-test-attempts flag * Format Dart code * Add DottedProgressCircle * Add setting for progress animation style * Revert "Add setting for progress animation style" This reverts commit c14cc6309b5cb4d4e75b2d1e5a899cb385a5ef82. * Revert "Add DottedProgressCircle" This reverts commit ef1c42795fe563ee87d2146a1eb177840c144912. * Use ProgressCircle with no animation * Format Dart code --- .github/workflows/android-build-test.yml | 8 +- .../lib/ui/home/benchmark_running_screen.dart | 12 +- .../gradient_circular_progress_indicator.dart | 106 ------------------ flutter/lib/ui/home/progress_circle.dart | 62 ++++++++++ flutter/lib/ui/home/progress_circles.dart | 54 --------- 5 files changed, 72 insertions(+), 170 deletions(-) delete mode 100644 flutter/lib/ui/home/gradient_circular_progress_indicator.dart create mode 100644 flutter/lib/ui/home/progress_circle.dart delete mode 100644 flutter/lib/ui/home/progress_circles.dart diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index 8ced2cd29..ccf84647c 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -204,7 +204,7 @@ jobs: --app $GCLOUD_BUCKET_PATH/test-main-tflite.apk \ --test $GCLOUD_BUCKET_PATH/test-helper-tflite.apk \ --timeout 30m \ - --num-flaky-test-attempts 2 \ + --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ --device model=redfin,version=30,locale=en,orientation=portrait \ --client-details=buildNumber=${{ github.run_number }} @@ -229,7 +229,7 @@ jobs: --app $GCLOUD_BUCKET_PATH/test-main-pixel.apk \ --test $GCLOUD_BUCKET_PATH/test-helper-pixel.apk \ --timeout 30m \ - --num-flaky-test-attempts 2 \ + --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ --device model=oriole,version=32,locale=en,orientation=portrait \ --client-details=buildNumber=${{ github.run_number }} @@ -254,7 +254,7 @@ jobs: --app $GCLOUD_BUCKET_PATH/test-main-qti.apk \ --test $GCLOUD_BUCKET_PATH/test-helper-qti.apk \ --timeout 30m \ - --num-flaky-test-attempts 2 \ + --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ --device model=b0q,version=33,locale=en,orientation=portrait \ --client-details=buildNumber=${{ github.run_number }} @@ -279,6 +279,6 @@ jobs: --app $GCLOUD_BUCKET_PATH/test-main-mtk.apk \ --test $GCLOUD_BUCKET_PATH/test-helper-mtk.apk \ --timeout 30m \ - --num-flaky-test-attempts 2 \ + --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ --device model=OP515BL1,version=33,locale=en,orientation=portrait \ --client-details=buildNumber=${{ github.run_number }} diff --git a/flutter/lib/ui/home/benchmark_running_screen.dart b/flutter/lib/ui/home/benchmark_running_screen.dart index 05b590ca8..3113e918f 100644 --- a/flutter/lib/ui/home/benchmark_running_screen.dart +++ b/flutter/lib/ui/home/benchmark_running_screen.dart @@ -13,7 +13,7 @@ import 'package:mlperfbench/localizations/app_localizations.dart'; import 'package:mlperfbench/state/task_runner.dart'; import 'package:mlperfbench/ui/app_styles.dart'; import 'package:mlperfbench/ui/formatter.dart'; -import 'package:mlperfbench/ui/home/progress_circles.dart'; +import 'package:mlperfbench/ui/home/progress_circle.dart'; import 'package:mlperfbench/ui/icons.dart'; class BenchmarkRunningScreen extends StatefulWidget { @@ -114,9 +114,9 @@ class _BenchmarkRunningScreenState extends State { ), ), ), - InfiniteProgressCircle( + ProgressCircle( + strokeWidth: 6, size: containerWidth + 20, - strokeWidth: 6.0, ), ], ); @@ -201,9 +201,9 @@ class _BenchmarkRunningScreenState extends State { const trailingWidth = 24.0; Widget? doneIcon; if (progress.currentBenchmark?.taskName == benchmarkInfo.taskName) { - doneIcon = const InfiniteProgressCircle( - size: trailingWidth, + doneIcon = const ProgressCircle( strokeWidth: 2, + size: trailingWidth - 4, ); } else if (progress.completedBenchmarks.contains(benchmarkInfo)) { doneIcon = const Icon( @@ -238,7 +238,7 @@ class _BenchmarkRunningScreenState extends State { trailing: SizedBox( width: trailingWidth, height: trailingWidth, - child: doneIcon, + child: Center(child: doneIcon), ), ); } diff --git a/flutter/lib/ui/home/gradient_circular_progress_indicator.dart b/flutter/lib/ui/home/gradient_circular_progress_indicator.dart deleted file mode 100644 index 9fa879ce4..000000000 --- a/flutter/lib/ui/home/gradient_circular_progress_indicator.dart +++ /dev/null @@ -1,106 +0,0 @@ -// Source: https://github.com/mryadavdilip/gradient_circular_progress_indicator/blob/e3621c0744bd1b88b00ae4e6a6068c8197789ef6/lib/gradient_circular_progress_indicator.dart - -import 'dart:math'; - -import 'package:flutter/material.dart'; - -class GradientCircularProgressIndicator extends StatelessWidget { - /// progress value between or equal to 0 and 1 (0 <= progress <= 1) - final double progress; - final Gradient gradient; - - /// default background Colors.transparent - final Color? backgroundColor; - - /// default stroke is (size of child divided by 10) - final double? stroke; - - /// default size is size of child - final double? size; - final Widget? child; - - const GradientCircularProgressIndicator({ - super.key, - required this.progress, - required this.gradient, - this.backgroundColor, - this.stroke, - this.size, - this.child, - }); - - @override - Widget build(BuildContext context) { - return CustomPaint( - size: size != null ? Size(size!, size!) : MediaQuery.of(context).size, - painter: _GradientCircularProgressPainter( - progress: progress, - gradient: gradient, - backgroundColor: backgroundColor ?? Colors.transparent, - stroke: stroke), - child: SizedBox( - height: size, - width: size, - child: child, - ), - ); - } -} - -class _GradientCircularProgressPainter extends CustomPainter { - final double progress; - final Gradient gradient; - final Color backgroundColor; // New parameter for background color - final double? stroke; - - _GradientCircularProgressPainter({ - required this.progress, - required this.gradient, - required this.backgroundColor, - this.stroke, - }); - - @override - void paint(Canvas canvas, Size size) { - final center = Offset(size.width / 2, size.height / 2); - final radius = size.width / 2; - const startAngle = -pi / 2; - const fullSweepAngle = 2 * pi; - final progressSweepAngle = 2 * pi * progress; - - final backgroundPaint = Paint() - ..color = backgroundColor // Set the background color - ..style = PaintingStyle.stroke - ..strokeWidth = stroke ?? size.width / 10; - - final gradientPaint = Paint() - ..shader = - gradient.createShader(Rect.fromCircle(center: center, radius: radius)) - ..strokeCap = StrokeCap.round - ..style = PaintingStyle.stroke - ..strokeWidth = stroke ?? size.width / 10; - - // Draw the background arc - canvas.drawArc( - Rect.fromCircle(center: center, radius: radius), - startAngle + progressSweepAngle, - fullSweepAngle - progressSweepAngle, - false, - backgroundPaint, - ); - - // Draw the gradient arc - canvas.drawArc( - Rect.fromCircle(center: center, radius: radius), - startAngle, - progressSweepAngle, - false, - gradientPaint, - ); - } - - @override - bool shouldRepaint(covariant CustomPainter oldDelegate) { - return true; - } -} diff --git a/flutter/lib/ui/home/progress_circle.dart b/flutter/lib/ui/home/progress_circle.dart new file mode 100644 index 000000000..1762bada0 --- /dev/null +++ b/flutter/lib/ui/home/progress_circle.dart @@ -0,0 +1,62 @@ +import 'package:flutter/material.dart'; + +class ProgressCircle extends StatelessWidget { + final double strokeWidth; + final double size; + + const ProgressCircle({ + super.key, + required this.size, + required this.strokeWidth, + }); + + @override + Widget build(BuildContext context) { + return _BorderedCircle( + strokeWidth: strokeWidth, + radius: size / 2, + color: Colors.white.withOpacity(0.88), + ); + } +} + +class _BorderedCircle extends StatelessWidget { + final double strokeWidth; + final double radius; + final Color color; + + const _BorderedCircle({ + required this.strokeWidth, + required this.radius, + required this.color, + }); + + @override + Widget build(BuildContext context) { + return CustomPaint( + painter: DrawCircle(strokeWidth, radius, color), + ); + } +} + +class DrawCircle extends CustomPainter { + final double strokeWidth; + final double radius; + final Color color; + + DrawCircle(this.strokeWidth, this.radius, this.color); + + @override + void paint(Canvas canvas, Size size) { + final paint = Paint() + ..color = color + ..strokeWidth = strokeWidth + ..style = PaintingStyle.stroke; + canvas.drawCircle(const Offset(0.0, 0.0), radius, paint); + } + + @override + bool shouldRepaint(CustomPainter oldDelegate) { + return false; + } +} diff --git a/flutter/lib/ui/home/progress_circles.dart b/flutter/lib/ui/home/progress_circles.dart deleted file mode 100644 index b22a86d7b..000000000 --- a/flutter/lib/ui/home/progress_circles.dart +++ /dev/null @@ -1,54 +0,0 @@ -import 'package:flutter/material.dart'; - -import 'package:mlperfbench/ui/home/gradient_circular_progress_indicator.dart'; - -class InfiniteProgressCircle extends StatefulWidget { - final double size; - final double strokeWidth; - - const InfiniteProgressCircle( - {Key? key, required this.size, required this.strokeWidth}) - : super(key: key); - - @override - State createState() => _InfiniteProgressCircleState(); -} - -class _InfiniteProgressCircleState extends State - with SingleTickerProviderStateMixin { - late AnimationController _controller; - - @override - void initState() { - _controller = - AnimationController(vsync: this, duration: const Duration(seconds: 2)); - _controller.addListener(() => setState(() {})); - _controller.repeat(); - super.initState(); - } - - @override - void dispose() { - _controller.dispose(); - super.dispose(); - } - - @override - Widget build(BuildContext context) { - return RotationTransition( - turns: Tween(begin: 0.0, end: 1.0).animate(_controller), - child: GradientCircularProgressIndicator( - progress: 1.0, - // Specify the progress value between 0 and 1 - gradient: const LinearGradient( - colors: [Colors.white, Colors.white, Colors.white10, Colors.white10], - begin: Alignment.topLeft, - end: Alignment.bottomRight, - ), - backgroundColor: Colors.transparent, - stroke: widget.strokeWidth, - size: widget.size, - ), - ); - } -} From bd79e128d1d22fa150ec999ede20d838a5e8b001 Mon Sep 17 00:00:00 2001 From: Anh Date: Fri, 1 Mar 2024 15:23:14 +0700 Subject: [PATCH 12/20] ci: use new device for test-android-apk-tflite (#860) * Use GitHub vars for --num-flaky-test-attempts flag * Use Pixel 6 for test-android-apk-tflite * Add expected_throughput for _kTFLiteBackend -> _kPixel6 * Update expected_throughput for _kTFLiteBackend -> _kPixel6 * Replace obsolete Windows image * Update expected_throughput for _kDN2103 * Use Pixel 6 - API level 31 for higher device capacity --- .github/cloudbuild/flutter-windows.yaml | 2 +- .github/workflows/android-build-test.yml | 4 ++-- .../integration_test/expected_throughput.dart | 18 ++++++++++++------ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/cloudbuild/flutter-windows.yaml b/.github/cloudbuild/flutter-windows.yaml index 44cc66a8f..4ce58a010 100644 --- a/.github/cloudbuild/flutter-windows.yaml +++ b/.github/cloudbuild/flutter-windows.yaml @@ -34,7 +34,7 @@ steps: - --region=$_CE_REGION - --zone=$_CE_ZONE - --machineType=$_CE_MACHINE_TYPE - - --image=projects/gce-uefi-images/global/images/windows-server-2019-dc-for-containers-v20200609 + - --image=projects/gce-uefi-images/global/images/windows-server-2019-dc-for-containers-v20200512 # When using other disk types image pulling and image building could take up to 3 hours - --diskType=pd-ssd - --workspace-bucket=$_WORKSPACE_TMP_BUCKET diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index ccf84647c..793adc06f 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -197,7 +197,7 @@ jobs: with: version: '>= 363.0.0' project_id: mobile-app-build-290400 - - name: Trigger Firebase Test Lab for [Google Pixel 5](redfin) + - name: Trigger Firebase Test Lab for [Google Pixel 6](oriole) run: | gcloud firebase test android run \ --type instrumentation \ @@ -205,7 +205,7 @@ jobs: --test $GCLOUD_BUCKET_PATH/test-helper-tflite.apk \ --timeout 30m \ --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ - --device model=redfin,version=30,locale=en,orientation=portrait \ + --device model=oriole,version=31,locale=en,orientation=portrait \ --client-details=buildNumber=${{ github.run_number }} test-android-apk-pixel: diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 10854dfdb..092b9fb00 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -35,6 +35,7 @@ const Map> _imageClassification = { _kCloudBuildX28: Interval(min: 4, max: 13), _kRyzen5600: Interval(min: 31, max: 37), _kPixel5: Interval(min: 80, max: 130), + _kPixel6: Interval(min: 600, max: 1100), _kIphoneOnGitHubAction: Interval(min: 1, max: 8), _kIphoneOnMacbookM1: Interval(min: 19, max: 27), }, @@ -48,7 +49,7 @@ const Map> _imageClassification = { _kS22Ultra: Interval(min: 1900, max: 2400), }, _kMediatekBackend: { - _kDN2103: Interval(min: 30, max: 50), + _kDN2103: Interval(min: 30, max: 55), }, }; @@ -58,6 +59,7 @@ const Map> _objectDetection = { _kCloudBuildX28: Interval(min: 3.5, max: 8), _kRyzen5600: Interval(min: 14, max: 22), _kPixel5: Interval(min: 40, max: 60), + _kPixel6: Interval(min: 200, max: 450), _kIphoneOnGitHubAction: Interval(min: 0.5, max: 4), _kIphoneOnMacbookM1: Interval(min: 9, max: 16), }, @@ -71,7 +73,7 @@ const Map> _objectDetection = { _kS22Ultra: Interval(min: 800, max: 1400), }, _kMediatekBackend: { - _kDN2103: Interval(min: 120, max: 180), + _kDN2103: Interval(min: 120, max: 210), }, }; @@ -81,6 +83,7 @@ const Map> _imageSegmentationV2 = { _kCloudBuildX28: Interval(min: 0.5, max: 4), _kRyzen5600: Interval(min: 5, max: 7), _kPixel5: Interval(min: 25, max: 40), + _kPixel6: Interval(min: 80, max: 180), _kIphoneOnGitHubAction: Interval(min: 0.1, max: 2.5), _kIphoneOnMacbookM1: Interval(min: 3, max: 6), }, @@ -94,7 +97,7 @@ const Map> _imageSegmentationV2 = { _kS22Ultra: Interval(min: 450, max: 700), }, _kMediatekBackend: { - _kDN2103: Interval(min: 45, max: 65), + _kDN2103: Interval(min: 45, max: 70), }, }; @@ -104,6 +107,7 @@ const Map> _naturalLanguageProcessing = { _kCloudBuildX28: Interval(min: 0.5, max: 1.3), _kRyzen5600: Interval(min: 2.8, max: 3.2), _kPixel5: Interval(min: 2.3, max: 3.0), + _kPixel6: Interval(min: 2, max: 75), _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1), _kIphoneOnMacbookM1: Interval(min: 1.8, max: 3), }, @@ -118,7 +122,7 @@ const Map> _naturalLanguageProcessing = { _kS22Ultra: Interval(min: 120, max: 180), }, _kMediatekBackend: { - _kDN2103: Interval(min: 1, max: 5), + _kDN2103: Interval(min: 1, max: 6), }, }; @@ -128,6 +132,7 @@ const Map> _superResolution = { _kCloudBuildX28: Interval(min: 0.1, max: 4), _kRyzen5600: Interval(min: 0.1, max: 3), _kPixel5: Interval(min: 4, max: 8), + _kPixel6: Interval(min: 7, max: 14), _kIphoneOnGitHubAction: Interval(min: 0.02, max: 1.0), _kIphoneOnMacbookM1: Interval(min: 0.1, max: 10), }, @@ -141,7 +146,7 @@ const Map> _superResolution = { _kS22Ultra: Interval(min: 35, max: 55), }, _kMediatekBackend: { - _kDN2103: Interval(min: 5, max: 12), + _kDN2103: Interval(min: 5, max: 15), }, }; @@ -151,6 +156,7 @@ const Map> _imageClassificationOffline = { _kCloudBuildX28: Interval(min: 7, max: 16), _kRyzen5600: Interval(min: 45, max: 60), _kPixel5: Interval(min: 120, max: 190), + _kPixel6: Interval(min: 800, max: 1700), _kIphoneOnGitHubAction: Interval(min: 2, max: 15), _kIphoneOnMacbookM1: Interval(min: 30, max: 45), }, @@ -164,7 +170,7 @@ const Map> _imageClassificationOffline = { _kS22Ultra: Interval(min: 2600, max: 3500), }, _kMediatekBackend: { - _kDN2103: Interval(min: 75, max: 110), + _kDN2103: Interval(min: 75, max: 140), }, }; From 3e19125185248b96db469d51834dedafe3213366 Mon Sep 17 00:00:00 2001 From: Anh Date: Fri, 1 Mar 2024 15:24:22 +0700 Subject: [PATCH 13/20] ci: replace obsolete Windows image (#861) Replace obsolete Windows image From 8455a4169c21c12e7505883d783763ea112c0403 Mon Sep 17 00:00:00 2001 From: Anh Date: Fri, 1 Mar 2024 15:37:08 +0700 Subject: [PATCH 14/20] Revert change in android-build-test.yml --- .github/workflows/android-build-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml index c0fd3e3e9..793adc06f 100644 --- a/.github/workflows/android-build-test.yml +++ b/.github/workflows/android-build-test.yml @@ -205,7 +205,7 @@ jobs: --test $GCLOUD_BUCKET_PATH/test-helper-tflite.apk \ --timeout 30m \ --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \ - --device model=redfin,version=30,locale=en,orientation=portrait \ + --device model=oriole,version=31,locale=en,orientation=portrait \ --client-details=buildNumber=${{ github.run_number }} test-android-apk-pixel: From aecaa00f7962ed996c3dc7aa717329342be60652 Mon Sep 17 00:00:00 2001 From: Anh Date: Fri, 1 Mar 2024 16:13:50 +0700 Subject: [PATCH 15/20] Update expected_throughput for _kIphoneOnGitHubAction --- .../integration_test/expected_throughput.dart | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 840e73dc5..8f473985e 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -36,11 +36,11 @@ const Map> _imageClassification = { _kRyzen5600: Interval(min: 31, max: 37), _kPixel5: Interval(min: 80, max: 130), _kPixel6: Interval(min: 600, max: 1100), - _kIphoneOnGitHubAction: Interval(min: 1, max: 8), + _kIphoneOnGitHubAction: Interval(min: 1, max: 12), _kIphoneOnMacbookM1: Interval(min: 19, max: 27), }, _kCoreMLBackend: { - _kIphoneOnGitHubAction: Interval(min: 1, max: 8), + _kIphoneOnGitHubAction: Interval(min: 1, max: 12), }, _kPixelBackend: { _kPixel6: Interval(min: 800, max: 1100), @@ -60,11 +60,11 @@ const Map> _imageClassificationV2 = { _kCloudBuildX28: Interval(min: 1, max: 9), _kRyzen5600: Interval(min: 1, max: 37), _kPixel5: Interval(min: 20, max: 75), - _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 9), _kIphoneOnMacbookM1: Interval(min: 10, max: 27), }, _kCoreMLBackend: { - _kIphoneOnGitHubAction: Interval(min: 0.4, max: 7), + _kIphoneOnGitHubAction: Interval(min: 0.4, max: 9), }, _kPixelBackend: { _kPixel6: Interval(min: 100, max: 600), @@ -84,11 +84,11 @@ const Map> _objectDetection = { _kRyzen5600: Interval(min: 14, max: 22), _kPixel5: Interval(min: 40, max: 60), _kPixel6: Interval(min: 200, max: 450), - _kIphoneOnGitHubAction: Interval(min: 0.5, max: 4), + _kIphoneOnGitHubAction: Interval(min: 0.5, max: 7), _kIphoneOnMacbookM1: Interval(min: 9, max: 16), }, _kCoreMLBackend: { - _kIphoneOnGitHubAction: Interval(min: 0.5, max: 4), + _kIphoneOnGitHubAction: Interval(min: 0.5, max: 7), }, _kPixelBackend: { _kPixel6: Interval(min: 200, max: 450), @@ -108,11 +108,11 @@ const Map> _imageSegmentationV2 = { _kRyzen5600: Interval(min: 5, max: 7), _kPixel5: Interval(min: 25, max: 40), _kPixel6: Interval(min: 80, max: 180), - _kIphoneOnGitHubAction: Interval(min: 0.1, max: 2.5), + _kIphoneOnGitHubAction: Interval(min: 0.1, max: 3.5), _kIphoneOnMacbookM1: Interval(min: 3, max: 6), }, _kCoreMLBackend: { - _kIphoneOnGitHubAction: Interval(min: 0.1, max: 2.5), + _kIphoneOnGitHubAction: Interval(min: 0.1, max: 3.5), }, _kPixelBackend: { _kPixel6: Interval(min: 100, max: 180), @@ -132,11 +132,11 @@ const Map> _naturalLanguageProcessing = { _kRyzen5600: Interval(min: 2.8, max: 3.2), _kPixel5: Interval(min: 2.3, max: 3.0), _kPixel6: Interval(min: 2, max: 75), - _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1), + _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1.1), _kIphoneOnMacbookM1: Interval(min: 1.8, max: 3), }, _kCoreMLBackend: { - _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1), + _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1.1), }, _kPixelBackend: { // pixel some time finish this task in 4 seconds, not sure why. @@ -181,7 +181,7 @@ const Map> _imageClassificationOffline = { _kRyzen5600: Interval(min: 45, max: 60), _kPixel5: Interval(min: 120, max: 190), _kPixel6: Interval(min: 800, max: 1700), - _kIphoneOnGitHubAction: Interval(min: 2, max: 15), + _kIphoneOnGitHubAction: Interval(min: 2, max: 20), _kIphoneOnMacbookM1: Interval(min: 30, max: 45), }, _kCoreMLBackend: { From 430253268cb5a7d29134932d2382a4c254fafb66 Mon Sep 17 00:00:00 2001 From: Anh Date: Fri, 1 Mar 2024 17:39:17 +0700 Subject: [PATCH 16/20] Update expected_throughput for _kPixel6 --- .../integration_test/expected_throughput.dart | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 8f473985e..220a30403 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -35,7 +35,7 @@ const Map> _imageClassification = { _kCloudBuildX28: Interval(min: 4, max: 13), _kRyzen5600: Interval(min: 31, max: 37), _kPixel5: Interval(min: 80, max: 130), - _kPixel6: Interval(min: 600, max: 1100), + _kPixel6: Interval(min: 600, max: 1200), _kIphoneOnGitHubAction: Interval(min: 1, max: 12), _kIphoneOnMacbookM1: Interval(min: 19, max: 27), }, @@ -43,7 +43,7 @@ const Map> _imageClassification = { _kIphoneOnGitHubAction: Interval(min: 1, max: 12), }, _kPixelBackend: { - _kPixel6: Interval(min: 800, max: 1100), + _kPixel6: Interval(min: 800, max: 1200), }, _kQtiBackend: { _kS22Ultra: Interval(min: 1900, max: 2400), @@ -53,13 +53,13 @@ const Map> _imageClassification = { }, }; -// TODO (anhappdev): update the expected value for _imageClassificationV2 after gathering some statistics const Map> _imageClassificationV2 = { _kTFLiteBackend: { _kCloudBuildX23: Interval(min: 1, max: 9), _kCloudBuildX28: Interval(min: 1, max: 9), _kRyzen5600: Interval(min: 1, max: 37), _kPixel5: Interval(min: 20, max: 75), + _kPixel6: Interval(min: 100, max: 600), _kIphoneOnGitHubAction: Interval(min: 0.4, max: 9), _kIphoneOnMacbookM1: Interval(min: 10, max: 27), }, @@ -83,7 +83,7 @@ const Map> _objectDetection = { _kCloudBuildX28: Interval(min: 3.5, max: 8), _kRyzen5600: Interval(min: 14, max: 22), _kPixel5: Interval(min: 40, max: 60), - _kPixel6: Interval(min: 200, max: 450), + _kPixel6: Interval(min: 200, max: 500), _kIphoneOnGitHubAction: Interval(min: 0.5, max: 7), _kIphoneOnMacbookM1: Interval(min: 9, max: 16), }, @@ -91,7 +91,7 @@ const Map> _objectDetection = { _kIphoneOnGitHubAction: Interval(min: 0.5, max: 7), }, _kPixelBackend: { - _kPixel6: Interval(min: 200, max: 450), + _kPixel6: Interval(min: 200, max: 500), }, _kQtiBackend: { _kS22Ultra: Interval(min: 800, max: 1400), @@ -107,7 +107,7 @@ const Map> _imageSegmentationV2 = { _kCloudBuildX28: Interval(min: 0.5, max: 4), _kRyzen5600: Interval(min: 5, max: 7), _kPixel5: Interval(min: 25, max: 40), - _kPixel6: Interval(min: 80, max: 180), + _kPixel6: Interval(min: 80, max: 190), _kIphoneOnGitHubAction: Interval(min: 0.1, max: 3.5), _kIphoneOnMacbookM1: Interval(min: 3, max: 6), }, @@ -115,7 +115,7 @@ const Map> _imageSegmentationV2 = { _kIphoneOnGitHubAction: Interval(min: 0.1, max: 3.5), }, _kPixelBackend: { - _kPixel6: Interval(min: 100, max: 180), + _kPixel6: Interval(min: 100, max: 190), }, _kQtiBackend: { _kS22Ultra: Interval(min: 450, max: 700), @@ -131,7 +131,7 @@ const Map> _naturalLanguageProcessing = { _kCloudBuildX28: Interval(min: 0.5, max: 1.3), _kRyzen5600: Interval(min: 2.8, max: 3.2), _kPixel5: Interval(min: 2.3, max: 3.0), - _kPixel6: Interval(min: 2, max: 75), + _kPixel6: Interval(min: 20, max: 85), _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1.1), _kIphoneOnMacbookM1: Interval(min: 1.8, max: 3), }, @@ -139,8 +139,7 @@ const Map> _naturalLanguageProcessing = { _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1.1), }, _kPixelBackend: { - // pixel some time finish this task in 4 seconds, not sure why. - _kPixel6: Interval(min: 2, max: 75), + _kPixel6: Interval(min: 20, max: 85), }, _kQtiBackend: { _kS22Ultra: Interval(min: 120, max: 180), @@ -156,7 +155,7 @@ const Map> _superResolution = { _kCloudBuildX28: Interval(min: 0.1, max: 4), _kRyzen5600: Interval(min: 0.1, max: 3), _kPixel5: Interval(min: 4, max: 8), - _kPixel6: Interval(min: 7, max: 14), + _kPixel6: Interval(min: 7, max: 17), _kIphoneOnGitHubAction: Interval(min: 0.02, max: 1.0), _kIphoneOnMacbookM1: Interval(min: 0.1, max: 10), }, @@ -164,7 +163,7 @@ const Map> _superResolution = { _kIphoneOnGitHubAction: Interval(min: 0.02, max: 1.0), }, _kPixelBackend: { - _kPixel6: Interval(min: 10, max: 14), + _kPixel6: Interval(min: 7, max: 17), }, _kQtiBackend: { _kS22Ultra: Interval(min: 35, max: 55), @@ -180,7 +179,7 @@ const Map> _imageClassificationOffline = { _kCloudBuildX28: Interval(min: 7, max: 16), _kRyzen5600: Interval(min: 45, max: 60), _kPixel5: Interval(min: 120, max: 190), - _kPixel6: Interval(min: 800, max: 1700), + _kPixel6: Interval(min: 700, max: 1800), _kIphoneOnGitHubAction: Interval(min: 2, max: 20), _kIphoneOnMacbookM1: Interval(min: 30, max: 45), }, @@ -188,7 +187,7 @@ const Map> _imageClassificationOffline = { _kIphoneOnGitHubAction: Interval(min: 2, max: 20), }, _kPixelBackend: { - _kPixel6: Interval(min: 1000, max: 1700), + _kPixel6: Interval(min: 900, max: 1800), }, _kQtiBackend: { _kS22Ultra: Interval(min: 2600, max: 3500), @@ -198,13 +197,13 @@ const Map> _imageClassificationOffline = { }, }; -// TODO (anhappdev): update the expected value for _imageClassificationOfflineV2 after gathering some statistics const Map> _imageClassificationOfflineV2 = { _kTFLiteBackend: { _kCloudBuildX23: Interval(min: 1, max: 9), _kCloudBuildX28: Interval(min: 1, max: 9), _kRyzen5600: Interval(min: 20, max: 60), _kPixel5: Interval(min: 20, max: 180), + _kPixel6: Interval(min: 100, max: 700), _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), _kIphoneOnMacbookM1: Interval(min: 10, max: 45), }, @@ -212,7 +211,7 @@ const Map> _imageClassificationOfflineV2 = { _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15), }, _kPixelBackend: { - _kPixel6: Interval(min: 100, max: 1700), + _kPixel6: Interval(min: 100, max: 700), }, _kQtiBackend: { _kS22Ultra: Interval(min: 2600, max: 3000), From b703cf4d4be906942e9a8feaa2691fe2c94fca0a Mon Sep 17 00:00:00 2001 From: Anh Date: Fri, 1 Mar 2024 18:41:43 +0700 Subject: [PATCH 17/20] Update expected_throughput for _kPixel6 --- flutter/integration_test/expected_throughput.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 220a30403..51eaa4b51 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -131,7 +131,7 @@ const Map> _naturalLanguageProcessing = { _kCloudBuildX28: Interval(min: 0.5, max: 1.3), _kRyzen5600: Interval(min: 2.8, max: 3.2), _kPixel5: Interval(min: 2.3, max: 3.0), - _kPixel6: Interval(min: 20, max: 85), + _kPixel6: Interval(min: 2, max: 85), _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1.1), _kIphoneOnMacbookM1: Interval(min: 1.8, max: 3), }, @@ -139,7 +139,7 @@ const Map> _naturalLanguageProcessing = { _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1.1), }, _kPixelBackend: { - _kPixel6: Interval(min: 20, max: 85), + _kPixel6: Interval(min: 2, max: 85), }, _kQtiBackend: { _kS22Ultra: Interval(min: 120, max: 180), From 5ef666daf1d88c2d46f2f5aee08810d84413e978 Mon Sep 17 00:00:00 2001 From: Anh Date: Mon, 4 Mar 2024 14:49:37 +0700 Subject: [PATCH 18/20] feat: add flag enforce_max_duration (#798) * Use dev version of loadgen * Set enforce_max_duration to false * Add min_duration_ms and max_duration_ms to dev Makefile * update enforce_max_duration logic * use updated loadgen * Show different colors for performance result based on multiple conditions. * Fix Dart linter issue * Use GitHub vars for --num-flaky-test-attempts flag * Update expected_throughput for _kS22Ultra * Use latest commit from inference/mobile_update branch --------- Co-authored-by: Koan-Sin Tan --- WORKSPACE | 10 ++--- docs/result-spec.md | 20 ++++++--- flutter/cpp/mlperf_driver.cc | 1 + .../integration_test/expected_throughput.dart | 12 ++--- flutter/lib/backend/loadgen_info.dart | 44 ++++++++++++++----- flutter/lib/benchmark/benchmark.dart | 24 +++++----- .../edit_json_schema.main.dart | 2 +- .../generation_helpers/sample_generator.dart | 12 +++-- .../lib/data/results/benchmark_result.dart | 20 ++------- .../lib/resources/export_result_helper.dart | 13 +----- flutter/lib/resources/result_manager.dart | 2 +- flutter/lib/state/task_runner.dart | 27 ++++++++---- flutter/lib/ui/app_styles.dart | 1 + .../benchmark_export_result_screen.dart | 2 +- .../ui/history/extended_result_screen.dart | 3 +- .../lib/ui/home/benchmark_result_screen.dart | 34 ++++++++++---- .../unit_test/backend/loadgen_info_test.dart | 16 ++++++- mobile_back_apple/dev-utils/Makefile | 8 ++-- tools/extended_result_schema.json | 40 ++++++++++++----- 19 files changed, 185 insertions(+), 106 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 718454c3a..48d1bab2a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,6 +1,7 @@ workspace(name = "mlperf_app") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") http_archive( name = "bazel_skylib", @@ -88,17 +89,14 @@ http_archive( urls = ["https://github.com/MediaTek-NeuroPilot/tflite-neuron-delegate/archive/refs/heads/update_for_dujac.zip"], ) -http_archive( +new_git_repository( name = "org_mlperf_inference", build_file = "@//flutter/android/third_party:loadgen.BUILD", + commit = "238d035ab41d7ddd390b35471af169ea641380f6", patch_args = ["-p1"], patch_cmds = ["python3 loadgen/version_generator.py loadgen/version_generated.cc loadgen"], patches = [], - sha256 = "e664f980e84fcab3573447c0cc3adddd1fcf900367c5dcbff17179ece24c484e", - strip_prefix = "inference-2da0c52666e21e4b296b09e1dbd287bf3a814e96", - urls = [ - "https://github.com/mlcommons/inference/archive/2da0c52666e21e4b296b09e1dbd287bf3a814e96.tar.gz", - ], + remote = "https://github.com/mlcommons/inference.git", ) # This is required to pass SNPE SDK path from external environment to sources, diff --git a/docs/result-spec.md b/docs/result-spec.md index e1a1bfa84..bc6bb8b74 100644 --- a/docs/result-spec.md +++ b/docs/result-spec.md @@ -76,12 +76,22 @@ If you enable Submission mode, both `performance_run` and `accuracy_run` values Actual duration of the benchmark in seconds from start to finish. * `measured_samples`: integer number Actual number of samples evaluated during the benchmark - * `loadgen_info`: map + * `loadgen`: map Info provided by loadgen. May be null for accuracy runs. - * `validity`: bool - Indicates whether all constraints were satisfied or not. - * `duration`: floating point number - Duration of the benchmark without loadgen overhead in seconds. + * `queryCount`: bool + Number of queries performed. + * `latencyMean`: bool + Mean latency in seconds. + * `latency90`: bool + 90th percentile in seconds. + * `isMinDurationMet`: bool + Indicates whether the min duration condition is met or not. + * `isMinQueryMet`: bool + Indicates whether the min query condition is met or not. + * `isEarlyStoppingMet`: bool + Indicates whether the early stopping condition is met or not. + * `isResultValid`: bool + Indicates whether the result is valid or not. * `start_datetime`: string Datetime of the moment when benchmark started Format is Iso 8601 in UTC timezone: `2022-04-14T03:54:54.687Z` diff --git a/flutter/cpp/mlperf_driver.cc b/flutter/cpp/mlperf_driver.cc index 8dc61b649..bdf1b06f3 100644 --- a/flutter/cpp/mlperf_driver.cc +++ b/flutter/cpp/mlperf_driver.cc @@ -105,6 +105,7 @@ void MlperfDriver::RunMLPerfTest(const std::string& mode, int min_query_count, // See https://github.com/mlcommons/inference/issues/1397 mlperf_settings.max_duration_ms = static_cast(std::ceil(max_duration * 1000.0)); + mlperf_settings.enforce_max_duration = true; if (scenario_ == "Offline") { mlperf_settings.scenario = ::mlperf::TestScenario::Offline; diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index 092b9fb00..61242f643 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -46,7 +46,7 @@ const Map> _imageClassification = { _kPixel6: Interval(min: 800, max: 1100), }, _kQtiBackend: { - _kS22Ultra: Interval(min: 1900, max: 2400), + _kS22Ultra: Interval(min: 1700, max: 2400), }, _kMediatekBackend: { _kDN2103: Interval(min: 30, max: 55), @@ -70,7 +70,7 @@ const Map> _objectDetection = { _kPixel6: Interval(min: 250, max: 450), }, _kQtiBackend: { - _kS22Ultra: Interval(min: 800, max: 1400), + _kS22Ultra: Interval(min: 700, max: 1400), }, _kMediatekBackend: { _kDN2103: Interval(min: 120, max: 210), @@ -94,7 +94,7 @@ const Map> _imageSegmentationV2 = { _kPixel6: Interval(min: 100, max: 180), }, _kQtiBackend: { - _kS22Ultra: Interval(min: 450, max: 700), + _kS22Ultra: Interval(min: 400, max: 700), }, _kMediatekBackend: { _kDN2103: Interval(min: 45, max: 70), @@ -119,7 +119,7 @@ const Map> _naturalLanguageProcessing = { _kPixel6: Interval(min: 2, max: 75), }, _kQtiBackend: { - _kS22Ultra: Interval(min: 120, max: 180), + _kS22Ultra: Interval(min: 100, max: 180), }, _kMediatekBackend: { _kDN2103: Interval(min: 1, max: 6), @@ -143,7 +143,7 @@ const Map> _superResolution = { _kPixel6: Interval(min: 10, max: 14), }, _kQtiBackend: { - _kS22Ultra: Interval(min: 35, max: 55), + _kS22Ultra: Interval(min: 25, max: 55), }, _kMediatekBackend: { _kDN2103: Interval(min: 5, max: 15), @@ -167,7 +167,7 @@ const Map> _imageClassificationOffline = { _kPixel6: Interval(min: 1000, max: 1700), }, _kQtiBackend: { - _kS22Ultra: Interval(min: 2600, max: 3500), + _kS22Ultra: Interval(min: 2500, max: 3500), }, _kMediatekBackend: { _kDN2103: Interval(min: 75, max: 140), diff --git a/flutter/lib/backend/loadgen_info.dart b/flutter/lib/backend/loadgen_info.dart index 1c14906a7..0311cb4e1 100644 --- a/flutter/lib/backend/loadgen_info.dart +++ b/flutter/lib/backend/loadgen_info.dart @@ -1,19 +1,28 @@ import 'dart:convert'; import 'dart:io'; +import 'package:json_annotation/json_annotation.dart'; + +part 'loadgen_info.g.dart'; + +@JsonSerializable(fieldRename: FieldRename.snake) class LoadgenInfo { - // Mean latency in seconds - final double meanLatency; final int queryCount; - // 90th percentile in seconds - final double latency90; - final bool validity; + final double latencyMean; // Mean latency in seconds + final double latency90; // 90th percentile in seconds + final bool isMinDurationMet; + final bool isMinQueryMet; + final bool isEarlyStoppingMet; + final bool isResultValid; LoadgenInfo({ - required this.meanLatency, required this.queryCount, + required this.latencyMean, required this.latency90, - required this.validity, + required this.isMinDurationMet, + required this.isMinQueryMet, + required this.isEarlyStoppingMet, + required this.isResultValid, }); static Future fromFile({required String filepath}) { @@ -57,6 +66,10 @@ class LoadgenInfo { const latency90Key = 'result_90.00_percentile_latency_ns'; // https://github.com/mlcommons/inference/blob/318cb131c0adf3bffcbc3379a502f40891331c54/loadgen/loadgen.cc#L1028-L1029 const validityKey = 'result_validity'; + // https://github.com/mlcommons/inference/blob/318cb131c0adf3bffcbc3379a502f40891331c54/loadgen/loadgen.cc#L1033C23-L1035 + const minDurationMetKey = 'result_min_duration_met'; + const minQueriesMetKey = 'result_min_queries_met'; + const earlyStoppingMetKey = 'early_stopping_met'; final result = await extractKeys( logLines: logLines, @@ -65,6 +78,9 @@ class LoadgenInfo { queryCountKey, latency90Key, validityKey, + minDurationMetKey, + minQueriesMetKey, + earlyStoppingMetKey, }, ); @@ -72,15 +88,23 @@ class LoadgenInfo { return null; } - final validity = result[validityKey] as String == 'VALID'; + final isResultValid = result[validityKey] as String == 'VALID'; const nanosecondsPerSecond = 1000 * Duration.microsecondsPerSecond; return LoadgenInfo( - meanLatency: (result[latencyKey] as int) / nanosecondsPerSecond, queryCount: result[queryCountKey] as int, + latencyMean: (result[latencyKey] as int) / nanosecondsPerSecond, latency90: (result[latency90Key] as int) / nanosecondsPerSecond, - validity: validity, + isMinDurationMet: result[minDurationMetKey] as bool, + isMinQueryMet: result[minQueriesMetKey] as bool, + isEarlyStoppingMet: result[earlyStoppingMetKey] as bool, + isResultValid: isResultValid, ); } + + factory LoadgenInfo.fromJson(Map json) => + _$LoadgenInfoFromJson(json); + + Map toJson() => _$LoadgenInfoToJson(this); } diff --git a/flutter/lib/benchmark/benchmark.dart b/flutter/lib/benchmark/benchmark.dart index 8da2fee1d..362f0e5ff 100644 --- a/flutter/lib/benchmark/benchmark.dart +++ b/flutter/lib/benchmark/benchmark.dart @@ -2,6 +2,7 @@ import 'package:collection/collection.dart'; import 'package:mlperfbench/app_constants.dart'; import 'package:mlperfbench/backend/bridge/run_settings.dart'; +import 'package:mlperfbench/backend/loadgen_info.dart'; import 'package:mlperfbench/benchmark/info.dart'; import 'package:mlperfbench/benchmark/run_mode.dart'; import 'package:mlperfbench/data/results/benchmark_result.dart'; @@ -19,17 +20,18 @@ class BenchmarkResult { final String acceleratorName; final String delegateName; final int batchSize; - final bool validity; - - BenchmarkResult( - {required this.throughput, - required this.accuracy, - required this.accuracy2, - required this.backendName, - required this.acceleratorName, - required this.delegateName, - required this.batchSize, - required this.validity}); + final LoadgenInfo? loadgenInfo; + + BenchmarkResult({ + required this.throughput, + required this.accuracy, + required this.accuracy2, + required this.backendName, + required this.acceleratorName, + required this.delegateName, + required this.batchSize, + required this.loadgenInfo, + }); } class Benchmark { diff --git a/flutter/lib/data/generation_helpers/edit_json_schema.main.dart b/flutter/lib/data/generation_helpers/edit_json_schema.main.dart index de7915a55..c36ee5899 100644 --- a/flutter/lib/data/generation_helpers/edit_json_schema.main.dart +++ b/flutter/lib/data/generation_helpers/edit_json_schema.main.dart @@ -44,7 +44,7 @@ Future main() async { makeNullable(definitions['Run']['properties']['throughput']); makeNullable(definitions['Run']['properties']['accuracy']); makeNullable(definitions['Run']['properties']['accuracy2']); - makeNullable(definitions['Run']['properties']['loadgen_info']); + makeNullable(definitions['Run']['properties']['loadgen']); makeNullable(definitions['Value']['properties']['android']); makeNullable(definitions['Value']['properties']['ios']); makeNullable(definitions['Value']['properties']['windows']); diff --git a/flutter/lib/data/generation_helpers/sample_generator.dart b/flutter/lib/data/generation_helpers/sample_generator.dart index db63da831..9124bfc28 100644 --- a/flutter/lib/data/generation_helpers/sample_generator.dart +++ b/flutter/lib/data/generation_helpers/sample_generator.dart @@ -1,5 +1,6 @@ import 'package:uuid/uuid.dart'; +import 'package:mlperfbench/backend/loadgen_info.dart'; import 'package:mlperfbench/data/build_info/build_info.dart'; import 'package:mlperfbench/data/environment/env_android.dart'; import 'package:mlperfbench/data/environment/env_ios.dart'; @@ -37,9 +38,14 @@ class SampleGenerator { measuredDuration: 123.456, measuredSamples: 8, startDatetime: DateTime.now(), - loadgenInfo: BenchmarkLoadgenInfo( - duration: 10.6, - validity: true, + loadgenInfo: LoadgenInfo( + queryCount: 12345, + latencyMean: 0.123, + latency90: 0.123, + isMinDurationMet: true, + isMinQueryMet: true, + isEarlyStoppingMet: true, + isResultValid: true, ), ); diff --git a/flutter/lib/data/results/benchmark_result.dart b/flutter/lib/data/results/benchmark_result.dart index 1ca982a09..894526b8e 100644 --- a/flutter/lib/data/results/benchmark_result.dart +++ b/flutter/lib/data/results/benchmark_result.dart @@ -1,5 +1,6 @@ import 'package:json_annotation/json_annotation.dart'; +import 'package:mlperfbench/backend/loadgen_info.dart'; import 'package:mlperfbench/data/results/backend_info.dart'; import 'package:mlperfbench/data/results/backend_settings.dart'; import 'package:mlperfbench/data/results/dataset_info.dart'; @@ -108,22 +109,6 @@ class Accuracy implements Comparable { String toUIString() => (normalized * 100).toStringAsFixed(1); } -@JsonSerializable(fieldRename: FieldRename.snake) -class BenchmarkLoadgenInfo { - final bool validity; - final double duration; - - BenchmarkLoadgenInfo({ - required this.validity, - required this.duration, - }); - - factory BenchmarkLoadgenInfo.fromJson(Map json) => - _$BenchmarkLoadgenInfoFromJson(json); - - Map toJson() => _$BenchmarkLoadgenInfoToJson(this); -} - @JsonSerializable(fieldRename: FieldRename.snake) class BenchmarkRunResult { final Throughput? throughput; @@ -133,7 +118,8 @@ class BenchmarkRunResult { final double measuredDuration; final int measuredSamples; final DateTime startDatetime; - final BenchmarkLoadgenInfo? loadgenInfo; + @JsonKey(name: 'loadgen') + final LoadgenInfo? loadgenInfo; BenchmarkRunResult({ required this.throughput, diff --git a/flutter/lib/resources/export_result_helper.dart b/flutter/lib/resources/export_result_helper.dart index ad67624c2..be16c110a 100644 --- a/flutter/lib/resources/export_result_helper.dart +++ b/flutter/lib/resources/export_result_helper.dart @@ -1,6 +1,5 @@ import 'package:mlperfbench/backend/bridge/run_result.dart'; import 'package:mlperfbench/backend/list.dart'; -import 'package:mlperfbench/backend/loadgen_info.dart'; import 'package:mlperfbench/benchmark/benchmark.dart'; import 'package:mlperfbench/benchmark/run_info.dart'; import 'package:mlperfbench/benchmark/run_mode.dart'; @@ -71,17 +70,7 @@ class ResultHelper { measuredDuration: result.duration, measuredSamples: result.numSamples, startDatetime: result.startTime, - loadgenInfo: _makeLoadgenInfo(info.loadgenInfo), - ); - } - - BenchmarkLoadgenInfo? _makeLoadgenInfo(LoadgenInfo? source) { - if (source == null) { - return null; - } - return BenchmarkLoadgenInfo( - validity: source.validity, - duration: source.meanLatency * source.queryCount, + loadgenInfo: info.loadgenInfo, ); } diff --git a/flutter/lib/resources/result_manager.dart b/flutter/lib/resources/result_manager.dart index 5b7919faa..dec4ffd2b 100644 --- a/flutter/lib/resources/result_manager.dart +++ b/flutter/lib/resources/result_manager.dart @@ -121,7 +121,7 @@ class ResultManager { acceleratorName: export.backendInfo.acceleratorName, delegateName: export.backendSettings.delegate, batchSize: export.backendSettings.batchSize, - validity: runResult.loadgenInfo?.validity ?? false, + loadgenInfo: runResult.loadgenInfo, ); } diff --git a/flutter/lib/state/task_runner.dart b/flutter/lib/state/task_runner.dart index 653e47940..be9083147 100644 --- a/flutter/lib/state/task_runner.dart +++ b/flutter/lib/state/task_runner.dart @@ -240,7 +240,7 @@ class TaskRunner { acceleratorName: performanceResult.acceleratorName, delegateName: benchmark.benchmarkSettings.delegateSelected, batchSize: benchmark.selectedDelegate.batchSize, - validity: performanceRunInfo.loadgenInfo!.validity, + loadgenInfo: performanceRunInfo.loadgenInfo!, ); resultHelper.performanceRunInfo = performanceRunInfo; } else if (mode == accuracyMode) { @@ -278,7 +278,7 @@ class TaskRunner { acceleratorName: accuracyResult.acceleratorName, delegateName: benchmark.benchmarkSettings.delegateSelected, batchSize: benchmark.selectedDelegate.batchSize, - validity: false, + loadgenInfo: accuracyRunInfo.loadgenInfo, ); } else { throw 'Unknown BenchmarkRunMode: $mode'; @@ -341,21 +341,32 @@ class _NativeRunHelper { } Future _invokeNativeRun() async { - final logPrefix = '${benchmark.id}: $runMode mode'; + final logPrefix = '[${benchmark.id}: $runMode mode]'; - print('$logPrefix: starting...'); + print('$logPrefix starting...'); final stopwatch = Stopwatch()..start(); final nativeResult = await backendBridge.run(runSettings); final elapsed = stopwatch.elapsed; - print('$logPrefix: elapsed: $elapsed'); - print('$logPrefix: result: $nativeResult'); + print('$logPrefix elapsed: $elapsed'); + print('$logPrefix result: $nativeResult'); if (!_checkAccuracy(nativeResult)) { - throw '$logPrefix: accuracy is invalid (backend may be corrupted)'; + throw '$logPrefix accuracy is invalid (backend may be corrupted)'; } final runInfo = await _makeRunInfo(nativeResult); - print('$logPrefix: throughput: ${runInfo.throughput}'); + + final throughput = runInfo.throughput; + if (throughput != null) { + print('$logPrefix throughput: $throughput'); + } + final loadgenInfo = runInfo.loadgenInfo; + if (loadgenInfo != null) { + print('$logPrefix isMinDurationMet: ${loadgenInfo.isMinDurationMet}'); + print('$logPrefix isMinQueryMet: ${loadgenInfo.isMinQueryMet}'); + print('$logPrefix isEarlyStoppingMet: ${loadgenInfo.isEarlyStoppingMet}'); + print('$logPrefix isResultValid: ${loadgenInfo.isResultValid}'); + } return runInfo; } diff --git a/flutter/lib/ui/app_styles.dart b/flutter/lib/ui/app_styles.dart index 6c14b8f13..663cd262d 100644 --- a/flutter/lib/ui/app_styles.dart +++ b/flutter/lib/ui/app_styles.dart @@ -17,6 +17,7 @@ class AppColors { static const darkText = Colors.black; static const resultValidText = Colors.indigo; static const resultInvalidText = Colors.red; + static const resultSemiValidText = Colors.purple; static const errorText = Colors.red; static const primary = _blue2; diff --git a/flutter/lib/ui/history/benchmark_export_result_screen.dart b/flutter/lib/ui/history/benchmark_export_result_screen.dart index 92dc19b26..83ad4350b 100644 --- a/flutter/lib/ui/history/benchmark_export_result_screen.dart +++ b/flutter/lib/ui/history/benchmark_export_result_screen.dart @@ -77,7 +77,7 @@ class _BenchmarkExportResultScreenState helper.makeInfo(l10n.historyRunDetailsPerfQps, perf.throughput?.toUIString() ?? l10n.resultsNotAvailable), helper.makeInfo(l10n.historyRunDetailsValid, - (perf.loadgenInfo?.validity ?? false).toString()), + (perf.loadgenInfo?.isResultValid).toString()), helper.makeInfo(l10n.historyRunDetailsDuration, perf.measuredDuration.toDurationUIString()), helper.makeInfo( diff --git a/flutter/lib/ui/history/extended_result_screen.dart b/flutter/lib/ui/history/extended_result_screen.dart index bfae2e401..2daa756b7 100644 --- a/flutter/lib/ui/history/extended_result_screen.dart +++ b/flutter/lib/ui/history/extended_result_screen.dart @@ -239,7 +239,8 @@ class _ExtendedResultViewState extends State { name: runInfo.benchmarkName, throughput: runInfo.performanceRun?.throughput?.toUIString() ?? l10n.resultsNotAvailable, - throughputValid: runInfo.performanceRun?.loadgenInfo?.validity ?? false, + throughputValid: + runInfo.performanceRun?.loadgenInfo?.isResultValid ?? false, accuracy: runInfo.accuracyRun?.accuracy?.formatted ?? l10n.resultsNotAvailable, onTap: () { diff --git a/flutter/lib/ui/home/benchmark_result_screen.dart b/flutter/lib/ui/home/benchmark_result_screen.dart index 12ddc9d82..8015ccb5d 100644 --- a/flutter/lib/ui/home/benchmark_result_screen.dart +++ b/flutter/lib/ui/home/benchmark_result_screen.dart @@ -223,7 +223,7 @@ class _BenchmarkResultScreenState extends State late final String? resultText2; late final double? progressBarValue2; late final BenchmarkResult? benchmarkResult; - late final bool resultIsValid; + late final Color resultTextColor; switch (_screenMode) { case _ScreenMode.performance: benchmarkResult = benchmark.performanceModeResult; @@ -233,7 +233,22 @@ class _BenchmarkResultScreenState extends State (throughput?.value ?? 0.0) / benchmark.info.maxThroughput; resultText2 = null; progressBarValue2 = null; - resultIsValid = benchmarkResult?.validity ?? false; + final loadgenInfo = benchmarkResult?.loadgenInfo; + if (loadgenInfo != null) { + if (loadgenInfo.isMinDurationMet == true && + loadgenInfo.isMinQueryMet == true && + loadgenInfo.isEarlyStoppingMet == true) { + resultTextColor = AppColors.resultValidText; + } else if (loadgenInfo.isMinDurationMet == true && + loadgenInfo.isMinQueryMet == false && + loadgenInfo.isEarlyStoppingMet == true) { + resultTextColor = AppColors.resultSemiValidText; + } else { + resultTextColor = AppColors.resultInvalidText; + } + } else { + resultTextColor = AppColors.resultInvalidText; + } break; case _ScreenMode.accuracy: benchmarkResult = benchmark.accuracyModeResult; @@ -241,10 +256,13 @@ class _BenchmarkResultScreenState extends State progressBarValue = benchmarkResult?.accuracy?.normalized; resultText2 = benchmarkResult?.accuracy2?.formatted; progressBarValue2 = benchmarkResult?.accuracy2?.normalized; - resultIsValid = - (benchmarkResult?.accuracy?.normalized ?? -1.0) >= 0.0 && - (benchmarkResult?.accuracy?.normalized ?? -1.0) <= 1.0 && - (benchmarkResult?.accuracy2?.normalized ?? -1.0) <= 1.0; + if ((benchmarkResult?.accuracy?.normalized ?? -1.0) >= 0.0 && + (benchmarkResult?.accuracy?.normalized ?? -1.0) <= 1.0 && + (benchmarkResult?.accuracy2?.normalized ?? -1.0) <= 1.0) { + resultTextColor = AppColors.resultValidText; + } else { + resultTextColor = AppColors.resultInvalidText; + } break; } final perfResult = benchmark.performanceModeResult; @@ -258,9 +276,7 @@ class _BenchmarkResultScreenState extends State var subtitleColumnChildren = []; subtitleColumnChildren.add(const SizedBox(height: 4)); final resultTextStyle = TextStyle( - color: resultIsValid - ? AppColors.resultValidText - : AppColors.resultInvalidText, + color: resultTextColor, fontSize: 18.0, fontWeight: FontWeight.bold, ); diff --git a/flutter/unit_test/backend/loadgen_info_test.dart b/flutter/unit_test/backend/loadgen_info_test.dart index 2698b3915..fe8c0f7fc 100644 --- a/flutter/unit_test/backend/loadgen_info_test.dart +++ b/flutter/unit_test/backend/loadgen_info_test.dart @@ -15,6 +15,12 @@ class _Examples { ':::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 41013900, "time_ms": 273.005400, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "external/org_mlperf_inference/loadgen/loadgen.cc", "line_no": 1124, "pid": 38760, "tid": 37352}}'; static const validity = ':::MLLOG {"key": "result_validity", "value": "INVALID", "time_ms": 273.005400, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "external/org_mlperf_inference/loadgen/loadgen.cc", "line_no": 1029, "pid": 38760, "tid": 37352}}'; + static const minDurationMet = + ':::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 41253.016458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "external/org_mlperf_inference/loadgen/results.cc", "line_no": 451, "pid": 33655, "tid": 13951061273224140806}}'; + static const minQueriesMet = + ':::MLLOG {"key": "result_min_queries_met", "value": false, "time_ms": 41253.016458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "external/org_mlperf_inference/loadgen/results.cc", "line_no": 452, "pid": 33655, "tid": 13951061273224140806}}'; + static const earlyStoppingMet = + ':::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 41253.016458, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "external/org_mlperf_inference/loadgen/results.cc", "line_no": 453, "pid": 33655, "tid": 13951061273224140806}}'; } void main() { @@ -51,6 +57,9 @@ void main() { _Examples.latency90, _Examples.validity, _Examples.queries, + _Examples.minDurationMet, + _Examples.minQueriesMet, + _Examples.earlyStoppingMet, ]; final info = await LoadgenInfo.extractLoadgenInfo( @@ -60,9 +69,12 @@ void main() { expect(info, isNotNull); info!; expect(info.latency90, 0.0410139); - expect(info.meanLatency, 0.031329862); + expect(info.latencyMean, 0.031329862); expect(info.queryCount, 8); - expect(info.validity, false); + expect(info.isResultValid, false); + expect(info.isMinDurationMet, true); + expect(info.isMinQueryMet, false); + expect(info.isEarlyStoppingMet, true); }); test('extract info: missing keys: all', () async { const lines = [ diff --git a/mobile_back_apple/dev-utils/Makefile b/mobile_back_apple/dev-utils/Makefile index 1fcb9f60e..50868717b 100644 --- a/mobile_back_apple/dev-utils/Makefile +++ b/mobile_back_apple/dev-utils/Makefile @@ -8,7 +8,7 @@ REPO_ROOT_DIR="/Users/anh/dev/mlcommons/mobile_app_open" #REPO_ROOT_DIR="/Users/freedom/work/mobile_app_open" -tflite: tflite-build tflite-run-ic +tflite: tflite-build tflite-run-sr coreml: coreml-build coreml-run-ic tflite-all: tflite-build tflite-run-ic tflite-run-ic-offline tflite-run-od tflite-run-is tflite-run-lu tflite-run-sr coreml-all: coreml-build coreml-run-ic coreml-run-ic-offline coreml-run-od coreml-run-is coreml-run-lu coreml-run-sr @@ -22,7 +22,7 @@ app: tflite-build: cd ${REPO_ROOT_DIR} && \ - bazel build -c opt --cxxopt=-std=c++14 --host_cxxopt=-std=c++14 //flutter/cpp/binary:main //mobile_back_tflite:tflitebackend + bazel build -c opt --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //flutter/cpp/binary:main //mobile_back_tflite:tflitebackend tflite-run-ic: cd ${REPO_ROOT_DIR} && \ @@ -93,10 +93,12 @@ tflite-run-sr: --lib_path="bazel-bin/mobile_back_tflite/cpp/backend_tflite/libtflitebackend.so" \ --images_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/psnr/LR" \ --ground_truth_directory="${REPO_ROOT_DIR}/mobile_back_apple/dev-resources/psnr/HR" \ + --max_duration_ms=60000 \ + --min_duration_ms=100 coreml-build: cd ${REPO_ROOT_DIR} && \ - bazel build -c opt --cxxopt=-fobjc-arc --cxxopt=-xobjective-c++ --cxxopt=-std=c++14 --host_cxxopt=-std=c++14 \ + bazel build -c opt --cxxopt=-fobjc-arc --cxxopt=-xobjective-c++ --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 \ //flutter/cpp/binary:main //mobile_back_apple:coremlbackend coreml-run-ic: diff --git a/tools/extended_result_schema.json b/tools/extended_result_schema.json index 87b782e0f..b25e9c052 100644 --- a/tools/extended_result_schema.json +++ b/tools/extended_result_schema.json @@ -377,10 +377,10 @@ "start_datetime": { "type": "string" }, - "loadgen_info": { + "loadgen": { "oneOf": [ { - "$ref": "#/definitions/LoadgenInfo" + "$ref": "#/definitions/Loadgen" }, { "type": "null" @@ -392,7 +392,7 @@ "accuracy", "accuracy2", "dataset", - "loadgen_info", + "loadgen", "measured_duration", "measured_samples", "start_datetime", @@ -442,22 +442,42 @@ ], "title": "Dataset" }, - "LoadgenInfo": { + "Loadgen": { "type": "object", "additionalProperties": false, "properties": { - "validity": { - "type": "boolean" + "query_count": { + "type": "integer" + }, + "latency_mean": { + "type": "number" }, - "duration": { + "latency90": { "type": "number" + }, + "is_min_duration_met": { + "type": "boolean" + }, + "is_min_query_met": { + "type": "boolean" + }, + "is_early_stopping_met": { + "type": "boolean" + }, + "is_result_valid": { + "type": "boolean" } }, "required": [ - "duration", - "validity" + "is_early_stopping_met", + "is_min_duration_met", + "is_min_query_met", + "is_result_valid", + "latency90", + "latency_mean", + "query_count" ], - "title": "LoadgenInfo" + "title": "Loadgen" }, "Throughput": { "type": "object", From 9de02d401f6dbf65c3900b2eb3a02c618786b4ce Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 5 Mar 2024 13:31:27 +0700 Subject: [PATCH 19/20] Change task order --- flutter/assets/tasks.pbtxt | 56 +++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt index 7795e39fd..3f69cba75 100644 --- a/flutter/assets/tasks.pbtxt +++ b/flutter/assets/tasks.pbtxt @@ -2,8 +2,8 @@ # proto-message: MLPerfConfig task { - id: "image_classification" - name: "Image Classification" + id: "image_classification_v2" + name: "Image Classification v2" min_query_count: 1024 min_duration: 60 max_duration: 600 @@ -29,11 +29,11 @@ task { } } model { - id: "mobilenetedgetpu_224_v1_0" - name: "MobileNetEdgeTPU 224 1.0" - offset: 1 - image_width: 224 - image_height: 224 + id: "MobileNetEdgeTPUv2-L" + name: "MobileNetEdgeTPUv2-L" + offset: 0 + image_width: 384 + image_height: 384 num_classes: 1000 } } @@ -180,8 +180,8 @@ task { } task { - id: "image_classification_offline" - name: "Image Classification (Offline)" + id: "image_classification_offline_v2" + name: "Image Classification v2 (Offline)" min_query_count: 24576 min_duration: 0 max_duration: 0 @@ -207,18 +207,18 @@ task { } } model { - id: "mobilenetedgetpu_224_v1_0" - name: "MobileNetEdgeTPU 224 1.0" - offset: 1 - image_width: 224 - image_height: 224 + id: "MobileNetEdgeTPUv2-L" + name: "MobileNetEdgeTPUv2-L" + offset: 0 + image_width: 384 + image_height: 384 num_classes: 1000 } } task { - id: "image_classification_v2" - name: "Image Classification v2" + id: "image_classification" + name: "Image Classification" min_query_count: 1024 min_duration: 60 max_duration: 600 @@ -244,18 +244,18 @@ task { } } model { - id: "MobileNetEdgeTPUv2-L" - name: "MobileNetEdgeTPUv2-L" - offset: 0 - image_width: 384 - image_height: 384 + id: "mobilenetedgetpu_224_v1_0" + name: "MobileNetEdgeTPU 224 1.0" + offset: 1 + image_width: 224 + image_height: 224 num_classes: 1000 } } task { - id: "image_classification_offline_v2" - name: "Image Classification v2 (Offline)" + id: "image_classification_offline" + name: "Image Classification (Offline)" min_query_count: 24576 min_duration: 0 max_duration: 0 @@ -281,11 +281,11 @@ task { } } model { - id: "MobileNetEdgeTPUv2-L" - name: "MobileNetEdgeTPUv2-L" - offset: 0 - image_width: 384 - image_height: 384 + id: "mobilenetedgetpu_224_v1_0" + name: "MobileNetEdgeTPU 224 1.0" + offset: 1 + image_width: 224 + image_height: 224 num_classes: 1000 } } From 3be9b81a6a9be2743dbb3c1d7ad1de8e5adac56e Mon Sep 17 00:00:00 2001 From: Anh Date: Tue, 5 Mar 2024 14:42:01 +0700 Subject: [PATCH 20/20] Update expected_throughput for _kDN2103 --- flutter/integration_test/expected_throughput.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart index a482dcdc0..e1eda1804 100644 --- a/flutter/integration_test/expected_throughput.dart +++ b/flutter/integration_test/expected_throughput.dart @@ -73,7 +73,7 @@ const Map> _imageClassificationV2 = { _kS22Ultra: Interval(min: 1700, max: 2400), }, _kMediatekBackend: { - _kDN2103: Interval(min: 30, max: 55), + _kDN2103: Interval(min: 5, max: 90), }, };