Merge master into submission-v4.0 (#862)

* Update S3 provider in android-build-test.yml (#855) * Update S3 provider in android-build-test.yml * Update env var name --------- Co-authored-by: Anh <[email protected]> * fix: performance drop caused by UI animation (#857) * Remove InfiniteProgressCircle * Use GitHub vars for --num-flaky-test-attempts flag * Format Dart code * Add DottedProgressCircle * Add setting for progress animation style * Revert "Add setting for progress animation style" This reverts commit c14cc63. * Revert "Add DottedProgressCircle" This reverts commit ef1c427. * Use ProgressCircle with no animation * Format Dart code * ci: use new device for test-android-apk-tflite (#860) * Use GitHub vars for --num-flaky-test-attempts flag * Use Pixel 6 for test-android-apk-tflite * Add expected_throughput for _kTFLiteBackend -> _kPixel6 * Update expected_throughput for _kTFLiteBackend -> _kPixel6 * Replace obsolete Windows image * Update expected_throughput for _kDN2103 * Use Pixel 6 - API level 31 for higher device capacity * ci: replace obsolete Windows image (#861) Replace obsolete Windows image * feat: add flag enforce_max_duration (#798) * Use dev version of loadgen * Set enforce_max_duration to false * Add min_duration_ms and max_duration_ms to dev Makefile * update enforce_max_duration logic * use updated loadgen * Show different colors for performance result based on multiple conditions. * Fix Dart linter issue * Use GitHub vars for --num-flaky-test-attempts flag * Update expected_throughput for _kS22Ultra * Use latest commit from inference/mobile_update branch --------- Co-authored-by: Koan-Sin Tan <[email protected]> --------- Co-authored-by: Nathan Wasson <[email protected]> Co-authored-by: Koan-Sin Tan <[email protected]>
mlcommons · Mar 4, 2024 · 28cddb2 · 28cddb2
1 parent ce20401
commit 28cddb2
Show file tree

Hide file tree

Showing 25 changed files with 279 additions and 292 deletions.
diff --git a/.github/cloudbuild/flutter-windows.yaml b/.github/cloudbuild/flutter-windows.yaml
@@ -34,7 +34,7 @@ steps:
       - --region=$_CE_REGION
       - --zone=$_CE_ZONE
       - --machineType=$_CE_MACHINE_TYPE
-      - --image=projects/gce-uefi-images/global/images/windows-server-2019-dc-for-containers-v20200609
+      - --image=projects/gce-uefi-images/global/images/windows-server-2019-dc-for-containers-v20200512
       # When using other disk types image pulling and image building could take up to 3 hours
       - --diskType=pd-ssd
       - --workspace-bucket=$_WORKSPACE_TMP_BUCKET

diff --git a/.github/workflows/android-build-test.yml b/.github/workflows/android-build-test.yml
@@ -86,23 +86,23 @@ jobs:
           python3 -m pip install awscli
       - name: Download Samsung libraries
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }}
         run: |
-          aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \
+          aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \
           cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/samsung_libs_v3.1_20231116.zip /tmp/ && \
           unzip /tmp/samsung_libs_v3.1_20231116.zip -d /tmp/samsung_libs_v3.1_20231116 && \
           rm /tmp/samsung_libs_v3.1_20231116.zip && \
           mkdir -p mobile_back_samsung/samsung/lib/internal && \
           mv /tmp/samsung_libs_v3.1_20231116/* mobile_back_samsung/samsung/lib/internal/
       - name: Download QTI libraries
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }}
         run: |
-          aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \
+          aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \
           cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/qaisw-2.13.4.230831213719_60417-linux.zip /tmp/ && \
-          unzip -P ${AWS_SECRET_ACCESS_KEY} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \
+          unzip -P ${{ secrets.READONLY_AWS_KEY }} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \
           rm /tmp/qaisw-2.13.4.230831213719_60417-linux.zip && \
           mv /tmp/qaisw-2.13.4.230831213719_60417-linux/qaisw-2.13.4.230831213719_60417 mobile_back_qti/
       - name: Cache bazel
@@ -197,15 +197,15 @@ jobs:
         with:
           version: '>= 363.0.0'
           project_id: mobile-app-build-290400
-      - name: Trigger Firebase Test Lab for [Google Pixel 5](redfin)
+      - name: Trigger Firebase Test Lab for [Google Pixel 6](oriole)
         run: |
           gcloud firebase test android run \
             --type instrumentation \
             --app $GCLOUD_BUCKET_PATH/test-main-tflite.apk \
             --test $GCLOUD_BUCKET_PATH/test-helper-tflite.apk \
             --timeout 30m \
-            --num-flaky-test-attempts 2 \
-            --device model=redfin,version=30,locale=en,orientation=portrait \
+            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
+            --device model=oriole,version=31,locale=en,orientation=portrait \
             --client-details=buildNumber=${{ github.run_number }}
 
   test-android-apk-pixel:
@@ -229,7 +229,7 @@ jobs:
             --app $GCLOUD_BUCKET_PATH/test-main-pixel.apk \
             --test $GCLOUD_BUCKET_PATH/test-helper-pixel.apk \
             --timeout 30m \
-            --num-flaky-test-attempts 2 \
+            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
             --device model=oriole,version=32,locale=en,orientation=portrait \
             --client-details=buildNumber=${{ github.run_number }}
 
@@ -254,7 +254,7 @@ jobs:
             --app $GCLOUD_BUCKET_PATH/test-main-qti.apk \
             --test $GCLOUD_BUCKET_PATH/test-helper-qti.apk \
             --timeout 30m \
-            --num-flaky-test-attempts 2 \
+            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
             --device model=b0q,version=33,locale=en,orientation=portrait \
             --client-details=buildNumber=${{ github.run_number }}
 
@@ -279,6 +279,6 @@ jobs:
             --app $GCLOUD_BUCKET_PATH/test-main-mtk.apk \
             --test $GCLOUD_BUCKET_PATH/test-helper-mtk.apk \
             --timeout 30m \
-            --num-flaky-test-attempts 2 \
+            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
             --device model=OP515BL1,version=33,locale=en,orientation=portrait \
             --client-details=buildNumber=${{ github.run_number }}
diff --git a/WORKSPACE b/WORKSPACE
@@ -1,6 +1,7 @@
 workspace(name = "mlperf_app")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
 
 http_archive(
     name = "bazel_skylib",
@@ -88,17 +89,14 @@ http_archive(
     urls = ["https://github.com/MediaTek-NeuroPilot/tflite-neuron-delegate/archive/refs/heads/update_for_dujac.zip"],
 )
 
-http_archive(
+new_git_repository(
     name = "org_mlperf_inference",
     build_file = "@//flutter/android/third_party:loadgen.BUILD",
+    commit = "238d035ab41d7ddd390b35471af169ea641380f6",
     patch_args = ["-p1"],
     patch_cmds = ["python3 loadgen/version_generator.py loadgen/version_generated.cc loadgen"],
     patches = [],
-    sha256 = "e664f980e84fcab3573447c0cc3adddd1fcf900367c5dcbff17179ece24c484e",
-    strip_prefix = "inference-2da0c52666e21e4b296b09e1dbd287bf3a814e96",
-    urls = [
-        "https://github.com/mlcommons/inference/archive/2da0c52666e21e4b296b09e1dbd287bf3a814e96.tar.gz",
-    ],
+    remote = "https://github.com/mlcommons/inference.git",
 )
 
 # This is required to pass SNPE SDK path from external environment to sources,

diff --git a/docs/result-spec.md b/docs/result-spec.md
@@ -76,12 +76,22 @@ If you enable Submission mode, both `performance_run` and `accuracy_run` values
     Actual duration of the benchmark in seconds from start to finish.
   * `measured_samples`: integer number  
     Actual number of samples evaluated during the benchmark
-  * `loadgen_info`: map  
+  * `loadgen`: map  
     Info provided by loadgen. May be null for accuracy runs.
-    * `validity`: bool  
-      Indicates whether all constraints were satisfied or not.
-    * `duration`: floating point number  
-      Duration of the benchmark without loadgen overhead in seconds.
+    * `queryCount`: bool  
+      Number of queries performed.
+    * `latencyMean`: bool  
+      Mean latency in seconds.
+    * `latency90`: bool  
+      90th percentile in seconds.
+    * `isMinDurationMet`: bool
+      Indicates whether the min duration condition is met or not.
+    * `isMinQueryMet`: bool
+      Indicates whether the min query condition is met or not.
+    * `isEarlyStoppingMet`: bool
+      Indicates whether the early stopping condition is met or not.
+    * `isResultValid`: bool  
+      Indicates whether the result is valid or not.
   * `start_datetime`: string  
     Datetime of the moment when benchmark started  
     Format is Iso 8601 in UTC timezone: `2022-04-14T03:54:54.687Z`

diff --git a/flutter/cpp/mlperf_driver.cc b/flutter/cpp/mlperf_driver.cc
@@ -105,6 +105,7 @@ void MlperfDriver::RunMLPerfTest(const std::string& mode, int min_query_count,
   // See https://github.com/mlcommons/inference/issues/1397
   mlperf_settings.max_duration_ms =
       static_cast<uint64_t>(std::ceil(max_duration * 1000.0));
+  mlperf_settings.enforce_max_duration = true;
 
   if (scenario_ == "Offline") {
     mlperf_settings.scenario = ::mlperf::TestScenario::Offline;

diff --git a/flutter/integration_test/expected_throughput.dart b/flutter/integration_test/expected_throughput.dart
@@ -35,6 +35,7 @@ const Map<String, Map<String, Interval>> _imageClassification = {
     _kCloudBuildX28: Interval(min: 4, max: 13),
     _kRyzen5600: Interval(min: 31, max: 37),
     _kPixel5: Interval(min: 80, max: 130),
+    _kPixel6: Interval(min: 600, max: 1100),
     _kIphoneOnGitHubAction: Interval(min: 1, max: 8),
     _kIphoneOnMacbookM1: Interval(min: 19, max: 27),
   },
@@ -45,10 +46,10 @@ const Map<String, Map<String, Interval>> _imageClassification = {
     _kPixel6: Interval(min: 800, max: 1100),
   },
   _kQtiBackend: {
-    _kS22Ultra: Interval(min: 1900, max: 2400),
+    _kS22Ultra: Interval(min: 1700, max: 2400),
   },
   _kMediatekBackend: {
-    _kDN2103: Interval(min: 30, max: 50),
+    _kDN2103: Interval(min: 30, max: 55),
   },
 };
 
@@ -58,6 +59,7 @@ const Map<String, Map<String, Interval>> _objectDetection = {
     _kCloudBuildX28: Interval(min: 3.5, max: 8),
     _kRyzen5600: Interval(min: 14, max: 22),
     _kPixel5: Interval(min: 40, max: 60),
+    _kPixel6: Interval(min: 200, max: 450),
     _kIphoneOnGitHubAction: Interval(min: 0.5, max: 4),
     _kIphoneOnMacbookM1: Interval(min: 9, max: 16),
   },
@@ -68,10 +70,10 @@ const Map<String, Map<String, Interval>> _objectDetection = {
     _kPixel6: Interval(min: 250, max: 450),
   },
   _kQtiBackend: {
-    _kS22Ultra: Interval(min: 800, max: 1400),
+    _kS22Ultra: Interval(min: 700, max: 1400),
   },
   _kMediatekBackend: {
-    _kDN2103: Interval(min: 120, max: 180),
+    _kDN2103: Interval(min: 120, max: 210),
   },
 };
 
@@ -81,6 +83,7 @@ const Map<String, Map<String, Interval>> _imageSegmentationV2 = {
     _kCloudBuildX28: Interval(min: 0.5, max: 4),
     _kRyzen5600: Interval(min: 5, max: 7),
     _kPixel5: Interval(min: 25, max: 40),
+    _kPixel6: Interval(min: 80, max: 180),
     _kIphoneOnGitHubAction: Interval(min: 0.1, max: 2.5),
     _kIphoneOnMacbookM1: Interval(min: 3, max: 6),
   },
@@ -91,10 +94,10 @@ const Map<String, Map<String, Interval>> _imageSegmentationV2 = {
     _kPixel6: Interval(min: 100, max: 180),
   },
   _kQtiBackend: {
-    _kS22Ultra: Interval(min: 450, max: 700),
+    _kS22Ultra: Interval(min: 400, max: 700),
   },
   _kMediatekBackend: {
-    _kDN2103: Interval(min: 45, max: 65),
+    _kDN2103: Interval(min: 45, max: 70),
   },
 };
 
@@ -104,6 +107,7 @@ const Map<String, Map<String, Interval>> _naturalLanguageProcessing = {
     _kCloudBuildX28: Interval(min: 0.5, max: 1.3),
     _kRyzen5600: Interval(min: 2.8, max: 3.2),
     _kPixel5: Interval(min: 2.3, max: 3.0),
+    _kPixel6: Interval(min: 2, max: 75),
     _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1),
     _kIphoneOnMacbookM1: Interval(min: 1.8, max: 3),
   },
@@ -115,10 +119,10 @@ const Map<String, Map<String, Interval>> _naturalLanguageProcessing = {
     _kPixel6: Interval(min: 2, max: 75),
   },
   _kQtiBackend: {
-    _kS22Ultra: Interval(min: 120, max: 180),
+    _kS22Ultra: Interval(min: 100, max: 180),
   },
   _kMediatekBackend: {
-    _kDN2103: Interval(min: 1, max: 5),
+    _kDN2103: Interval(min: 1, max: 6),
   },
 };
 
@@ -128,6 +132,7 @@ const Map<String, Map<String, Interval>> _superResolution = {
     _kCloudBuildX28: Interval(min: 0.1, max: 4),
     _kRyzen5600: Interval(min: 0.1, max: 3),
     _kPixel5: Interval(min: 4, max: 8),
+    _kPixel6: Interval(min: 7, max: 14),
     _kIphoneOnGitHubAction: Interval(min: 0.02, max: 1.0),
     _kIphoneOnMacbookM1: Interval(min: 0.1, max: 10),
   },
@@ -138,10 +143,10 @@ const Map<String, Map<String, Interval>> _superResolution = {
     _kPixel6: Interval(min: 10, max: 14),
   },
   _kQtiBackend: {
-    _kS22Ultra: Interval(min: 35, max: 55),
+    _kS22Ultra: Interval(min: 25, max: 55),
   },
   _kMediatekBackend: {
-    _kDN2103: Interval(min: 5, max: 12),
+    _kDN2103: Interval(min: 5, max: 15),
   },
 };
 
@@ -151,6 +156,7 @@ const Map<String, Map<String, Interval>> _imageClassificationOffline = {
     _kCloudBuildX28: Interval(min: 7, max: 16),
     _kRyzen5600: Interval(min: 45, max: 60),
     _kPixel5: Interval(min: 120, max: 190),
+    _kPixel6: Interval(min: 800, max: 1700),
     _kIphoneOnGitHubAction: Interval(min: 2, max: 15),
     _kIphoneOnMacbookM1: Interval(min: 30, max: 45),
   },
@@ -161,10 +167,10 @@ const Map<String, Map<String, Interval>> _imageClassificationOffline = {
     _kPixel6: Interval(min: 1000, max: 1700),
   },
   _kQtiBackend: {
-    _kS22Ultra: Interval(min: 2600, max: 3500),
+    _kS22Ultra: Interval(min: 2500, max: 3500),
   },
   _kMediatekBackend: {
-    _kDN2103: Interval(min: 75, max: 110),
+    _kDN2103: Interval(min: 75, max: 140),
   },
 };
 

diff --git a/flutter/lib/backend/loadgen_info.dart b/flutter/lib/backend/loadgen_info.dart
@@ -1,19 +1,28 @@
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:json_annotation/json_annotation.dart';
+
+part 'loadgen_info.g.dart';
+
+@JsonSerializable(fieldRename: FieldRename.snake)
 class LoadgenInfo {
-  // Mean latency in seconds
-  final double meanLatency;
   final int queryCount;
-  // 90th percentile in seconds
-  final double latency90;
-  final bool validity;
+  final double latencyMean; // Mean latency in seconds
+  final double latency90; // 90th percentile in seconds
+  final bool isMinDurationMet;
+  final bool isMinQueryMet;
+  final bool isEarlyStoppingMet;
+  final bool isResultValid;
 
   LoadgenInfo({
-    required this.meanLatency,
     required this.queryCount,
+    required this.latencyMean,
     required this.latency90,
-    required this.validity,
+    required this.isMinDurationMet,
+    required this.isMinQueryMet,
+    required this.isEarlyStoppingMet,
+    required this.isResultValid,
   });
 
   static Future<LoadgenInfo?> fromFile({required String filepath}) {
@@ -57,6 +66,10 @@ class LoadgenInfo {
     const latency90Key = 'result_90.00_percentile_latency_ns';
     // https://github.com/mlcommons/inference/blob/318cb131c0adf3bffcbc3379a502f40891331c54/loadgen/loadgen.cc#L1028-L1029
     const validityKey = 'result_validity';
+    // https://github.com/mlcommons/inference/blob/318cb131c0adf3bffcbc3379a502f40891331c54/loadgen/loadgen.cc#L1033C23-L1035
+    const minDurationMetKey = 'result_min_duration_met';
+    const minQueriesMetKey = 'result_min_queries_met';
+    const earlyStoppingMetKey = 'early_stopping_met';
 
     final result = await extractKeys(
       logLines: logLines,
@@ -65,22 +78,33 @@ class LoadgenInfo {
         queryCountKey,
         latency90Key,
         validityKey,
+        minDurationMetKey,
+        minQueriesMetKey,
+        earlyStoppingMetKey,
       },
     );
 
     if (result.isEmpty) {
       return null;
     }
 
-    final validity = result[validityKey] as String == 'VALID';
+    final isResultValid = result[validityKey] as String == 'VALID';
 
     const nanosecondsPerSecond = 1000 * Duration.microsecondsPerSecond;
 
     return LoadgenInfo(
-      meanLatency: (result[latencyKey] as int) / nanosecondsPerSecond,
       queryCount: result[queryCountKey] as int,
+      latencyMean: (result[latencyKey] as int) / nanosecondsPerSecond,
       latency90: (result[latency90Key] as int) / nanosecondsPerSecond,
-      validity: validity,
+      isMinDurationMet: result[minDurationMetKey] as bool,
+      isMinQueryMet: result[minQueriesMetKey] as bool,
+      isEarlyStoppingMet: result[earlyStoppingMetKey] as bool,
+      isResultValid: isResultValid,
     );
   }
+
+  factory LoadgenInfo.fromJson(Map<String, dynamic> json) =>
+      _$LoadgenInfoFromJson(json);
+
+  Map<String, dynamic> toJson() => _$LoadgenInfoToJson(this);
 }
diff --git a/flutter/lib/benchmark/benchmark.dart b/flutter/lib/benchmark/benchmark.dart
@@ -2,6 +2,7 @@ import 'package:collection/collection.dart';
 
 import 'package:mlperfbench/app_constants.dart';
 import 'package:mlperfbench/backend/bridge/run_settings.dart';
+import 'package:mlperfbench/backend/loadgen_info.dart';
 import 'package:mlperfbench/benchmark/info.dart';
 import 'package:mlperfbench/benchmark/run_mode.dart';
 import 'package:mlperfbench/data/results/benchmark_result.dart';
@@ -19,17 +20,18 @@ class BenchmarkResult {
   final String acceleratorName;
   final String delegateName;
   final int batchSize;
-  final bool validity;
-
-  BenchmarkResult(
-      {required this.throughput,
-      required this.accuracy,
-      required this.accuracy2,
-      required this.backendName,
-      required this.acceleratorName,
-      required this.delegateName,
-      required this.batchSize,
-      required this.validity});
+  final LoadgenInfo? loadgenInfo;
+
+  BenchmarkResult({
+    required this.throughput,
+    required this.accuracy,
+    required this.accuracy2,
+    required this.backendName,
+    required this.acceleratorName,
+    required this.delegateName,
+    required this.batchSize,
+    required this.loadgenInfo,
+  });
 }
 
 class Benchmark {