Merge master into submission-v4.0

.github/workflows/android-build-test.yml

@@ -16,7 +16,7 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Log in to the Container registry
@@ -55,12 +55,18 @@ jobs:
       FLUTTER_BUILD_NUMBER: ${{ github.run_number }}
       FIREBASE_ANDROID_API_KEY: ${{ secrets.FIREBASE_ANDROID_API_KEY }}
       FIREBASE_ANDROID_APP_ID: ${{ secrets.FIREBASE_ANDROID_APP_ID }}
-      FIREBASE_PROJECT_ID: ${{ secrets.FIREBASE_PROJECT_ID }}
+      FIREBASE_PROJECT_ID: mobile-app-build-290400
       FIREBASE_MESSAGING_SENDER_ID: ${{ secrets.FIREBASE_MESSAGING_SENDER_ID }}
       FIREBASE_DATABASE_URL: ${{ secrets.FIREBASE_DATABASE_URL }}
       FIREBASE_STORAGE_BUCKET: ${{ secrets.FIREBASE_STORAGE_BUCKET }}
+      SIGNING_FOR_RELEASE: true
+      SIGNING_STORE_FILE: /tmp/mlperf-keystore.jks
+      SIGNING_STORE_FILE_BASE64: ${{ secrets.SIGNING_STORE_FILE_BASE64 }}
+      SIGNING_STORE_PASSWORD: ${{ secrets.SIGNING_STORE_PASSWORD }}
+      SIGNING_KEY_ALIAS: upload
+      SIGNING_KEY_PASSWORD: ${{ secrets.SIGNING_KEY_PASSWORD }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       # Preinstalled tools from host at `/opt/hostedtoolcache` is not needed since we run commands inside our own Docker container.
       # `/opt/hostedtoolcache` is mounted to `/__t`. We delete it to free up disk space.
       - name: Free up disk space
@@ -72,6 +78,10 @@ jobs:
           df -h
       - name: Configure Git
         run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: Create keystore file
+        run: |
+          echo "$SIGNING_STORE_FILE_BASE64" | base64 -d > "$SIGNING_STORE_FILE"
+          echo e4699f58310b85cbcb9e835e569e7e17  "$SIGNING_STORE_FILE" | md5sum --check -
       - name: Set up authentication for Google Cloud SDK
         uses: google-github-actions/auth@v2
         with:
@@ -81,30 +91,24 @@ jobs:
         with:
           version: '>= 363.0.0'
           project_id: mobile-app-build-290400
-      - name: Install AWS CLI
-        run: |
-          python3 -m pip install awscli
       - name: Download Samsung libraries
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }}
+          SAMSUNG_LIB: samsung_libs_v3.1_20231116
         run: |
-          aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \
-          cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/samsung_libs_v3.1_20231116.zip /tmp/ && \
-          unzip /tmp/samsung_libs_v3.1_20231116.zip -d /tmp/samsung_libs_v3.1_20231116 && \
-          rm /tmp/samsung_libs_v3.1_20231116.zip && \
+          gsutil cp gs://mobile-app-build-290400_github-actions/lib/v3.1/${SAMSUNG_LIB}.zip /tmp/ && \
+          unzip /tmp/${SAMSUNG_LIB}.zip -d /tmp/${SAMSUNG_LIB} && \
+          rm /tmp/${SAMSUNG_LIB}.zip && \
           mkdir -p mobile_back_samsung/samsung/lib/internal && \
-          mv /tmp/samsung_libs_v3.1_20231116/* mobile_back_samsung/samsung/lib/internal/
+          mv /tmp/${SAMSUNG_LIB}/* mobile_back_samsung/samsung/lib/internal/
       - name: Download QTI libraries
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_R2_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_R2_SECRET_KEY }}
+          QTI_LIB: qaisw-2.13.4.230831213719_60417
         run: |
-          aws s3 --endpoint-url=https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com \
-          cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/qaisw-2.13.4.230831213719_60417-linux.zip /tmp/ && \
-          unzip -P ${{ secrets.READONLY_AWS_KEY }} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \
-          rm /tmp/qaisw-2.13.4.230831213719_60417-linux.zip && \
-          mv /tmp/qaisw-2.13.4.230831213719_60417-linux/qaisw-2.13.4.230831213719_60417 mobile_back_qti/
+          gsutil cp gs://mobile-app-build-290400_github-actions/lib/v3.1/${QTI_LIB}.zip /tmp/ && \
+          unzip /tmp/${QTI_LIB}.zip -d /tmp/${QTI_LIB} && \
+          rm /tmp/${QTI_LIB}.zip && \
+          mkdir -p mobile_back_qti/${QTI_LIB} && \
+          mv /tmp/${QTI_LIB}/* mobile_back_qti/${QTI_LIB}/
       - name: Cache bazel
         uses: actions/cache@v3
         with:
@@ -175,12 +179,14 @@ jobs:
           WITH_APPLE: 0
         run: |
           make flutter/android/release
+          gsutil cp flutter/build/app/outputs/bundle/release/app-release.aab $GCLOUD_BUCKET_PATH/app-release.aab
+          gsutil cp flutter/build/app/outputs/flutter-apk/app-release.apk $GCLOUD_BUCKET_PATH/app-release.apk
       - name: Archive Android APKs
         uses: actions/upload-artifact@v3
         with:
           name: android-apks-${{ github.run_number }}
           path: output/android-apks/*.apk
-          retention-days: 30
+          retention-days: 28
           if-no-files-found: error
 
   test-android-apk-tflite:
@@ -282,3 +288,33 @@ jobs:
             --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
             --device model=OP515BL1,version=33,locale=en,orientation=portrait \
             --client-details=buildNumber=${{ github.run_number }}
+
+  deploy-playstore:
+    needs:
+      - build-android-apk
+      - test-android-apk-tflite
+    runs-on: ubuntu-22.04
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up authentication for Google Cloud SDK
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }}
+      - name: Set up Google Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          version: '>= 363.0.0'
+          project_id: mobile-app-build-290400
+      - name: Download Android release app
+        run: |
+          gsutil cp $GCLOUD_BUCKET_PATH/app-release.aab /tmp/app-release.aab
+      - name: Upload Android release app to Google Play
+        uses: r0adkll/upload-google-play@v1
+        with:
+          releaseFiles: /tmp/app-release.aab
+          packageName: org.mlcommons.android.mlperfbench
+          serviceAccountJsonPlainText: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }}
+          # track: "alpha", "beta", "internal" or "production"
+          track: internal
+          status: draft

.github/workflows/format-and-lint.yml

@@ -118,11 +118,6 @@ jobs:
         run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - name: markdown-format
         run: make lint/markdown
-      - name: markdown-link
-        uses: gaurav-nelson/github-action-markdown-link-check@v1
-        with:
-          use-verbose-mode: yes
-          use-quiet-mode: yes
 
   yaml:
     needs: build-image

docs/build-and-run.md

@@ -114,6 +114,17 @@ See [Windows environment setup guide](./environment-setup/env-setup-windows.md#s
 Run `make OFFICIAL_BUILD=false FLUTTER_BUILD_NUMBER=0 flutter/android/release` to build APK from scratch,
 or run `make flutter/android/apk` after `make flutter/prepare` and you already have native libs.
 
+By default, the Android app will be signed with a debug key. To sign it with a release key for publishing to Play Store,
+follow the instructions at <https://docs.flutter.dev/deployment/android#sign-the-app> and set the environment variables:
+
+```shell
+export SIGNING_FOR_RELEASE=true
+export SIGNING_STORE_FILE=/path/to/keystore.jks
+export SIGNING_STORE_PASSWORD=abc
+export SIGNING_KEY_ALIAS=abc
+export SIGNING_KEY_PASSWORD=abc
+```
+
 You can build the app using docker.  
 Run `make OFFICIAL_BUILD=false FLUTTER_BUILD_NUMBER=0 docker/flutter/android/release` to build release APK.  
 Run `make docker/flutter/android/libs` to build just `.so` libs. This command is helpful if you want to build Android version of the app on Windows.

flutter/android/android.mk

@@ -19,7 +19,7 @@ ANDROID_NDK_VERSION?=25
 ANDROID_NDK_API_LEVEL?=33
 
 flutter/android: flutter/android/libs flutter/check/firebase-env
-flutter/android/release: flutter/check-release-env flutter/android flutter/prepare flutter/android/apk
+flutter/android/release: flutter/check-release-env flutter/android flutter/prepare flutter/android/apk flutter/android/appbundle
 flutter/android/libs: flutter/android/libs/checksum flutter/android/libs/build flutter/android/libs/copy
 # run `make flutter/android/apk` before `flutter/android/test-apk`
 flutter/android/test-apk: flutter/android/test-apk/main flutter/android/test-apk/helper
@@ -76,7 +76,16 @@ flutter/android/apk:
 		${flutter_firebase_crashlytics_arg} \
 		${flutter_build_number_arg} \
 		${flutter_folder_args}
-	cp -f flutter/build/app/outputs/flutter-apk/app-release.apk ${flutter_android_apk_release_path}
+	cp -f flutter/build/app/outputs/flutter-apk/app-release.apk ${flutter_android_apk_release_path}.apk
+.PHONY: flutter/android/appbundle
+flutter/android/appbundle:
+	mkdir -p $$(dirname ${flutter_android_apk_release_path})
+	cd flutter && ${_start_args} flutter --no-version-check build appbundle \
+		${flutter_official_build_arg} \
+		${flutter_firebase_crashlytics_arg} \
+		${flutter_build_number_arg} \
+		${flutter_folder_args}
+	cp -f flutter/build/app/outputs/bundle/release/app-release.aab ${flutter_android_apk_release_path}.aab
 
 FLUTTER_ANDROID_APK_TEST_MAIN?=test-main.apk
 flutter_android_apk_test_main_path=${FLUTTER_ANDROID_APK_FOLDER}/${FLUTTER_ANDROID_APK_TEST_MAIN}

flutter/android/app/build.gradle

@@ -54,17 +54,32 @@ android {
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
 
+   signingConfigs {
+       release {
+           keyAlias System.env['SIGNING_KEY_ALIAS']
+           keyPassword System.env['SIGNING_KEY_PASSWORD']
+           storeFile System.env['SIGNING_STORE_FILE'] ? file(System.env['SIGNING_STORE_FILE']) : null
+           storePassword System.env['SIGNING_STORE_PASSWORD']
+       }
+   }
+
     buildTypes {
         release {
-            // TODO: Add your own signing config for the release build.
-            // Signing with the debug keys for now, so `flutter run --release` works.
-            signingConfig signingConfigs.debug
-
+            // https://docs.flutter.dev/deployment/android#sign-the-app
+            if (System.env['SIGNING_FOR_RELEASE'] == 'true') {
+                println 'Using signingConfigs.release'
+                signingConfig signingConfigs.release
+            } else {
+                println 'Using signingConfigs.debug'
+                signingConfig signingConfigs.debug
+            }
             // By default we do not upload mapping file to Firebase
             firebaseCrashlytics {
                 if (System.env['FIREBASE_CRASHLYTICS_ENABLED'] == 'true') {
+                    println 'mappingFileUploadEnabled set to true'
                     mappingFileUploadEnabled true
                 } else {
+                    println 'mappingFileUploadEnabled set to false'
                     mappingFileUploadEnabled false
                 }
             }

flutter/lib/backend/loadgen_info.dart

@@ -5,6 +5,8 @@ import 'package:json_annotation/json_annotation.dart';
 
 part 'loadgen_info.g.dart';
 
+enum ResultValidityEnum { valid, invalid, semivalid }
+
 @JsonSerializable(fieldRename: FieldRename.snake)
 class LoadgenInfo {
   final int queryCount;
@@ -107,4 +109,18 @@ class LoadgenInfo {
       _$LoadgenInfoFromJson(json);
 
   Map<String, dynamic> toJson() => _$LoadgenInfoToJson(this);
+
+  ResultValidityEnum get resultValidity {
+    if (isMinDurationMet == true &&
+        isMinQueryMet == true &&
+        isEarlyStoppingMet == true) {
+      return ResultValidityEnum.valid;
+    } else if (isMinDurationMet == true &&
+        isMinQueryMet == false &&
+        isEarlyStoppingMet == true) {
+      return ResultValidityEnum.semivalid;
+    } else {
+      return ResultValidityEnum.invalid;
+    }
+  }
 }

flutter/lib/data/result_filter.dart

@@ -2,7 +2,6 @@ import 'package:json_annotation/json_annotation.dart';
 
 import 'package:mlperfbench/data/environment/environment_info.dart';
 import 'package:mlperfbench/data/extended_result.dart';
-import 'package:mlperfbench/data/results/benchmark_result.dart';
 import 'package:mlperfbench/data/string_constants.dart';
 
 part 'result_filter.g.dart';
@@ -68,20 +67,17 @@ class ResultFilter {
     bool manufacturerMatched =
         resultManufacturer.containsIgnoreCase(manufacturer ?? '');
     bool socMatched = resultSoc.containsIgnoreCase(soc ?? '');
-
+    bool benchmarkIdMatched = benchmarkId == null
+        ? true
+        : result.results.any((e) => e.benchmarkId == benchmarkId);
     return fromCreationDateMatched &&
         toCreationDateMatched &&
         platformMatched &&
         deviceModelMatched &&
         backendMatched &&
         manufacturerMatched &&
-        socMatched;
-  }
-
-  bool matchBenchmark(BenchmarkExportResult result) {
-    bool benchmarkIdMatched =
-        benchmarkId == null ? true : result.benchmarkId == benchmarkId;
-    return benchmarkIdMatched;
+        socMatched &&
+        benchmarkIdMatched;
   }
 
   bool get anyFilterActive {

flutter/lib/data/result_sort.dart

@@ -1,51 +1,24 @@
-import 'package:mlperfbench/data/results/benchmark_result.dart';
+import 'package:mlperfbench/data/extended_result.dart';
 
-enum SortByEnum { dateAsc, dateDesc, taskThroughputAsc, taskThroughputDesc }
+enum SortByEnum { dateAsc, dateDesc }
 
 class ResultSort {
   SortByEnum sortBy = SortByEnum.dateDesc;
 
   ResultSort();
 
-  List<BenchmarkExportResult> apply(List<BenchmarkExportResult> items) {
-    List<BenchmarkExportResult> sortedItems =
-        List<BenchmarkExportResult>.from(items);
+  List<ExtendedResult> apply(List<ExtendedResult> results) {
+    List<ExtendedResult> items = List<ExtendedResult>.from(results);
     switch (sortBy) {
       case SortByEnum.dateAsc:
-        sortedItems.sort((a, b) => _sortByDates(a, b, true));
+        items
+            .sort((a, b) => a.meta.creationDate.compareTo(b.meta.creationDate));
         break;
       case SortByEnum.dateDesc:
-        sortedItems.sort((a, b) => _sortByDates(a, b, false));
+        items
+            .sort((a, b) => b.meta.creationDate.compareTo(a.meta.creationDate));
         break;
-      case SortByEnum.taskThroughputAsc:
-        sortedItems.sort((a, b) => _sortByTaskThroughput(b, a));
-        break;
-      case SortByEnum.taskThroughputDesc:
-        sortedItems.sort((a, b) => _sortByTaskThroughput(a, b));
-        break;
-    }
-    return sortedItems;
-  }
-
-  int _sortByDates(
-      BenchmarkExportResult a, BenchmarkExportResult b, bool isAsc) {
-    DateTime aDateTime = a.performanceRun?.startDatetime ??
-        a.accuracyRun?.startDatetime ??
-        DateTime.now();
-    DateTime bDateTime = b.performanceRun?.startDatetime ??
-        b.accuracyRun?.startDatetime ??
-        DateTime.now();
-
-    if (isAsc) {
-      return aDateTime.compareTo(bDateTime);
     }
-    return bDateTime.compareTo(aDateTime);
-  }
-
-  int _sortByTaskThroughput(BenchmarkExportResult a, BenchmarkExportResult b) {
-    const noThroughput = Throughput(value: 0);
-    final aThroughput = a.performanceRun?.throughput ?? noThroughput;
-    final bThroughput = b.performanceRun?.throughput ?? noThroughput;
-    return bThroughput.compareTo(aThroughput);
+    return items;
   }
 }

flutter/lib/l10n/app_en.arb

@@ -154,9 +154,7 @@
   "historyFilterDeviceModel": "Device Model",
   "historyFilterManufacturer": "Manufacturer",
   "historyFilterSoC": "SoC",
-  "historySortBy": "Sort by",
   "historySortByDate": "Date",
-  "historySortByTaskThroughput": "Task Throughput",
 
   "historyDetailsTitle": "Result details",
   "historyDetailsBuildTypeDebug": "debug build",