Merge master into submission-v4.0

.github/cloudbuild/flutter-windows.yaml

@@ -34,7 +34,7 @@ steps:
       - --region=$_CE_REGION
       - --zone=$_CE_ZONE
       - --machineType=$_CE_MACHINE_TYPE
-      - --image=projects/gce-uefi-images/global/images/windows-server-2019-dc-for-containers-v20200609
+      - --image=projects/gce-uefi-images/global/images/windows-server-2019-dc-for-containers-v20200512
       # When using other disk types image pulling and image building could take up to 3 hours
       - --diskType=pd-ssd
       - --workspace-bucket=$_WORKSPACE_TMP_BUCKET

.github/workflows/android-build-test.yml

@@ -16,7 +16,7 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Log in to the Container registry
@@ -55,12 +55,18 @@ jobs:
       FLUTTER_BUILD_NUMBER: ${{ github.run_number }}
       FIREBASE_ANDROID_API_KEY: ${{ secrets.FIREBASE_ANDROID_API_KEY }}
       FIREBASE_ANDROID_APP_ID: ${{ secrets.FIREBASE_ANDROID_APP_ID }}
-      FIREBASE_PROJECT_ID: ${{ secrets.FIREBASE_PROJECT_ID }}
+      FIREBASE_PROJECT_ID: mobile-app-build-290400
       FIREBASE_MESSAGING_SENDER_ID: ${{ secrets.FIREBASE_MESSAGING_SENDER_ID }}
       FIREBASE_DATABASE_URL: ${{ secrets.FIREBASE_DATABASE_URL }}
       FIREBASE_STORAGE_BUCKET: ${{ secrets.FIREBASE_STORAGE_BUCKET }}
+      SIGNING_FOR_RELEASE: true
+      SIGNING_STORE_FILE: /tmp/mlperf-keystore.jks
+      SIGNING_STORE_FILE_BASE64: ${{ secrets.SIGNING_STORE_FILE_BASE64 }}
+      SIGNING_STORE_PASSWORD: ${{ secrets.SIGNING_STORE_PASSWORD }}
+      SIGNING_KEY_ALIAS: upload
+      SIGNING_KEY_PASSWORD: ${{ secrets.SIGNING_KEY_PASSWORD }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       # Preinstalled tools from host at `/opt/hostedtoolcache` is not needed since we run commands inside our own Docker container.
       # `/opt/hostedtoolcache` is mounted to `/__t`. We delete it to free up disk space.
       - name: Free up disk space
@@ -72,6 +78,10 @@ jobs:
           df -h
       - name: Configure Git
         run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+      - name: Create keystore file
+        run: |
+          echo "$SIGNING_STORE_FILE_BASE64" | base64 -d > "$SIGNING_STORE_FILE"
+          echo e4699f58310b85cbcb9e835e569e7e17  "$SIGNING_STORE_FILE" | md5sum --check -
       - name: Set up authentication for Google Cloud SDK
         uses: google-github-actions/auth@v2
         with:
@@ -81,30 +91,24 @@ jobs:
         with:
           version: '>= 363.0.0'
           project_id: mobile-app-build-290400
-      - name: Install AWS CLI
-        run: |
-          python3 -m pip install awscli
       - name: Download Samsung libraries
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }}
+          SAMSUNG_LIB: samsung_libs_v3.1_20231116
         run: |
-          aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \
-          cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/samsung_libs_v3.1_20231116.zip /tmp/ && \
-          unzip /tmp/samsung_libs_v3.1_20231116.zip -d /tmp/samsung_libs_v3.1_20231116 && \
-          rm /tmp/samsung_libs_v3.1_20231116.zip && \
+          gsutil cp gs://mobile-app-build-290400_github-actions/lib/v3.1/${SAMSUNG_LIB}.zip /tmp/ && \
+          unzip /tmp/${SAMSUNG_LIB}.zip -d /tmp/${SAMSUNG_LIB} && \
+          rm /tmp/${SAMSUNG_LIB}.zip && \
           mkdir -p mobile_back_samsung/samsung/lib/internal && \
-          mv /tmp/samsung_libs_v3.1_20231116/* mobile_back_samsung/samsung/lib/internal/
+          mv /tmp/${SAMSUNG_LIB}/* mobile_back_samsung/samsung/lib/internal/
       - name: Download QTI libraries
         env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.READONLY_AWS_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.READONLY_AWS_KEY }}
+          QTI_LIB: qaisw-2.13.4.230831213719_60417
         run: |
-          aws s3 --endpoint-url=https://s3.us-east-1.lyvecloud.seagate.com \
-          cp --quiet s3://mlcommons-mobile-wg-private/libs/v3.1/qaisw-2.13.4.230831213719_60417-linux.zip /tmp/ && \
-          unzip -P ${AWS_SECRET_ACCESS_KEY} /tmp/qaisw-2.13.4.230831213719_60417-linux.zip -d /tmp/qaisw-2.13.4.230831213719_60417-linux && \
-          rm /tmp/qaisw-2.13.4.230831213719_60417-linux.zip && \
-          mv /tmp/qaisw-2.13.4.230831213719_60417-linux/qaisw-2.13.4.230831213719_60417 mobile_back_qti/
+          gsutil cp gs://mobile-app-build-290400_github-actions/lib/v3.1/${QTI_LIB}.zip /tmp/ && \
+          unzip /tmp/${QTI_LIB}.zip -d /tmp/${QTI_LIB} && \
+          rm /tmp/${QTI_LIB}.zip && \
+          mkdir -p mobile_back_qti/${QTI_LIB} && \
+          mv /tmp/${QTI_LIB}/* mobile_back_qti/${QTI_LIB}/
       - name: Cache bazel
         uses: actions/cache@v3
         with:
@@ -175,12 +179,14 @@ jobs:
           WITH_APPLE: 0
         run: |
           make flutter/android/release
+          gsutil cp flutter/build/app/outputs/bundle/release/app-release.aab $GCLOUD_BUCKET_PATH/app-release.aab
+          gsutil cp flutter/build/app/outputs/flutter-apk/app-release.apk $GCLOUD_BUCKET_PATH/app-release.apk
       - name: Archive Android APKs
         uses: actions/upload-artifact@v3
         with:
           name: android-apks-${{ github.run_number }}
           path: output/android-apks/*.apk
-          retention-days: 30
+          retention-days: 28
           if-no-files-found: error
 
   test-android-apk-tflite:
@@ -197,15 +203,15 @@ jobs:
         with:
           version: '>= 363.0.0'
           project_id: mobile-app-build-290400
-      - name: Trigger Firebase Test Lab for [Google Pixel 5](redfin)
+      - name: Trigger Firebase Test Lab for [Google Pixel 6](oriole)
         run: |
           gcloud firebase test android run \
             --type instrumentation \
             --app $GCLOUD_BUCKET_PATH/test-main-tflite.apk \
             --test $GCLOUD_BUCKET_PATH/test-helper-tflite.apk \
             --timeout 30m \
-            --num-flaky-test-attempts 2 \
-            --device model=redfin,version=30,locale=en,orientation=portrait \
+            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
+            --device model=oriole,version=31,locale=en,orientation=portrait \
             --client-details=buildNumber=${{ github.run_number }}
 
   test-android-apk-pixel:
@@ -229,7 +235,7 @@ jobs:
             --app $GCLOUD_BUCKET_PATH/test-main-pixel.apk \
             --test $GCLOUD_BUCKET_PATH/test-helper-pixel.apk \
             --timeout 30m \
-            --num-flaky-test-attempts 2 \
+            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
             --device model=oriole,version=32,locale=en,orientation=portrait \
             --client-details=buildNumber=${{ github.run_number }}
 
@@ -254,7 +260,7 @@ jobs:
             --app $GCLOUD_BUCKET_PATH/test-main-qti.apk \
             --test $GCLOUD_BUCKET_PATH/test-helper-qti.apk \
             --timeout 30m \
-            --num-flaky-test-attempts 2 \
+            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
             --device model=b0q,version=33,locale=en,orientation=portrait \
             --client-details=buildNumber=${{ github.run_number }}
 
@@ -279,6 +285,36 @@ jobs:
             --app $GCLOUD_BUCKET_PATH/test-main-mtk.apk \
             --test $GCLOUD_BUCKET_PATH/test-helper-mtk.apk \
             --timeout 30m \
-            --num-flaky-test-attempts 2 \
+            --num-flaky-test-attempts ${{ vars.NUM_FLAKY_TEST_ATTEMPTS }} \
             --device model=OP515BL1,version=33,locale=en,orientation=portrait \
             --client-details=buildNumber=${{ github.run_number }}
+
+  deploy-playstore:
+    needs:
+      - build-android-apk
+      - test-android-apk-tflite
+    runs-on: ubuntu-22.04
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up authentication for Google Cloud SDK
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }}
+      - name: Set up Google Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+          version: '>= 363.0.0'
+          project_id: mobile-app-build-290400
+      - name: Download Android release app
+        run: |
+          gsutil cp $GCLOUD_BUCKET_PATH/app-release.aab /tmp/app-release.aab
+      - name: Upload Android release app to Google Play
+        uses: r0adkll/upload-google-play@v1
+        with:
+          releaseFiles: /tmp/app-release.aab
+          packageName: org.mlcommons.android.mlperfbench
+          serviceAccountJsonPlainText: ${{ secrets.GCLOUD_SERVICE_ACCOUNT_MOBILE_APP_BUILD }}
+          # track: "alpha", "beta", "internal" or "production"
+          track: internal
+          status: draft

.github/workflows/format-and-lint.yml

@@ -118,11 +118,6 @@ jobs:
         run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - name: markdown-format
         run: make lint/markdown
-      - name: markdown-link
-        uses: gaurav-nelson/github-action-markdown-link-check@v1
-        with:
-          use-verbose-mode: yes
-          use-quiet-mode: yes
 
   yaml:
     needs: build-image

WORKSPACE

@@ -1,6 +1,7 @@
 workspace(name = "mlperf_app")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
 
 http_archive(
     name = "bazel_skylib",
@@ -88,17 +89,14 @@ http_archive(
     urls = ["https://github.com/MediaTek-NeuroPilot/tflite-neuron-delegate/archive/refs/heads/update_for_dujac.zip"],
 )
 
-http_archive(
+new_git_repository(
     name = "org_mlperf_inference",
     build_file = "@//flutter/android/third_party:loadgen.BUILD",
+    commit = "238d035ab41d7ddd390b35471af169ea641380f6",
     patch_args = ["-p1"],
     patch_cmds = ["python3 loadgen/version_generator.py loadgen/version_generated.cc loadgen"],
     patches = [],
-    sha256 = "e664f980e84fcab3573447c0cc3adddd1fcf900367c5dcbff17179ece24c484e",
-    strip_prefix = "inference-2da0c52666e21e4b296b09e1dbd287bf3a814e96",
-    urls = [
-        "https://github.com/mlcommons/inference/archive/2da0c52666e21e4b296b09e1dbd287bf3a814e96.tar.gz",
-    ],
+    remote = "https://github.com/mlcommons/inference.git",
 )
 
 # This is required to pass SNPE SDK path from external environment to sources,

docs/build-and-run.md

@@ -114,6 +114,17 @@ See [Windows environment setup guide](./environment-setup/env-setup-windows.md#s
 Run `make OFFICIAL_BUILD=false FLUTTER_BUILD_NUMBER=0 flutter/android/release` to build APK from scratch,
 or run `make flutter/android/apk` after `make flutter/prepare` and you already have native libs.
 
+By default, the Android app will be signed with a debug key. To sign it with a release key for publishing to Play Store,
+follow the instructions at <https://docs.flutter.dev/deployment/android#sign-the-app> and set the environment variables:
+
+```shell
+export SIGNING_FOR_RELEASE=true
+export SIGNING_STORE_FILE=/path/to/keystore.jks
+export SIGNING_STORE_PASSWORD=abc
+export SIGNING_KEY_ALIAS=abc
+export SIGNING_KEY_PASSWORD=abc
+```
+
 You can build the app using docker.  
 Run `make OFFICIAL_BUILD=false FLUTTER_BUILD_NUMBER=0 docker/flutter/android/release` to build release APK.  
 Run `make docker/flutter/android/libs` to build just `.so` libs. This command is helpful if you want to build Android version of the app on Windows.

docs/result-spec.md

@@ -76,12 +76,22 @@ If you enable Submission mode, both `performance_run` and `accuracy_run` values
     Actual duration of the benchmark in seconds from start to finish.
   * `measured_samples`: integer number  
     Actual number of samples evaluated during the benchmark
-  * `loadgen_info`: map  
+  * `loadgen`: map  
     Info provided by loadgen. May be null for accuracy runs.
-    * `validity`: bool  
-      Indicates whether all constraints were satisfied or not.
-    * `duration`: floating point number  
-      Duration of the benchmark without loadgen overhead in seconds.
+    * `queryCount`: bool  
+      Number of queries performed.
+    * `latencyMean`: bool  
+      Mean latency in seconds.
+    * `latency90`: bool  
+      90th percentile in seconds.
+    * `isMinDurationMet`: bool
+      Indicates whether the min duration condition is met or not.
+    * `isMinQueryMet`: bool
+      Indicates whether the min query condition is met or not.
+    * `isEarlyStoppingMet`: bool
+      Indicates whether the early stopping condition is met or not.
+    * `isResultValid`: bool  
+      Indicates whether the result is valid or not.
   * `start_datetime`: string  
     Datetime of the moment when benchmark started  
     Format is Iso 8601 in UTC timezone: `2022-04-14T03:54:54.687Z`

flutter/android/android.mk

@@ -19,7 +19,7 @@ ANDROID_NDK_VERSION?=25
 ANDROID_NDK_API_LEVEL?=33
 
 flutter/android: flutter/android/libs flutter/check/firebase-env
-flutter/android/release: flutter/check-release-env flutter/android flutter/prepare flutter/android/apk
+flutter/android/release: flutter/check-release-env flutter/android flutter/prepare flutter/android/apk flutter/android/appbundle
 flutter/android/libs: flutter/android/libs/checksum flutter/android/libs/build flutter/android/libs/copy
 # run `make flutter/android/apk` before `flutter/android/test-apk`
 flutter/android/test-apk: flutter/android/test-apk/main flutter/android/test-apk/helper
@@ -76,7 +76,16 @@ flutter/android/apk:
 		${flutter_firebase_crashlytics_arg} \
 		${flutter_build_number_arg} \
 		${flutter_folder_args}
-	cp -f flutter/build/app/outputs/flutter-apk/app-release.apk ${flutter_android_apk_release_path}
+	cp -f flutter/build/app/outputs/flutter-apk/app-release.apk ${flutter_android_apk_release_path}.apk
+.PHONY: flutter/android/appbundle
+flutter/android/appbundle:
+	mkdir -p $$(dirname ${flutter_android_apk_release_path})
+	cd flutter && ${_start_args} flutter --no-version-check build appbundle \
+		${flutter_official_build_arg} \
+		${flutter_firebase_crashlytics_arg} \
+		${flutter_build_number_arg} \
+		${flutter_folder_args}
+	cp -f flutter/build/app/outputs/bundle/release/app-release.aab ${flutter_android_apk_release_path}.aab
 
 FLUTTER_ANDROID_APK_TEST_MAIN?=test-main.apk
 flutter_android_apk_test_main_path=${FLUTTER_ANDROID_APK_FOLDER}/${FLUTTER_ANDROID_APK_TEST_MAIN}

flutter/android/app/build.gradle

@@ -54,17 +54,32 @@ android {
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
 
+   signingConfigs {
+       release {
+           keyAlias System.env['SIGNING_KEY_ALIAS']
+           keyPassword System.env['SIGNING_KEY_PASSWORD']
+           storeFile System.env['SIGNING_STORE_FILE'] ? file(System.env['SIGNING_STORE_FILE']) : null
+           storePassword System.env['SIGNING_STORE_PASSWORD']
+       }
+   }
+
     buildTypes {
         release {
-            // TODO: Add your own signing config for the release build.
-            // Signing with the debug keys for now, so `flutter run --release` works.
-            signingConfig signingConfigs.debug
-
+            // https://docs.flutter.dev/deployment/android#sign-the-app
+            if (System.env['SIGNING_FOR_RELEASE'] == 'true') {
+                println 'Using signingConfigs.release'
+                signingConfig signingConfigs.release
+            } else {
+                println 'Using signingConfigs.debug'
+                signingConfig signingConfigs.debug
+            }
             // By default we do not upload mapping file to Firebase
             firebaseCrashlytics {
                 if (System.env['FIREBASE_CRASHLYTICS_ENABLED'] == 'true') {
+                    println 'mappingFileUploadEnabled set to true'
                     mappingFileUploadEnabled true
                 } else {
+                    println 'mappingFileUploadEnabled set to false'
                     mappingFileUploadEnabled false
                 }
             }

flutter/cpp/mlperf_driver.cc

@@ -105,6 +105,7 @@ void MlperfDriver::RunMLPerfTest(const std::string& mode, int min_query_count,
   // See https://github.com/mlcommons/inference/issues/1397
   mlperf_settings.max_duration_ms =
       static_cast<uint64_t>(std::ceil(max_duration * 1000.0));
+  mlperf_settings.enforce_max_duration = true;
 
   if (scenario_ == "Offline") {
     mlperf_settings.scenario = ::mlperf::TestScenario::Offline;