[v4.0] feat: add new tasks for image classification v2

.bazelrc

@@ -21,7 +21,7 @@ build:android --copt=-w
 build:ios --copt=-w
 build:windows --copt=/W0
 
-# Build in C++ 14 mode.
+# Build in C++ 17 mode.
 build:android --cxxopt=-std=c++17
 build:android --host_cxxopt=-std=c++17
 build:ios --cxxopt=-std=c++17

flutter/assets/tasks.pbtxt

@@ -2,8 +2,8 @@
 # proto-message: MLPerfConfig
 
 task {
-  id: "image_classification"
-  name: "Image Classification"
+  id: "image_classification_v2"
+  name: "Image Classification v2"
   min_query_count: 1024
   min_duration: 60
   max_duration: 600
@@ -29,11 +29,11 @@ task {
     }
   }
   model {
-    id: "mobilenetedgetpu_224_v1_0"
-    name: "MobileNetEdgeTPU 224 1.0"
-    offset: 1
-    image_width: 224
-    image_height: 224
+    id: "MobileNetEdgeTPUv2-L"
+    name: "MobileNetEdgeTPUv2-L"
+    offset: 0
+    image_width: 384
+    image_height: 384
     num_classes: 1000
   }
 }
@@ -77,7 +77,7 @@ task {
 
 task {
   id: "image_segmentation_v2"
-  name: "v2.0 Image Segmentation"
+  name: "Image Segmentation v2"
   min_query_count: 1024
   min_duration: 60
   max_duration: 600
@@ -179,6 +179,80 @@ task {
   }
 }
 
+task {
+  id: "image_classification_offline_v2"
+  name: "Image Classification v2 (Offline)"
+  min_query_count: 24576
+  min_duration: 0
+  max_duration: 0
+  max_throughput: 2000
+  max_accuracy: 1.0
+  scenario: "Offline"
+  datasets {
+    type: IMAGENET
+    full {
+      name: "Imagenet classification validation set"
+      input_path: "local:///mlperf_datasets/imagenet/img"
+      groundtruth_path: "local:///mlperf_datasets/imagenet/imagenet_val_full.txt"
+    }
+    lite {
+      name: "Open images subset for classification"
+      input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet.zip"
+      groundtruth_path: ""
+    }
+    tiny {
+      name: "Imagenet dataset for integration test"
+      input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet_tiny.zip"
+      groundtruth_path: "https://github.com/mlcommons/mobile_models/raw/main/v3_0/assets/imagenet_tiny-groundtruth.txt"
+    }
+  }
+  model {
+    id: "MobileNetEdgeTPUv2-L"
+    name: "MobileNetEdgeTPUv2-L"
+    offset: 0
+    image_width: 384
+    image_height: 384
+    num_classes: 1000
+  }
+}
+
+task {
+  id: "image_classification"
+  name: "Image Classification"
+  min_query_count: 1024
+  min_duration: 60
+  max_duration: 600
+  max_throughput: 1000
+  max_accuracy: 1.0
+  scenario: "SingleStream"
+  datasets {
+    type: IMAGENET
+    full {
+      name: "Imagenet classification validation set"
+      input_path: "local:///mlperf_datasets/imagenet/img"
+      groundtruth_path: "local:///mlperf_datasets/imagenet/imagenet_val_full.txt"
+    }
+    lite {
+      name: "Open images subset for classification"
+      input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet.zip"
+      groundtruth_path: ""
+    }
+    tiny {
+      name: "Imagenet dataset for integration test"
+      input_path: "https://github.com/mlcommons/mobile_models/raw/main/v0_7/datasets/imagenet_tiny.zip"
+      groundtruth_path: "https://github.com/mlcommons/mobile_models/raw/main/v3_0/assets/imagenet_tiny-groundtruth.txt"
+    }
+  }
+  model {
+    id: "mobilenetedgetpu_224_v1_0"
+    name: "MobileNetEdgeTPU 224 1.0"
+    offset: 1
+    image_width: 224
+    image_height: 224
+    num_classes: 1000
+  }
+}
+
 task {
   id: "image_classification_offline"
   name: "Image Classification (Offline)"

flutter/integration_test/expected_accuracy.dart

@@ -27,6 +27,17 @@ const Map<String, Interval> _imageClassification = {
   'neuron-mdla': Interval(min: 0.88, max: 0.91),
 };
 
+const Map<String, Interval> _imageClassificationV2 = {
+  'cpu': Interval(min: 0.88, max: 0.91),
+  'npu': Interval(min: 0.88, max: 0.91),
+  'tpu': Interval(min: 0.88, max: 0.91),
+  'ane': Interval(min: 0.79, max: 0.91),
+  'cpu&gpu&ane': Interval(min: 0.79, max: 0.91),
+  'snpe_dsp': Interval(min: 0.88, max: 0.91),
+  'psnpe_dsp': Interval(min: 0.88, max: 0.91),
+  'neuron-mdla': Interval(min: 0.79, max: 0.81),
+};
+
 const Map<String, Interval> _objectDetection = {
   'cpu': Interval(min: 0.31, max: 0.32),
   'npu': Interval(min: 0.28, max: 0.31),
@@ -77,9 +88,11 @@ const Map<String, Interval> _superResolution = {
 
 const benchmarkExpectedAccuracy = {
   BenchmarkId.imageClassification: _imageClassification,
+  BenchmarkId.imageClassificationV2: _imageClassificationV2,
   BenchmarkId.objectDetection: _objectDetection,
   BenchmarkId.imageSegmentationV2: _imageSegmentation,
   BenchmarkId.naturalLanguageProcessing: _naturalLanguageProcessing,
   BenchmarkId.superResolution: _superResolution,
   BenchmarkId.imageClassificationOffline: _imageClassification,
+  BenchmarkId.imageClassificationOfflineV2: _imageClassificationV2,
 };

flutter/integration_test/expected_throughput.dart

@@ -40,10 +40,10 @@ const Map<String, Map<String, Interval>> _imageClassification = {
     _kIphoneOnMacbookM1: Interval(min: 19, max: 27),
   },
   _kCoreMLBackend: {
-    _kIphoneOnGitHubAction: Interval(min: 1, max: 8),
+    _kIphoneOnGitHubAction: Interval(min: 1, max: 12),
   },
   _kPixelBackend: {
-    _kPixel6: Interval(min: 800, max: 1100),
+    _kPixel6: Interval(min: 800, max: 1200),
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 1700, max: 2400),
@@ -53,6 +53,30 @@ const Map<String, Map<String, Interval>> _imageClassification = {
   },
 };
 
+const Map<String, Map<String, Interval>> _imageClassificationV2 = {
+  _kTFLiteBackend: {
+    _kCloudBuildX23: Interval(min: 1, max: 9),
+    _kCloudBuildX28: Interval(min: 1, max: 9),
+    _kRyzen5600: Interval(min: 1, max: 37),
+    _kPixel5: Interval(min: 20, max: 75),
+    _kPixel6: Interval(min: 100, max: 600),
+    _kIphoneOnGitHubAction: Interval(min: 0.4, max: 9),
+    _kIphoneOnMacbookM1: Interval(min: 10, max: 27),
+  },
+  _kCoreMLBackend: {
+    _kIphoneOnGitHubAction: Interval(min: 0.4, max: 9),
+  },
+  _kPixelBackend: {
+    _kPixel6: Interval(min: 100, max: 600),
+  },
+  _kQtiBackend: {
+    _kS22Ultra: Interval(min: 1700, max: 2400),
+  },
+  _kMediatekBackend: {
+    _kDN2103: Interval(min: 5, max: 90),
+  },
+};
+
 const Map<String, Map<String, Interval>> _objectDetection = {
   _kTFLiteBackend: {
     _kCloudBuildX23: Interval(min: 4, max: 7),
@@ -64,10 +88,10 @@ const Map<String, Map<String, Interval>> _objectDetection = {
     _kIphoneOnMacbookM1: Interval(min: 9, max: 16),
   },
   _kCoreMLBackend: {
-    _kIphoneOnGitHubAction: Interval(min: 0.5, max: 4),
+    _kIphoneOnGitHubAction: Interval(min: 0.5, max: 7),
   },
   _kPixelBackend: {
-    _kPixel6: Interval(min: 250, max: 450),
+    _kPixel6: Interval(min: 200, max: 500),
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 700, max: 1400),
@@ -88,10 +112,10 @@ const Map<String, Map<String, Interval>> _imageSegmentationV2 = {
     _kIphoneOnMacbookM1: Interval(min: 3, max: 6),
   },
   _kCoreMLBackend: {
-    _kIphoneOnGitHubAction: Interval(min: 0.1, max: 2.5),
+    _kIphoneOnGitHubAction: Interval(min: 0.1, max: 3.5),
   },
   _kPixelBackend: {
-    _kPixel6: Interval(min: 100, max: 180),
+    _kPixel6: Interval(min: 100, max: 190),
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 400, max: 700),
@@ -112,11 +136,10 @@ const Map<String, Map<String, Interval>> _naturalLanguageProcessing = {
     _kIphoneOnMacbookM1: Interval(min: 1.8, max: 3),
   },
   _kCoreMLBackend: {
-    _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1),
+    _kIphoneOnGitHubAction: Interval(min: 0.1, max: 1.1),
   },
   _kPixelBackend: {
-    // pixel some time finish this task in 4 seconds, not sure why.
-    _kPixel6: Interval(min: 2, max: 75),
+    _kPixel6: Interval(min: 2, max: 85),
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 100, max: 180),
@@ -140,7 +163,7 @@ const Map<String, Map<String, Interval>> _superResolution = {
     _kIphoneOnGitHubAction: Interval(min: 0.02, max: 1.0),
   },
   _kPixelBackend: {
-    _kPixel6: Interval(min: 10, max: 14),
+    _kPixel6: Interval(min: 7, max: 17),
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 25, max: 55),
@@ -164,7 +187,7 @@ const Map<String, Map<String, Interval>> _imageClassificationOffline = {
     _kIphoneOnGitHubAction: Interval(min: 2, max: 20),
   },
   _kPixelBackend: {
-    _kPixel6: Interval(min: 1000, max: 1700),
+    _kPixel6: Interval(min: 900, max: 1800),
   },
   _kQtiBackend: {
     _kS22Ultra: Interval(min: 2500, max: 3500),
@@ -174,11 +197,37 @@ const Map<String, Map<String, Interval>> _imageClassificationOffline = {
   },
 };
 
+const Map<String, Map<String, Interval>> _imageClassificationOfflineV2 = {
+  _kTFLiteBackend: {
+    _kCloudBuildX23: Interval(min: 1, max: 9),
+    _kCloudBuildX28: Interval(min: 1, max: 9),
+    _kRyzen5600: Interval(min: 20, max: 60),
+    _kPixel5: Interval(min: 20, max: 180),
+    _kPixel6: Interval(min: 100, max: 700),
+    _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15),
+    _kIphoneOnMacbookM1: Interval(min: 10, max: 45),
+  },
+  _kCoreMLBackend: {
+    _kIphoneOnGitHubAction: Interval(min: 0.4, max: 15),
+  },
+  _kPixelBackend: {
+    _kPixel6: Interval(min: 100, max: 700),
+  },
+  _kQtiBackend: {
+    _kS22Ultra: Interval(min: 2600, max: 3000),
+  },
+  _kMediatekBackend: {
+    _kDN2103: Interval(min: 5, max: 90),
+  },
+};
+
 const benchmarkExpectedThroughput = {
   BenchmarkId.imageClassification: _imageClassification,
+  BenchmarkId.imageClassificationV2: _imageClassificationV2,
   BenchmarkId.objectDetection: _objectDetection,
   BenchmarkId.imageSegmentationV2: _imageSegmentationV2,
   BenchmarkId.naturalLanguageProcessing: _naturalLanguageProcessing,
   BenchmarkId.superResolution: _superResolution,
   BenchmarkId.imageClassificationOffline: _imageClassificationOffline,
+  BenchmarkId.imageClassificationOfflineV2: _imageClassificationOfflineV2,
 };

flutter/integration_test/first_test.dart

@@ -1,5 +1,6 @@
 import 'package:flutter_test/flutter_test.dart';
 import 'package:integration_test/integration_test.dart';
+import 'package:mlperfbench/app_constants.dart';
 import 'package:mlperfbench/benchmark/run_mode.dart';
 import 'package:mlperfbench/firebase/firebase_manager.dart';
 import 'package:mlperfbench/firebase/firebase_options.gen.dart';
@@ -32,7 +33,7 @@ void main() {
   if (enablePerfTest) {
     prefs[StoreConstants.testMinDuration] = 15;
     prefs[StoreConstants.testMinQueryCount] = 64;
-    prefs[StoreConstants.testCooldownDuration] = 10;
+    prefs[StoreConstants.testCooldownDuration] = 2;
   }
   SharedPreferences.setMockInitialValues(prefs);
 
@@ -57,10 +58,10 @@ void main() {
 }
 
 void checkTasks(ExtendedResult extendedResult) {
-  final length = extendedResult.results.length;
-  const expectedTasksCount = 6;
+  final tasksCount = extendedResult.results.length;
+  final expectedTasksCount = BenchmarkId.allIds.length;
 
-  expect(length, expectedTasksCount, reason: 'tasks count does not match');
+  expect(tasksCount, expectedTasksCount, reason: 'tasks count does not match');
 
   for (final benchmarkResult in extendedResult.results) {
     print('checking ${benchmarkResult.benchmarkId}');

flutter/integration_test/utils.dart

@@ -34,23 +34,31 @@ Future<void> validateSettings(WidgetTester tester) async {
   final state = tester.state(find.byType(MaterialApp));
   final benchmarkState = state.context.read<BenchmarkState>();
   for (var benchmark in benchmarkState.benchmarks) {
-    expect(benchmark.selectedDelegate.batchSize, greaterThanOrEqualTo(0));
-    expect(benchmark.selectedDelegate.modelPath.isNotEmpty, isTrue);
-    expect(benchmark.selectedDelegate.modelChecksum.isNotEmpty, isTrue);
-    expect(benchmark.selectedDelegate.acceleratorName.isNotEmpty, isTrue);
-    expect(benchmark.selectedDelegate.acceleratorDesc.isNotEmpty, isTrue);
-    expect(benchmark.benchmarkSettings.framework.isNotEmpty, isTrue);
+    expect(benchmark.selectedDelegate.batchSize, greaterThanOrEqualTo(0),
+        reason: 'batchSize must >= 0');
+    expect(benchmark.selectedDelegate.modelPath.isNotEmpty, isTrue,
+        reason: 'modelPath cannot be empty');
+    expect(benchmark.selectedDelegate.modelChecksum.isNotEmpty, isTrue,
+        reason: 'modelChecksum cannot be empty');
+    expect(benchmark.selectedDelegate.acceleratorName.isNotEmpty, isTrue,
+        reason: 'acceleratorName cannot be empty');
+    expect(benchmark.selectedDelegate.acceleratorDesc.isNotEmpty, isTrue,
+        reason: 'acceleratorDesc cannot be empty');
+    expect(benchmark.benchmarkSettings.framework.isNotEmpty, isTrue,
+        reason: 'framework cannot be empty');
     expect(benchmark.selectedDelegate.delegateName,
-        equals(benchmark.benchmarkSettings.delegateSelected));
+        equals(benchmark.benchmarkSettings.delegateSelected),
+        reason: 'delegateSelected must be the same as delegateName');
 
     final selected = benchmark.benchmarkSettings.delegateSelected;
     final choices = benchmark.benchmarkSettings.delegateChoice
         .map((e) => e.delegateName)
         .toList();
-    expect(choices.isNotEmpty, isTrue);
-    final reason =
-        'delegate_selected=$selected must be one of delegate_choice=$choices';
-    expect(choices.contains(selected), isTrue, reason: reason);
+    expect(choices.isNotEmpty, isTrue,
+        reason: 'There must be at least one delegate choice');
+    expect(choices.contains(selected), isTrue,
+        reason:
+            'delegate_selected=$selected must be one of delegate_choice=$choices');
   }
 }
 

flutter/lib/app_constants.dart

@@ -24,16 +24,20 @@ class BenchmarkId {
   static const objectDetection = 'object_detection';
   static const imageSegmentationV2 = 'image_segmentation_v2';
   static const naturalLanguageProcessing = 'natural_language_processing';
-  static const imageClassificationOffline = 'image_classification_offline';
   static const superResolution = 'super_resolution';
+  static const imageClassificationOffline = 'image_classification_offline';
+  static const imageClassificationV2 = 'image_classification_v2';
+  static const imageClassificationOfflineV2 = 'image_classification_offline_v2';
 
   static const allIds = [
     imageClassification,
     objectDetection,
     imageSegmentationV2,
     naturalLanguageProcessing,
-    imageClassificationOffline,
     superResolution,
+    imageClassificationOffline,
+    imageClassificationV2,
+    imageClassificationOfflineV2,
   ];
 }