diff --git a/flutter/assets/tasks.pbtxt b/flutter/assets/tasks.pbtxt index 7db04db0e..a78283118 100644 --- a/flutter/assets/tasks.pbtxt +++ b/flutter/assets/tasks.pbtxt @@ -42,10 +42,10 @@ task { } tiny { name: "Imagenet dataset for integration test" - input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v0_7/imagenet_tiny.zip" - input_checksum: "42cb83d80d0341d6719d68b32322adf2" - groundtruth_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v3_0/imagenet_tiny-groundtruth.txt" - groundtruth_checksum: "cc6d83ca25daeb2475f36c079e53bc26" + input_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v5_0/imagenet_tiny.zip" + input_checksum: "2cde6f1d741b9c1834db261ac6e43a6e" + groundtruth_path: "https://mobile.mlcommons-storage.org/app-resources/datasets/v5_0/imagenet_tiny_val.txt" + groundtruth_checksum: "086f99eff3d1b93fd41e7f35498570a7" } } model { diff --git a/flutter/integration_test/expected_accuracy.dart b/flutter/integration_test/expected_accuracy.dart index 712bee11b..6533c6d1a 100644 --- a/flutter/integration_test/expected_accuracy.dart +++ b/flutter/integration_test/expected_accuracy.dart @@ -18,15 +18,27 @@ key: OR | */ const Map _imageClassificationV2 = { + 'cpu': Interval(min: 0.82, max: 0.84), + 'npu': Interval(min: 0.80, max: 0.82), + 'tpu': Interval(min: 0.82, max: 0.84), + 'ane': Interval(min: 0.69, max: 0.91), + 'cpu&gpu&ane': Interval(min: 0.69, max: 0.91), + 'snpe_dsp': Interval(min: 0.83, max: 0.85), + 'psnpe_dsp': Interval(min: 0.83, max: 0.85), + 'neuron-mdla': Interval(min: 0.81, max: 0.83), + 'samsung_npu': Interval(min: 0.83, max: 0.85), +}; + +const Map _imageClassificationOfflineV2 = { 'cpu': Interval(min: 0.88, max: 0.91), - 'npu': Interval(min: 0.69, max: 0.91), - 'tpu': Interval(min: 0.88, max: 0.91), + 'npu': Interval(min: 0.69, max: 0.71), + 'tpu': Interval(min: 0.89, max: 0.91), 'ane': Interval(min: 0.69, max: 0.91), 'cpu&gpu&ane': Interval(min: 0.69, max: 0.91), - 'snpe_dsp': Interval(min: 0.88, max: 0.91), - 'psnpe_dsp': Interval(min: 0.88, max: 0.91), - 'neuron-mdla': Interval(min: 0.79, max: 0.91), - 'samsung_npu': Interval(min: 0.99, max: 1.0), + 'snpe_dsp': Interval(min: 0.89, max: 0.91), + 'psnpe_dsp': Interval(min: 0.89, max: 0.91), + 'neuron-mdla': Interval(min: 0.79, max: 0.81), + 'samsung_npu': Interval(min: 0.98, max: 1.0), }; const Map _objectDetection = { @@ -102,5 +114,5 @@ const benchmarkExpectedAccuracy = { BenchmarkId.naturalLanguageProcessing: _naturalLanguageProcessing, BenchmarkId.superResolution: _superResolution, BenchmarkId.stableDiffusion: _stableDiffusion, - BenchmarkId.imageClassificationOfflineV2: _imageClassificationV2, + BenchmarkId.imageClassificationOfflineV2: _imageClassificationOfflineV2, };