diff --git a/.github/workflows/update-results.yml b/.github/workflows/update-results.yml index b48d6e3..ee4468c 100644 --- a/.github/workflows/update-results.yml +++ b/.github/workflows/update-results.yml @@ -35,7 +35,7 @@ jobs: cm run script --tags=convert,from-csv,to-md --csv_file=summary.csv --md_file=README.md USER="arjunsuresh" - EMAIL=admin@gateoverflow.com + EMAIL=admin@gateoverflow.org git config --global user.name "$USER" git config --global user.email "$EMAIL" diff --git a/README.md b/README.md index d5643d2..56b95f7 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,32 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2024-10-07 21:31:15,197 submission_checker1.py:2936 INFO] Results=9, NoResults=0, Power Results=0 -[2024-10-07 21:31:15,197 submission_checker1.py:2943 INFO] --- -[2024-10-07 21:31:15,197 submission_checker1.py:2944 INFO] Closed Results=0, Closed Power Results=0 +[2024-10-07 21:33:27,305 submission_checker1.py:2936 INFO] Results=9, NoResults=0, Power Results=0 +[2024-10-07 21:33:27,305 submission_checker1.py:2943 INFO] --- +[2024-10-07 21:33:27,305 submission_checker1.py:2944 INFO] Closed Results=0, Closed Power Results=0 -[2024-10-07 21:31:15,198 submission_checker1.py:2949 INFO] Open Results=9, Open Power Results=0 +[2024-10-07 21:33:27,305 submission_checker1.py:2949 INFO] Open Results=9, Open Power Results=0 -[2024-10-07 21:31:15,198 submission_checker1.py:2954 INFO] Network Results=0, Network Power Results=0 +[2024-10-07 21:33:27,305 submission_checker1.py:2954 INFO] Network Results=0, Network Power Results=0 -[2024-10-07 21:31:15,198 submission_checker1.py:2959 INFO] --- -[2024-10-07 21:31:15,198 submission_checker1.py:2961 INFO] Systems=8, Power Systems=0 -[2024-10-07 21:31:15,198 submission_checker1.py:2962 INFO] Closed Systems=0, Closed Power Systems=0 -[2024-10-07 21:31:15,198 submission_checker1.py:2967 INFO] Open Systems=8, Open Power Systems=0 -[2024-10-07 21:31:15,198 submission_checker1.py:2972 INFO] Network Systems=0, Network Power Systems=0 -[2024-10-07 21:31:15,198 submission_checker1.py:2977 INFO] --- -[2024-10-07 21:31:15,198 submission_checker1.py:2982 INFO] SUMMARY: submission looks OK +[2024-10-07 21:33:27,305 submission_checker1.py:2959 INFO] --- +[2024-10-07 21:33:27,305 submission_checker1.py:2961 INFO] Systems=8, Power Systems=0 +[2024-10-07 21:33:27,305 submission_checker1.py:2962 INFO] Closed Systems=0, Closed Power Systems=0 +[2024-10-07 21:33:27,305 submission_checker1.py:2967 INFO] Open Systems=8, Open Power Systems=0 +[2024-10-07 21:33:27,305 submission_checker1.py:2972 INFO] Network Systems=0, Network Power Systems=0 +[2024-10-07 21:33:27,305 submission_checker1.py:2977 INFO] --- +[2024-10-07 21:33:27,305 submission_checker1.py:2982 INFO] SUMMARY: submission looks OK INFO:root: ! call "postprocess" from /home/runner/CM/repos/mlcommons@cm4mlops/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:-------------|:-------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:--------------------------------------------------------------|------------------:|:----------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:----------------------------------------------------------------------------------------------------------|:---------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.345721 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 52.9478 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v4.1 | 0 | False | Tokens/s | fp32 | -| 2 | MLCommons | available | open | datacenter | 48ed6105bd85 | 48ed6105bd85-nvidia-gpu-TensorRT-scc24-main | stable-diffusion-xl | stable-diffusion-xl | Offline | 1.13292 | CLIP_SCORE: 15.586050063371658 FID_SCORE: 236.8087101317688 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/48ed6105bd85-nvidia-gpu-TensorRT-scc24-main/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.6. | 1 | 0 | v4.1 | 0 | False | Samples/s | int8 | -| 3 | MLCommons | available | open | datacenter | 48ed6105bd85 | 48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.373636 | CLIP_SCORE: 15.236237794160843 FID_SCORE: 238.78369342212613 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.6. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | datacenter | f9ac88850adc | f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.376944 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | datacenter | 48ed6105bd85 | 48ed6105bd85-nvidia-gpu-TensorRT-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 1.13598 | CLIP_SCORE: 15.586050063371658 FID_SCORE: 236.8087101317688 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/48ed6105bd85-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.6. | 1 | 0 | v4.1 | 0 | False | Samples/s | int8 | -| 6 | MLCommons | available | open | datacenter | 13fce262fb79 | 13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.375843 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374549 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | e8dbfdd7ca14 | e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 1.13976 | CLIP_SCORE: 15.617164582014084 FID_SCORE: 233.28573786792805 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | int8 | \ No newline at end of file +| 0 | MLCommons | available | open | datacenter | 48ed6105bd85 | 48ed6105bd85-nvidia-gpu-TensorRT-scc24-main | stable-diffusion-xl | stable-diffusion-xl | Offline | 1.13292 | CLIP_SCORE: 15.586050063371658 FID_SCORE: 236.8087101317688 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/48ed6105bd85-nvidia-gpu-TensorRT-scc24-main/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.6. | 1 | 0 | v4.1 | 0 | False | Samples/s | int8 | +| 1 | MLCommons | available | open | datacenter | e8dbfdd7ca14 | e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 1.13976 | CLIP_SCORE: 15.617164582014084 FID_SCORE: 233.28573786792805 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | int8 | +| 2 | MLCommons | available | open | datacenter | 48ed6105bd85 | 48ed6105bd85-nvidia-gpu-TensorRT-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 1.13598 | CLIP_SCORE: 15.586050063371658 FID_SCORE: 236.8087101317688 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/48ed6105bd85-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.6. | 1 | 0 | v4.1 | 0 | False | Samples/s | int8 | +| 3 | MLCommons | available | open | datacenter | 13fce262fb79 | 13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.375843 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 52.9478 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v4.1 | 0 | False | Tokens/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.345721 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | 48ed6105bd85 | 48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.373636 | CLIP_SCORE: 15.236237794160843 FID_SCORE: 238.78369342212613 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base/stable-diffusion-xl/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v2.3.6. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | f9ac88850adc | f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.376944 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374549 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v4.1 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/summary.csv b/summary.csv index 36c9cce..a9082e8 100644 --- a/summary.csv +++ b/summary.csv @@ -1,10 +1,10 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,stable-diffusion-xl,stable-diffusion-xl,Offline,0.345721,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v4.1,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,52.9478,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v4.1,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","48ed6105bd85",48ed6105bd85-nvidia-gpu-TensorRT-scc24-main,stable-diffusion-xl,stable-diffusion-xl,Offline,1.13292,CLIP_SCORE: 15.586050063371658 FID_SCORE: 236.8087101317688,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/48ed6105bd85-nvidia-gpu-TensorRT-scc24-main/stable-diffusion-xl/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v2.3.6. ",1,0,v4.1,0,False,Samples/s,"int8" -MLCommons,available,open,"datacenter","48ed6105bd85",48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.373636,CLIP_SCORE: 15.236237794160843 FID_SCORE: 238.78369342212613,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base/stable-diffusion-xl/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v2.3.6. ",1,0,v4.1,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","f9ac88850adc",f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.376944,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","e8dbfdd7ca14",e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,1.13976,CLIP_SCORE: 15.617164582014084 FID_SCORE: 233.28573786792805,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"int8" MLCommons,available,open,"datacenter","48ed6105bd85",48ed6105bd85-nvidia-gpu-TensorRT-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,1.13598,CLIP_SCORE: 15.586050063371658 FID_SCORE: 236.8087101317688,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/48ed6105bd85-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v2.3.6. ",1,0,v4.1,0,False,Samples/s,"int8" MLCommons,available,open,"datacenter","13fce262fb79",13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.375843,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,52.9478,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v4.1,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,stable-diffusion-xl,stable-diffusion-xl,Offline,0.345721,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v4.1,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","48ed6105bd85",48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.373636,CLIP_SCORE: 15.236237794160843 FID_SCORE: 238.78369342212613,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base/stable-diffusion-xl/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v2.3.6. ",1,0,v4.1,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","f9ac88850adc",f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.376944,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374549,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","e8dbfdd7ca14",e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,1.13976,CLIP_SCORE: 15.617164582014084 FID_SCORE: 233.28573786792805,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base/stable-diffusion-xl/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v2.3.9. ",1,0,v4.1,0,False,Samples/s,"int8" diff --git a/summary.json b/summary.json index c96ff3d..fb4ceb4 100644 --- a/summary.json +++ b/summary.json @@ -3,61 +3,61 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.4.1-default_config", + "SystemType":"datacenter", + "SystemName":"48ed6105bd85", + "Platform":"48ed6105bd85-nvidia-gpu-TensorRT-scc24-main", "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.345721, - "Accuracy":"CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006", + "Result":1.13292, + "Accuracy":"CLIP_SCORE: 15.586050063371658 FID_SCORE: 236.8087101317688", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.4.1-default_config\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.4.1", - "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":"Automated by MLCommons CM v2.3.4. ", + "Location":"open\/MLCommons\/results\/48ed6105bd85-nvidia-gpu-TensorRT-scc24-main\/stable-diffusion-xl\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", + "notes":"Automated by MLCommons CM v2.3.6. ", "compliance":1, "errors":0, "version":"v4.1", "inferred":0, "has_power":false, "Units":"Samples\/s", - "weight_data_types":"fp32" + "weight_data_types":"int8" }, "1":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.4.1-default_config", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemType":"datacenter", + "SystemName":"e8dbfdd7ca14", + "Platform":"e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":52.9478, - "Accuracy":null, + "Result":1.13976, + "Accuracy":"CLIP_SCORE: 15.617164582014084 FID_SCORE: 233.28573786792805", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.4.1-default_config\/gptj-99\/offline", - "framework":"pytorch v2.4.1", - "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":"Automated by MLCommons CM v2.3.4. ", + "Location":"open\/MLCommons\/results\/e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base\/stable-diffusion-xl\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", + "notes":"Automated by MLCommons CM v2.3.9. ", "compliance":1, "errors":0, "version":"v4.1", "inferred":0, "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" + "Units":"Samples\/s", + "weight_data_types":"int8" }, "2":{ "Organization":"MLCommons", @@ -65,11 +65,11 @@ "Division":"open", "SystemType":"datacenter", "SystemName":"48ed6105bd85", - "Platform":"48ed6105bd85-nvidia-gpu-TensorRT-scc24-main", + "Platform":"48ed6105bd85-nvidia-gpu-TensorRT-scc24-base", "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":1.13292, + "Result":1.13598, "Accuracy":"CLIP_SCORE: 15.586050063371658 FID_SCORE: 236.8087101317688", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", @@ -77,7 +77,7 @@ "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/48ed6105bd85-nvidia-gpu-TensorRT-scc24-main\/stable-diffusion-xl\/offline", + "Location":"open\/MLCommons\/results\/48ed6105bd85-nvidia-gpu-TensorRT-scc24-base\/stable-diffusion-xl\/offline", "framework":"TensorRT", "operating_system":"Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", "notes":"Automated by MLCommons CM v2.3.6. ", @@ -94,23 +94,23 @@ "Availability":"available", "Division":"open", "SystemType":"datacenter", - "SystemName":"48ed6105bd85", - "Platform":"48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base", + "SystemName":"13fce262fb79", + "Platform":"13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base", "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.373636, - "Accuracy":"CLIP_SCORE: 15.236237794160843 FID_SCORE: 238.78369342212613", + "Result":0.375843, + "Accuracy":"CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base\/stable-diffusion-xl\/offline", - "framework":"TensorRT", - "operating_system":"Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "notes":"Automated by MLCommons CM v2.3.6. ", + "Location":"open\/MLCommons\/results\/13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.4.1", + "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", + "notes":"Automated by MLCommons CM v2.3.9. ", "compliance":1, "errors":0, "version":"v4.1", @@ -123,84 +123,84 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"f9ac88850adc", - "Platform":"f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemType":"edge", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.4.1-default_config", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.376944, - "Accuracy":"CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006", + "Result":52.9478, + "Accuracy":null, "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base\/stable-diffusion-xl\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.4.1-default_config\/gptj-99\/offline", "framework":"pytorch v2.4.1", "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":"Automated by MLCommons CM v2.3.9. ", + "notes":"Automated by MLCommons CM v2.3.4. ", "compliance":1, "errors":0, "version":"v4.1", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"48ed6105bd85", - "Platform":"48ed6105bd85-nvidia-gpu-TensorRT-scc24-base", + "SystemType":"edge", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.4.1-default_config", "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":1.13598, - "Accuracy":"CLIP_SCORE: 15.586050063371658 FID_SCORE: 236.8087101317688", + "Result":0.345721, + "Accuracy":"CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/48ed6105bd85-nvidia-gpu-TensorRT-scc24-base\/stable-diffusion-xl\/offline", - "framework":"TensorRT", - "operating_system":"Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "notes":"Automated by MLCommons CM v2.3.6. ", + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.4.1-default_config\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.4.1", + "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", + "notes":"Automated by MLCommons CM v2.3.4. ", "compliance":1, "errors":0, "version":"v4.1", "inferred":0, "has_power":false, "Units":"Samples\/s", - "weight_data_types":"int8" + "weight_data_types":"fp32" }, "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"datacenter", - "SystemName":"13fce262fb79", - "Platform":"13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base", + "SystemName":"48ed6105bd85", + "Platform":"48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base", "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.375843, - "Accuracy":"CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006", + "Result":0.373636, + "Accuracy":"CLIP_SCORE: 15.236237794160843 FID_SCORE: 238.78369342212613", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/13fce262fb79-reference-gpu-pytorch_v2.4.1-scc24-base\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.4.1", - "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":"Automated by MLCommons CM v2.3.9. ", + "Location":"open\/MLCommons\/results\/48ed6105bd85-reference-gpu-pytorch_v2.1.0a0-scc24-base\/stable-diffusion-xl\/offline", + "framework":"TensorRT", + "operating_system":"Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", + "notes":"Automated by MLCommons CM v2.3.6. ", "compliance":1, "errors":0, "version":"v4.1", @@ -214,12 +214,12 @@ "Availability":"available", "Division":"open", "SystemType":"datacenter", - "SystemName":"3b07702db56d", - "Platform":"3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base", + "SystemName":"f9ac88850adc", + "Platform":"f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base", "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.374549, + "Result":0.376944, "Accuracy":"CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", @@ -227,7 +227,7 @@ "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base\/stable-diffusion-xl\/offline", + "Location":"open\/MLCommons\/results\/f9ac88850adc-reference-gpu-pytorch_v2.4.1-scc24-base\/stable-diffusion-xl\/offline", "framework":"pytorch v2.4.1", "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", "notes":"Automated by MLCommons CM v2.3.9. ", @@ -244,22 +244,22 @@ "Availability":"available", "Division":"open", "SystemType":"datacenter", - "SystemName":"e8dbfdd7ca14", - "Platform":"e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base", + "SystemName":"3b07702db56d", + "Platform":"3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base", "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":1.13976, - "Accuracy":"CLIP_SCORE: 15.617164582014084 FID_SCORE: 233.28573786792805", + "Result":0.374549, + "Accuracy":"CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/e8dbfdd7ca14-nvidia-gpu-TensorRT-scc24-base\/stable-diffusion-xl\/offline", - "framework":"TensorRT", - "operating_system":"Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", + "Location":"open\/MLCommons\/results\/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.4.1", + "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", "notes":"Automated by MLCommons CM v2.3.9. ", "compliance":1, "errors":0, @@ -267,6 +267,6 @@ "inferred":0, "has_power":false, "Units":"Samples\/s", - "weight_data_types":"int8" + "weight_data_types":"fp32" } } \ No newline at end of file diff --git a/summary.xlsx b/summary.xlsx index 2dc43ea..610f5a7 100644 Binary files a/summary.xlsx and b/summary.xlsx differ