Add MMLU leaderboard reproduction instructions to documentation (#2587)

stanford-crfm · Apr 24, 2024 · 5f6be0b · 5f6be0b
1 parent 818b3d4
commit 5f6be0b
Showing 1 changed file with 13 additions and 3 deletions.
diff --git a/docs/reproducing_leaderboards.md b/docs/reproducing_leaderboards.md
@@ -47,7 +47,7 @@ export PRIORITY=2
 ### Classic before v0.2.4
 
 ```bash
-export RUN_ENTRIES_CONF_PATH=run_specs.conf
+export RUN_ENTRIES_CONF_PATH=run_entries.conf
 export SCHEMA_PATH=schema_classic.yaml
 export NUM_TRAIN_TRIALS=3
 export MAX_EVAL_INSTANCES=1000
@@ -57,7 +57,7 @@ export PRIORITY=2
 ### Classic v0.2.4 and after
 
 ```bash
-export RUN_ENTRIES_CONF_PATH=run_specs_lite.conf
+export RUN_ENTRIES_CONF_PATH=run_entries_lite.conf
 export SCHEMA_PATH=schema_classic.yaml
 export NUM_TRAIN_TRIALS=1
 export MAX_EVAL_INSTANCES=1000
@@ -67,9 +67,19 @@ export PRIORITY=2
 ### HEIM
 
 ```bash
-export RUN_ENTRIES_CONF_PATH=run_specs_heim.conf
+export RUN_ENTRIES_CONF_PATH=run_entries_heim.conf
 export SCHEMA_PATH=schema_heim.yaml
 export NUM_TRAIN_TRIALS=1
 export NUM_EVAL_INSTANCES=1000
 export PRIORITY=2
 ```
+
+### MMLU
+
+```bash
+export RUN_ENTRIES_CONF_PATH=run_entries_mmlu.conf
+export SCHEMA_PATH=schema_mmlu.yaml
+export NUM_TRAIN_TRIALS=1
+export NUM_EVAL_INSTANCES=10000
+export PRIORITY=4
+```