re-add GritLM-7B-noinstruct (#28)

* Standardize grit results * Fix name Co-authored-by: Kenneth Enevoldsen <[email protected]> * Fix name * Merge results --------- Co-authored-by: Kenneth Enevoldsen <[email protected]>
embeddings-benchmark · Sep 11, 2024 · 1d30609 · 1d30609
1 parent b7f3700
commit 1d30609
Show file tree

Hide file tree

Showing 22 changed files with 17,781 additions and 15,419 deletions.
diff --git a/paths.json b/paths.json
diff --git a/results.py b/results.py
@@ -54,7 +54,8 @@
     "Cohere-embed-multilingual-v3.0",
     "DanskBERT",
     "FollowIR-7B",
-    "GritLM-7B",
+    "GritLM__GritLM-7B",
+    "GritLM__GritLM-7B-noinstruct",
     "LASER2",
     "LLM2Vec-Llama-2-supervised",
     "LLM2Vec-Llama-2-unsupervised",

diff --git a/results/GritLM-7B/no_revision_available/Core17InstructionRetrieval.json b/results/GritLM-7B/no_revision_available/Core17InstructionRetrieval.json
diff --git a/results/GritLM-7B/no_revision_available/News21InstructionRetrieval.json b/results/GritLM-7B/no_revision_available/News21InstructionRetrieval.json
diff --git a/results/GritLM-7B/no_revision_available/Robust04InstructionRetrieval.json b/results/GritLM-7B/no_revision_available/Robust04InstructionRetrieval.json
diff --git a/...s/GritLM__GritLM-7B-noinstruct/13f00a0e36500c80ce12870ea513846a066004af/ARCChallenge.json b/...s/GritLM__GritLM-7B-noinstruct/13f00a0e36500c80ce12870ea513846a066004af/ARCChallenge.json
@@ -0,0 +1,158 @@
+{
+  "dataset_revision": "c481e0da3dcbbad8bce7721dea9085b74320a0a3",
+  "evaluation_time": 78.18728423118591,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.12.27",
+  "scores": {
+    "test": [
+      {
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "main_score": 0.16567,
+        "map_at_1": 0.08276,
+        "map_at_10": 0.1332,
+        "map_at_100": 0.14363,
+        "map_at_1000": 0.14482,
+        "map_at_20": 0.13866,
+        "map_at_3": 0.11277,
+        "map_at_5": 0.12433,
+        "mrr_at_1": 0.08276450511945392,
+        "mrr_at_10": 0.13319620510320165,
+        "mrr_at_100": 0.14362586972749813,
+        "mrr_at_1000": 0.1448178804221324,
+        "mrr_at_20": 0.1386578989402944,
+        "mrr_at_3": 0.11277019340159264,
+        "mrr_at_5": 0.12433162684869167,
+        "nauc_map_at_1000_diff1": 0.11606267607104377,
+        "nauc_map_at_1000_max": -0.10084652504177026,
+        "nauc_map_at_1000_std": -0.11828505538008827,
+        "nauc_map_at_100_diff1": 0.11617938969146996,
+        "nauc_map_at_100_max": -0.10103608095245299,
+        "nauc_map_at_100_std": -0.11836784322419454,
+        "nauc_map_at_10_diff1": 0.12171741485767183,
+        "nauc_map_at_10_max": -0.0951712196064573,
+        "nauc_map_at_10_std": -0.12040642685533308,
+        "nauc_map_at_1_diff1": 0.1637549340679917,
+        "nauc_map_at_1_max": -0.06866843294505832,
+        "nauc_map_at_1_std": -0.11623156565719857,
+        "nauc_map_at_20_diff1": 0.1189920257095774,
+        "nauc_map_at_20_max": -0.0991905913147805,
+        "nauc_map_at_20_std": -0.12012391309456927,
+        "nauc_map_at_3_diff1": 0.13195461509097473,
+        "nauc_map_at_3_max": -0.08656050010347069,
+        "nauc_map_at_3_std": -0.12814440057787524,
+        "nauc_map_at_5_diff1": 0.12381744141594893,
+        "nauc_map_at_5_max": -0.08188524850924572,
+        "nauc_map_at_5_std": -0.11540567655833621,
+        "nauc_mrr_at_1000_diff1": 0.11606267607104377,
+        "nauc_mrr_at_1000_max": -0.10084652504177026,
+        "nauc_mrr_at_1000_std": -0.11828505538008827,
+        "nauc_mrr_at_100_diff1": 0.11617938969146996,
+        "nauc_mrr_at_100_max": -0.10103608095245299,
+        "nauc_mrr_at_100_std": -0.11836784322419454,
+        "nauc_mrr_at_10_diff1": 0.12171741485767183,
+        "nauc_mrr_at_10_max": -0.0951712196064573,
+        "nauc_mrr_at_10_std": -0.12040642685533308,
+        "nauc_mrr_at_1_diff1": 0.1637549340679917,
+        "nauc_mrr_at_1_max": -0.06866843294505832,
+        "nauc_mrr_at_1_std": -0.11623156565719857,
+        "nauc_mrr_at_20_diff1": 0.1189920257095774,
+        "nauc_mrr_at_20_max": -0.0991905913147805,
+        "nauc_mrr_at_20_std": -0.12012391309456927,
+        "nauc_mrr_at_3_diff1": 0.13195461509097473,
+        "nauc_mrr_at_3_max": -0.08656050010347069,
+        "nauc_mrr_at_3_std": -0.12814440057787524,
+        "nauc_mrr_at_5_diff1": 0.12381744141594893,
+        "nauc_mrr_at_5_max": -0.08188524850924572,
+        "nauc_mrr_at_5_std": -0.11540567655833621,
+        "nauc_ndcg_at_1000_diff1": 0.07782564462113492,
+        "nauc_ndcg_at_1000_max": -0.1311850591294924,
+        "nauc_ndcg_at_1000_std": -0.1050073073458404,
+        "nauc_ndcg_at_100_diff1": 0.08039808605161892,
+        "nauc_ndcg_at_100_max": -0.130185279752856,
+        "nauc_ndcg_at_100_std": -0.10462409602995773,
+        "nauc_ndcg_at_10_diff1": 0.10518245547538639,
+        "nauc_ndcg_at_10_max": -0.11041301107009849,
+        "nauc_ndcg_at_10_std": -0.11849884556559163,
+        "nauc_ndcg_at_1_diff1": 0.1637549340679917,
+        "nauc_ndcg_at_1_max": -0.06866843294505832,
+        "nauc_ndcg_at_1_std": -0.11623156565719857,
+        "nauc_ndcg_at_20_diff1": 0.09856770854187791,
+        "nauc_ndcg_at_20_max": -0.12213906580274646,
+        "nauc_ndcg_at_20_std": -0.11819865442488073,
+        "nauc_ndcg_at_3_diff1": 0.12119705302223491,
+        "nauc_ndcg_at_3_max": -0.09227415998343752,
+        "nauc_ndcg_at_3_std": -0.12911306275481269,
+        "nauc_ndcg_at_5_diff1": 0.10981759290664823,
+        "nauc_ndcg_at_5_max": -0.08336011154639276,
+        "nauc_ndcg_at_5_std": -0.10750180809854237,
+        "nauc_precision_at_1000_diff1": -0.1695783827238421,
+        "nauc_precision_at_1000_max": -0.3245680277313244,
+        "nauc_precision_at_1000_std": -0.012669573395770703,
+        "nauc_precision_at_100_diff1": -0.013170183794179467,
+        "nauc_precision_at_100_max": -0.2004149608472424,
+        "nauc_precision_at_100_std": -0.05897637081303174,
+        "nauc_precision_at_10_diff1": 0.07275127233245807,
+        "nauc_precision_at_10_max": -0.14279613765432786,
+        "nauc_precision_at_10_std": -0.11534315061842612,
+        "nauc_precision_at_1_diff1": 0.1637549340679917,
+        "nauc_precision_at_1_max": -0.06866843294505832,
+        "nauc_precision_at_1_std": -0.11623156565719857,
+        "nauc_precision_at_20_diff1": 0.05928787026612005,
+        "nauc_precision_at_20_max": -0.17129862954688976,
+        "nauc_precision_at_20_std": -0.11542140579282543,
+        "nauc_precision_at_3_diff1": 0.09613544495372242,
+        "nauc_precision_at_3_max": -0.10552411799048211,
+        "nauc_precision_at_3_std": -0.13081876562122427,
+        "nauc_precision_at_5_diff1": 0.08113003762777546,
+        "nauc_precision_at_5_max": -0.08527591607021416,
+        "nauc_precision_at_5_std": -0.08915421346354885,
+        "nauc_recall_at_1000_diff1": -0.16957838272384387,
+        "nauc_recall_at_1000_max": -0.3245680277313267,
+        "nauc_recall_at_1000_std": -0.012669573395772498,
+        "nauc_recall_at_100_diff1": -0.013170183794179738,
+        "nauc_recall_at_100_max": -0.20041496084724292,
+        "nauc_recall_at_100_std": -0.05897637081303231,
+        "nauc_recall_at_10_diff1": 0.0727512723324577,
+        "nauc_recall_at_10_max": -0.1427961376543279,
+        "nauc_recall_at_10_std": -0.11534315061842598,
+        "nauc_recall_at_1_diff1": 0.1637549340679917,
+        "nauc_recall_at_1_max": -0.06866843294505832,
+        "nauc_recall_at_1_std": -0.11623156565719857,
+        "nauc_recall_at_20_diff1": 0.059287870266119605,
+        "nauc_recall_at_20_max": -0.17129862954689018,
+        "nauc_recall_at_20_std": -0.11542140579282577,
+        "nauc_recall_at_3_diff1": 0.09613544495372225,
+        "nauc_recall_at_3_max": -0.10552411799048216,
+        "nauc_recall_at_3_std": -0.13081876562122424,
+        "nauc_recall_at_5_diff1": 0.08113003762777533,
+        "nauc_recall_at_5_max": -0.08527591607021426,
+        "nauc_recall_at_5_std": -0.08915421346354895,
+        "ndcg_at_1": 0.08276,
+        "ndcg_at_10": 0.16567,
+        "ndcg_at_100": 0.2226,
+        "ndcg_at_1000": 0.25982,
+        "ndcg_at_20": 0.18539,
+        "ndcg_at_3": 0.1232,
+        "ndcg_at_5": 0.14417,
+        "precision_at_1": 0.08276,
+        "precision_at_10": 0.02713,
+        "precision_at_100": 0.00552,
+        "precision_at_1000": 0.00086,
+        "precision_at_20": 0.01745,
+        "precision_at_3": 0.05119,
+        "precision_at_5": 0.04096,
+        "recall_at_1": 0.08276,
+        "recall_at_10": 0.27133,
+        "recall_at_100": 0.55205,
+        "recall_at_1000": 0.85666,
+        "recall_at_20": 0.34898,
+        "recall_at_3": 0.15358,
+        "recall_at_5": 0.20478
+      }
+    ]
+  },
+  "task_name": "ARCChallenge"
+}
diff --git a/results/GritLM__GritLM-7B-noinstruct/13f00a0e36500c80ce12870ea513846a066004af/AlphaNLI.json b/results/GritLM__GritLM-7B-noinstruct/13f00a0e36500c80ce12870ea513846a066004af/AlphaNLI.json
@@ -0,0 +1,158 @@
+{
+  "dataset_revision": "303f40ef3d50918d3dc43577d33f2f7344ad72c1",
+  "evaluation_time": 1270.652179479599,
+  "kg_co2_emissions": null,
+  "mteb_version": "1.12.27",
+  "scores": {
+    "test": [
+      {
+        "hf_subset": "default",
+        "languages": [
+          "eng-Latn"
+        ],
+        "main_score": 0.29556,
+        "map_at_1": 0.19582,
+        "map_at_10": 0.26009,
+        "map_at_100": 0.26884,
+        "map_at_1000": 0.26961,
+        "map_at_20": 0.26455,
+        "map_at_3": 0.24086,
+        "map_at_5": 0.25173,
+        "mrr_at_1": 0.195822454308094,
+        "mrr_at_10": 0.2600939222512328,
+        "mrr_at_100": 0.26883610136872893,
+        "mrr_at_1000": 0.26961346115632173,
+        "mrr_at_20": 0.2645490625655516,
+        "mrr_at_3": 0.24086161879895546,
+        "mrr_at_5": 0.2517297650130545,
+        "nauc_map_at_1000_diff1": 0.3581262993761119,
+        "nauc_map_at_1000_max": 0.18737882517084573,
+        "nauc_map_at_1000_std": -0.06452508289227181,
+        "nauc_map_at_100_diff1": 0.3580396187920006,
+        "nauc_map_at_100_max": 0.18715247309578215,
+        "nauc_map_at_100_std": -0.06467289761940416,
+        "nauc_map_at_10_diff1": 0.3602649101950382,
+        "nauc_map_at_10_max": 0.19084692411391246,
+        "nauc_map_at_10_std": -0.06424330205810602,
+        "nauc_map_at_1_diff1": 0.41684305133357324,
+        "nauc_map_at_1_max": 0.23999061803519373,
+        "nauc_map_at_1_std": -0.03497237739685838,
+        "nauc_map_at_20_diff1": 0.35923735124817535,
+        "nauc_map_at_20_max": 0.18692626854610142,
+        "nauc_map_at_20_std": -0.06671937446108235,
+        "nauc_map_at_3_diff1": 0.37685381270276136,
+        "nauc_map_at_3_max": 0.20715287494533172,
+        "nauc_map_at_3_std": -0.05590743263735323,
+        "nauc_map_at_5_diff1": 0.3654358413863913,
+        "nauc_map_at_5_max": 0.19607730137035934,
+        "nauc_map_at_5_std": -0.06337248072588127,
+        "nauc_mrr_at_1000_diff1": 0.3581262993761119,
+        "nauc_mrr_at_1000_max": 0.18737882517084573,
+        "nauc_mrr_at_1000_std": -0.06452508289227181,
+        "nauc_mrr_at_100_diff1": 0.3580396187920006,
+        "nauc_mrr_at_100_max": 0.18715247309578215,
+        "nauc_mrr_at_100_std": -0.06467289761940416,
+        "nauc_mrr_at_10_diff1": 0.3602649101950382,
+        "nauc_mrr_at_10_max": 0.19084692411391246,
+        "nauc_mrr_at_10_std": -0.06424330205810602,
+        "nauc_mrr_at_1_diff1": 0.41684305133357324,
+        "nauc_mrr_at_1_max": 0.23999061803519373,
+        "nauc_mrr_at_1_std": -0.03497237739685838,
+        "nauc_mrr_at_20_diff1": 0.35923735124817535,
+        "nauc_mrr_at_20_max": 0.18692626854610142,
+        "nauc_mrr_at_20_std": -0.06671937446108235,
+        "nauc_mrr_at_3_diff1": 0.37685381270276136,
+        "nauc_mrr_at_3_max": 0.20715287494533172,
+        "nauc_mrr_at_3_std": -0.05590743263735323,
+        "nauc_mrr_at_5_diff1": 0.3654358413863913,
+        "nauc_mrr_at_5_max": 0.19607730137035934,
+        "nauc_mrr_at_5_std": -0.06337248072588127,
+        "nauc_ndcg_at_1000_diff1": 0.3273834438022227,
+        "nauc_ndcg_at_1000_max": 0.1622409096338372,
+        "nauc_ndcg_at_1000_std": -0.0669388102417427,
+        "nauc_ndcg_at_100_diff1": 0.3223003368537524,
+        "nauc_ndcg_at_100_max": 0.1547146909189916,
+        "nauc_ndcg_at_100_std": -0.07012998864619853,
+        "nauc_ndcg_at_10_diff1": 0.33558897971035817,
+        "nauc_ndcg_at_10_max": 0.16674222031452857,
+        "nauc_ndcg_at_10_std": -0.07599286983639812,
+        "nauc_ndcg_at_1_diff1": 0.41684305133357324,
+        "nauc_ndcg_at_1_max": 0.23999061803519373,
+        "nauc_ndcg_at_1_std": -0.03497237739685838,
+        "nauc_ndcg_at_20_diff1": 0.3321776819933026,
+        "nauc_ndcg_at_20_max": 0.1526275128682169,
+        "nauc_ndcg_at_20_std": -0.08555865426380696,
+        "nauc_ndcg_at_3_diff1": 0.36622693404119916,
+        "nauc_ndcg_at_3_max": 0.1979519921767349,
+        "nauc_ndcg_at_3_std": -0.06086233490189131,
+        "nauc_ndcg_at_5_diff1": 0.346250107952567,
+        "nauc_ndcg_at_5_max": 0.17847240288665858,
+        "nauc_ndcg_at_5_std": -0.07384965773020805,
+        "nauc_precision_at_1000_diff1": 0.15722783022505635,
+        "nauc_precision_at_1000_max": 0.062389360128002475,
+        "nauc_precision_at_1000_std": -0.020194075968903987,
+        "nauc_precision_at_100_diff1": 0.17926769479182755,
+        "nauc_precision_at_100_max": 0.03764458108362797,
+        "nauc_precision_at_100_std": -0.06516684759990607,
+        "nauc_precision_at_10_diff1": 0.26861850851333074,
+        "nauc_precision_at_10_max": 0.09981912991436129,
+        "nauc_precision_at_10_std": -0.10775280987881723,
+        "nauc_precision_at_1_diff1": 0.41684305133357324,
+        "nauc_precision_at_1_max": 0.23999061803519373,
+        "nauc_precision_at_1_std": -0.03497237739685838,
+        "nauc_precision_at_20_diff1": 0.25543225738691927,
+        "nauc_precision_at_20_max": 0.04693182684250898,
+        "nauc_precision_at_20_std": -0.1445516047213592,
+        "nauc_precision_at_3_diff1": 0.33871394447027126,
+        "nauc_precision_at_3_max": 0.17398390212102777,
+        "nauc_precision_at_3_std": -0.07350206090918988,
+        "nauc_precision_at_5_diff1": 0.2946779502612536,
+        "nauc_precision_at_5_max": 0.1307848039211153,
+        "nauc_precision_at_5_std": -0.10206440369947867,
+        "nauc_recall_at_1000_diff1": 0.15722783022505793,
+        "nauc_recall_at_1000_max": 0.0623893601280026,
+        "nauc_recall_at_1000_std": -0.020194075968902568,
+        "nauc_recall_at_100_diff1": 0.17926769479182714,
+        "nauc_recall_at_100_max": 0.03764458108362699,
+        "nauc_recall_at_100_std": -0.06516684759990606,
+        "nauc_recall_at_10_diff1": 0.2686185085133306,
+        "nauc_recall_at_10_max": 0.09981912991436125,
+        "nauc_recall_at_10_std": -0.10775280987881715,
+        "nauc_recall_at_1_diff1": 0.41684305133357324,
+        "nauc_recall_at_1_max": 0.23999061803519373,
+        "nauc_recall_at_1_std": -0.03497237739685838,
+        "nauc_recall_at_20_diff1": 0.2554322573869195,
+        "nauc_recall_at_20_max": 0.046931826842508566,
+        "nauc_recall_at_20_std": -0.14455160472135906,
+        "nauc_recall_at_3_diff1": 0.3387139444702712,
+        "nauc_recall_at_3_max": 0.17398390212102802,
+        "nauc_recall_at_3_std": -0.07350206090918995,
+        "nauc_recall_at_5_diff1": 0.2946779502612537,
+        "nauc_recall_at_5_max": 0.13078480392111536,
+        "nauc_recall_at_5_std": -0.10206440369947854,
+        "ndcg_at_1": 0.19582,
+        "ndcg_at_10": 0.29556,
+        "ndcg_at_100": 0.34419,
+        "ndcg_at_1000": 0.36667,
+        "ndcg_at_20": 0.31208,
+        "ndcg_at_3": 0.25545,
+        "ndcg_at_5": 0.27506,
+        "precision_at_1": 0.19582,
+        "precision_at_10": 0.04093,
+        "precision_at_100": 0.00651,
+        "precision_at_1000": 0.00083,
+        "precision_at_20": 0.02376,
+        "precision_at_3": 0.09922,
+        "precision_at_5": 0.06906,
+        "recall_at_1": 0.19582,
+        "recall_at_10": 0.40927,
+        "recall_at_100": 0.65078,
+        "recall_at_1000": 0.83159,
+        "recall_at_20": 0.4752,
+        "recall_at_3": 0.29765,
+        "recall_at_5": 0.3453
+      }
+    ]
+  },
+  "task_name": "AlphaNLI"
+}