From b7f3700c5a62a4546b8045b9e70e882f3c70c48d Mon Sep 17 00:00:00 2001 From: Roman Solomatin Date: Wed, 11 Sep 2024 11:36:37 +0300 Subject: [PATCH] add micracle results for russian models (#29) * add micracle results * upd paths * upd paths.json --- paths.json | 16 ++ .../MIRACLReranking.json | 130 ++++++++++++++ .../MIRACLRetrieval.json | 158 ++++++++++++++++++ .../MIRACLReranking.json | 130 ++++++++++++++ .../MIRACLRetrieval.json | 158 ++++++++++++++++++ .../MIRACLReranking.json | 130 ++++++++++++++ .../MIRACLRetrieval.json | 158 ++++++++++++++++++ .../MIRACLReranking.json | 130 ++++++++++++++ .../MIRACLRetrieval.json | 158 ++++++++++++++++++ .../MIRACLReranking.json | 130 ++++++++++++++ .../MIRACLRetrieval.json | 158 ++++++++++++++++++ .../MIRACLReranking.json | 130 ++++++++++++++ .../MIRACLRetrieval.json | 158 ++++++++++++++++++ .../MIRACLReranking.json | 130 ++++++++++++++ .../MIRACLRetrieval.json | 158 ++++++++++++++++++ .../MIRACLReranking.json | 130 ++++++++++++++ .../MIRACLRetrieval.json | 158 ++++++++++++++++++ 17 files changed, 2320 insertions(+) create mode 100644 results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLReranking.json create mode 100644 results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLRetrieval.json create mode 100644 results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLReranking.json create mode 100644 results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLRetrieval.json create mode 100644 results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLReranking.json create mode 100644 results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLRetrieval.json create mode 100644 results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLReranking.json create mode 100644 results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLRetrieval.json create mode 100644 results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLReranking.json create mode 100644 results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLRetrieval.json create mode 100644 results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLReranking.json create mode 100644 results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLRetrieval.json create mode 100644 results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLReranking.json create mode 100644 results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLRetrieval.json create mode 100644 results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLReranking.json create mode 100644 results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLRetrieval.json diff --git a/paths.json b/paths.json index 1dfb90713..731625e31 100644 --- a/paths.json +++ b/paths.json @@ -1246,6 +1246,7 @@ "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MedrxivClusteringS2S.json", "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/Diversity5LegalBenchClassification.json", "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/RiaNewsRetrieval.json", + "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLRetrieval.json", "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/YahooAnswersTopicsClassification.json", "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/ContractNLIPermissiblePostAgreementPossessionLegalBenchClassification.json", "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/OPP115PolicyChangeLegalBenchClassification.json", @@ -1327,6 +1328,7 @@ "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/LearnedHandsCourtsLegalBenchClassification.json", "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/TelemarketingSalesRuleLegalBenchClassification.json", "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/NYSJudicialEthicsLegalBenchClassification.json", + "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLReranking.json", "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/XNLI.json", "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/ContractNLIPermissibleCopyLegalBenchClassification.json", "results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/SCDDTrainingLegalBenchClassification.json", @@ -3574,6 +3576,7 @@ "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/RuSTSBenchmarkSTS.json", "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/CyrillicTurkicLangClassification.json", "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/RiaNewsRetrieval.json", + "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLRetrieval.json", "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/GeoreviewClassification.json", "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/SensitiveTopicsClassification.json", "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/KinopoiskClassification.json", @@ -3589,6 +3592,7 @@ "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/HeadlineClassification.json", "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/XQuADRetrieval.json", "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/BUCC.v2.json", + "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLReranking.json", "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/XNLI.json", "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/XNLIV2.json", "results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/STS22.json", @@ -9928,6 +9932,7 @@ "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/RuSTSBenchmarkSTS.json", "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/CyrillicTurkicLangClassification.json", "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/RiaNewsRetrieval.json", + "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLRetrieval.json", "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/GeoreviewClassification.json", "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/SensitiveTopicsClassification.json", "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/KinopoiskClassification.json", @@ -9942,6 +9947,7 @@ "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/HeadlineClassification.json", "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/XQuADRetrieval.json", "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/BUCC.v2.json", + "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLReranking.json", "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/XNLI.json", "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/XNLIV2.json", "results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/STS22.json", @@ -11951,6 +11957,7 @@ "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/RuSTSBenchmarkSTS.json", "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/CyrillicTurkicLangClassification.json", "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/RiaNewsRetrieval.json", + "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLRetrieval.json", "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/GeoreviewClassification.json", "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/SensitiveTopicsClassification.json", "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/KinopoiskClassification.json", @@ -11965,6 +11972,7 @@ "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/HeadlineClassification.json", "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/XQuADRetrieval.json", "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/BUCC.v2.json", + "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLReranking.json", "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/XNLI.json", "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/XNLIV2.json", "results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/STS22.json", @@ -12477,6 +12485,7 @@ "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/RuSTSBenchmarkSTS.json", "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/CyrillicTurkicLangClassification.json", "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/RiaNewsRetrieval.json", + "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLRetrieval.json", "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/GeoreviewClassification.json", "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/SensitiveTopicsClassification.json", "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/KinopoiskClassification.json", @@ -12492,6 +12501,7 @@ "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/HeadlineClassification.json", "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/XQuADRetrieval.json", "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/BUCC.v2.json", + "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLReranking.json", "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/XNLI.json", "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/XNLIV2.json", "results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/STS22.json", @@ -13662,6 +13672,7 @@ "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/RuSTSBenchmarkSTS.json", "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/CyrillicTurkicLangClassification.json", "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/RiaNewsRetrieval.json", + "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLRetrieval.json", "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/GeoreviewClassification.json", "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/SensitiveTopicsClassification.json", "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/KinopoiskClassification.json", @@ -13676,6 +13687,7 @@ "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/HeadlineClassification.json", "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/XQuADRetrieval.json", "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/BUCC.v2.json", + "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLReranking.json", "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/XNLI.json", "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/XNLIV2.json", "results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/STS22.json", @@ -13708,6 +13720,7 @@ "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/RuSTSBenchmarkSTS.json", "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/CyrillicTurkicLangClassification.json", "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/RiaNewsRetrieval.json", + "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLRetrieval.json", "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/GeoreviewClassification.json", "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/SensitiveTopicsClassification.json", "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/KinopoiskClassification.json", @@ -13723,6 +13736,7 @@ "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/HeadlineClassification.json", "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/XQuADRetrieval.json", "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/BUCC.v2.json", + "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLReranking.json", "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/XNLI.json", "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/XNLIV2.json", "results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/STS22.json", @@ -15092,6 +15106,7 @@ "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RuSTSBenchmarkSTS.json", "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/CyrillicTurkicLangClassification.json", "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/RiaNewsRetrieval.json", + "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLRetrieval.json", "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/GeoreviewClassification.json", "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/SensitiveTopicsClassification.json", "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/KinopoiskClassification.json", @@ -15106,6 +15121,7 @@ "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/HeadlineClassification.json", "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/XQuADRetrieval.json", "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/BUCC.v2.json", + "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLReranking.json", "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/XNLI.json", "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/XNLIV2.json", "results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/STS22.json", diff --git a/results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLReranking.json b/results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLReranking.json new file mode 100644 index 000000000..57bc9c40b --- /dev/null +++ b/results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 102.19886922836304, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.03793, + "MAP@10(MIRACL)": 0.08492, + "MAP@100(MIRACL)": 0.11854, + "MAP@1000(MIRACL)": 0.11854, + "MAP@20(MIRACL)": 0.09699, + "MAP@3(MIRACL)": 0.0599, + "MAP@5(MIRACL)": 0.07087, + "NDCG@1(MIRACL)": 0.06576, + "NDCG@10(MIRACL)": 0.13094, + "NDCG@100(MIRACL)": 0.27866, + "NDCG@1000(MIRACL)": 0.27866, + "NDCG@20(MIRACL)": 0.16843, + "NDCG@3(MIRACL)": 0.08274, + "NDCG@5(MIRACL)": 0.09898, + "P@1(MIRACL)": 0.06576, + "P@10(MIRACL)": 0.04707, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.03773, + "P@3(MIRACL)": 0.05801, + "P@5(MIRACL)": 0.05277, + "Recall@1(MIRACL)": 0.03793, + "Recall@10(MIRACL)": 0.20941, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.32753, + "Recall@3(MIRACL)": 0.08533, + "Recall@5(MIRACL)": 0.12619, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.13094, + "nAUC_MAP@1000_diff1(MIRACL)": 0.11656879622209783, + "nAUC_MAP@1000_max(MIRACL)": -0.02184173429173281, + "nAUC_MAP@1000_std(MIRACL)": -0.0005910451991668005, + "nAUC_MAP@100_diff1(MIRACL)": 0.11656879622209783, + "nAUC_MAP@100_max(MIRACL)": -0.02184173429173281, + "nAUC_MAP@100_std(MIRACL)": -0.0005910451991668005, + "nAUC_MAP@10_diff1(MIRACL)": 0.12820268321122477, + "nAUC_MAP@10_max(MIRACL)": -0.006007170516302484, + "nAUC_MAP@10_std(MIRACL)": 0.004887023887105919, + "nAUC_MAP@1_diff1(MIRACL)": 0.19979024924352953, + "nAUC_MAP@1_max(MIRACL)": 0.0965067328936865, + "nAUC_MAP@1_std(MIRACL)": -0.01469295152548763, + "nAUC_MAP@20_diff1(MIRACL)": 0.11943138095364866, + "nAUC_MAP@20_max(MIRACL)": -0.014742898115901397, + "nAUC_MAP@20_std(MIRACL)": 0.007627922730200203, + "nAUC_MAP@3_diff1(MIRACL)": 0.16247267806292065, + "nAUC_MAP@3_max(MIRACL)": 0.01916771582088975, + "nAUC_MAP@3_std(MIRACL)": -0.024768207978978725, + "nAUC_MAP@5_diff1(MIRACL)": 0.13153704499454139, + "nAUC_MAP@5_max(MIRACL)": 0.007096539434435867, + "nAUC_MAP@5_std(MIRACL)": -0.013659018365901245, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.09046593862073116, + "nAUC_NDCG@1000_max(MIRACL)": -0.049656945308565785, + "nAUC_NDCG@1000_std(MIRACL)": -0.02852729085649775, + "nAUC_NDCG@100_diff1(MIRACL)": 0.09046593862073116, + "nAUC_NDCG@100_max(MIRACL)": -0.049656945308565785, + "nAUC_NDCG@100_std(MIRACL)": -0.02852729085649775, + "nAUC_NDCG@10_diff1(MIRACL)": 0.10077472820623631, + "nAUC_NDCG@10_max(MIRACL)": -0.033319860687037434, + "nAUC_NDCG@10_std(MIRACL)": 0.008223662807163275, + "nAUC_NDCG@1_diff1(MIRACL)": 0.151186819507912, + "nAUC_NDCG@1_max(MIRACL)": 0.11537026630871994, + "nAUC_NDCG@1_std(MIRACL)": -0.045578082017163145, + "nAUC_NDCG@20_diff1(MIRACL)": 0.08640831332839452, + "nAUC_NDCG@20_max(MIRACL)": -0.05269863130012071, + "nAUC_NDCG@20_std(MIRACL)": 0.007783868824801839, + "nAUC_NDCG@3_diff1(MIRACL)": 0.13178318008309772, + "nAUC_NDCG@3_max(MIRACL)": 0.0035297506848497, + "nAUC_NDCG@3_std(MIRACL)": -0.042422835970525714, + "nAUC_NDCG@5_diff1(MIRACL)": 0.10348566598107344, + "nAUC_NDCG@5_max(MIRACL)": -0.015154481875360658, + "nAUC_NDCG@5_std(MIRACL)": -0.02018522684348813, + "nAUC_P@1000_diff1(MIRACL)": 0.006436408505254566, + "nAUC_P@1000_max(MIRACL)": -0.062365784452251265, + "nAUC_P@1000_std(MIRACL)": -0.06679081786133326, + "nAUC_P@100_diff1(MIRACL)": 0.006436408505254932, + "nAUC_P@100_max(MIRACL)": -0.06236578445225068, + "nAUC_P@100_std(MIRACL)": -0.06679081786133294, + "nAUC_P@10_diff1(MIRACL)": 0.035924513017525805, + "nAUC_P@10_max(MIRACL)": -0.07407327390595522, + "nAUC_P@10_std(MIRACL)": 0.011824346308663887, + "nAUC_P@1_diff1(MIRACL)": 0.151186819507912, + "nAUC_P@1_max(MIRACL)": 0.11537026630871994, + "nAUC_P@1_std(MIRACL)": -0.045578082017163145, + "nAUC_P@20_diff1(MIRACL)": 0.020405546456346138, + "nAUC_P@20_max(MIRACL)": -0.09600408910657676, + "nAUC_P@20_std(MIRACL)": -0.0001021731695539199, + "nAUC_P@3_diff1(MIRACL)": 0.08187396234458519, + "nAUC_P@3_max(MIRACL)": -0.02981991700564444, + "nAUC_P@3_std(MIRACL)": -0.04315477063398759, + "nAUC_P@5_diff1(MIRACL)": 0.029326475776068565, + "nAUC_P@5_max(MIRACL)": -0.05538591644280579, + "nAUC_P@5_std(MIRACL)": -0.013627413052371293, + "nAUC_Recall@1000_diff1(MIRACL)": 0.0626568052254435, + "nAUC_Recall@1000_max(MIRACL)": -0.056351555553082765, + "nAUC_Recall@1000_std(MIRACL)": -0.07720527712702241, + "nAUC_Recall@100_diff1(MIRACL)": 0.0626568052254435, + "nAUC_Recall@100_max(MIRACL)": -0.056351555553082765, + "nAUC_Recall@100_std(MIRACL)": -0.07720527712702241, + "nAUC_Recall@10_diff1(MIRACL)": 0.0699596207061801, + "nAUC_Recall@10_max(MIRACL)": -0.06623123931077808, + "nAUC_Recall@10_std(MIRACL)": 0.04314390563870134, + "nAUC_Recall@1_diff1(MIRACL)": 0.19979024924352953, + "nAUC_Recall@1_max(MIRACL)": 0.0965067328936865, + "nAUC_Recall@1_std(MIRACL)": -0.01469295152548763, + "nAUC_Recall@20_diff1(MIRACL)": 0.04151985357457182, + "nAUC_Recall@20_max(MIRACL)": -0.09346206581673573, + "nAUC_Recall@20_std(MIRACL)": 0.03867752916801929, + "nAUC_Recall@3_diff1(MIRACL)": 0.15002227895926992, + "nAUC_Recall@3_max(MIRACL)": -0.03085604524638985, + "nAUC_Recall@3_std(MIRACL)": -0.04078077585553242, + "nAUC_Recall@5_diff1(MIRACL)": 0.08035497411159716, + "nAUC_Recall@5_max(MIRACL)": -0.053504090706605265, + "nAUC_Recall@5_std(MIRACL)": -0.009017940713406399 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLRetrieval.json b/results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLRetrieval.json new file mode 100644 index 000000000..395644fa3 --- /dev/null +++ b/results/DeepPavlov__distilrubert-small-cased-conversational/e348066b4a7279b97138038299bddc6580a9169a/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 4192.884420633316, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.0239, + "map_at_1": 0.01134, + "map_at_10": 0.01714, + "map_at_100": 0.01947, + "map_at_1000": 0.02009, + "map_at_20": 0.01806, + "map_at_3": 0.01464, + "map_at_5": 0.01594, + "mrr_at_1": 0.020766773162939296, + "mrr_at_10": 0.03221385465794412, + "mrr_at_100": 0.03591213689721048, + "mrr_at_1000": 0.03657304592716092, + "mrr_at_20": 0.034060104900166394, + "mrr_at_3": 0.027289669861554844, + "mrr_at_5": 0.03068423855165069, + "nauc_map_at_1000_diff1": 0.3124150957686266, + "nauc_map_at_1000_max": 0.23886121824102766, + "nauc_map_at_1000_std": 0.12257259031067662, + "nauc_map_at_100_diff1": 0.3156167174366099, + "nauc_map_at_100_max": 0.24520207347402345, + "nauc_map_at_100_std": 0.1167261093523615, + "nauc_map_at_10_diff1": 0.3558642162336333, + "nauc_map_at_10_max": 0.2863654286756792, + "nauc_map_at_10_std": 0.08606598881857085, + "nauc_map_at_1_diff1": 0.4732459963619434, + "nauc_map_at_1_max": 0.40147711944185954, + "nauc_map_at_1_std": 0.11598894600228143, + "nauc_map_at_20_diff1": 0.3394270943255352, + "nauc_map_at_20_max": 0.2696914957826567, + "nauc_map_at_20_std": 0.09836975881577859, + "nauc_map_at_3_diff1": 0.40877470709927316, + "nauc_map_at_3_max": 0.34888690030812164, + "nauc_map_at_3_std": 0.07145056501985438, + "nauc_map_at_5_diff1": 0.37405622869664157, + "nauc_map_at_5_max": 0.3148159658216705, + "nauc_map_at_5_std": 0.0699650218224813, + "nauc_mrr_at_1000_diff1": 0.29925968156542787, + "nauc_mrr_at_1000_max": 0.2070507917135228, + "nauc_mrr_at_1000_std": 0.1392745874098323, + "nauc_mrr_at_100_diff1": 0.300597481511976, + "nauc_mrr_at_100_max": 0.21007219978742486, + "nauc_mrr_at_100_std": 0.13709904049595376, + "nauc_mrr_at_10_diff1": 0.3347099588177761, + "nauc_mrr_at_10_max": 0.2486856720027666, + "nauc_mrr_at_10_std": 0.11525158040262058, + "nauc_mrr_at_1_diff1": 0.45392702701399307, + "nauc_mrr_at_1_max": 0.38930711913810623, + "nauc_mrr_at_1_std": 0.09927959002214515, + "nauc_mrr_at_20_diff1": 0.31618016469719096, + "nauc_mrr_at_20_max": 0.22491693681778774, + "nauc_mrr_at_20_std": 0.1277868226799658, + "nauc_mrr_at_3_diff1": 0.3630225224521912, + "nauc_mrr_at_3_max": 0.31351264132723305, + "nauc_mrr_at_3_std": 0.07889033518603211, + "nauc_mrr_at_5_diff1": 0.336012591725747, + "nauc_mrr_at_5_max": 0.2663127817716159, + "nauc_mrr_at_5_std": 0.0910506407825106, + "nauc_ndcg_at_1000_diff1": 0.1866264652986054, + "nauc_ndcg_at_1000_max": 0.060306158246926714, + "nauc_ndcg_at_1000_std": 0.23801913665442243, + "nauc_ndcg_at_100_diff1": 0.19984286301445142, + "nauc_ndcg_at_100_max": 0.10508475598896722, + "nauc_ndcg_at_100_std": 0.20359754446171371, + "nauc_ndcg_at_10_diff1": 0.3207865769578437, + "nauc_ndcg_at_10_max": 0.22453440599903504, + "nauc_ndcg_at_10_std": 0.11363674831429697, + "nauc_ndcg_at_1_diff1": 0.45392702701399307, + "nauc_ndcg_at_1_max": 0.38930711913810623, + "nauc_ndcg_at_1_std": 0.09927959002214515, + "nauc_ndcg_at_20_diff1": 0.280772540160207, + "nauc_ndcg_at_20_max": 0.18262917790500266, + "nauc_ndcg_at_20_std": 0.14515012476065897, + "nauc_ndcg_at_3_diff1": 0.35746510038297064, + "nauc_ndcg_at_3_max": 0.3204934865665906, + "nauc_ndcg_at_3_std": 0.06783561934328586, + "nauc_ndcg_at_5_diff1": 0.3229476923433982, + "nauc_ndcg_at_5_max": 0.26187556113661187, + "nauc_ndcg_at_5_std": 0.07567176288675988, + "nauc_precision_at_1000_diff1": 0.10103406339272379, + "nauc_precision_at_1000_max": -0.08190770848382328, + "nauc_precision_at_1000_std": 0.3464876453067858, + "nauc_precision_at_100_diff1": 0.0815681401650689, + "nauc_precision_at_100_max": -0.04975656550672437, + "nauc_precision_at_100_std": 0.3132017952622958, + "nauc_precision_at_10_diff1": 0.2176912712875044, + "nauc_precision_at_10_max": 0.121060780988917, + "nauc_precision_at_10_std": 0.1612087103093556, + "nauc_precision_at_1_diff1": 0.45392702701399307, + "nauc_precision_at_1_max": 0.38930711913810623, + "nauc_precision_at_1_std": 0.09927959002214515, + "nauc_precision_at_20_diff1": 0.17250528853834854, + "nauc_precision_at_20_max": 0.04702865063984905, + "nauc_precision_at_20_std": 0.21671884167472752, + "nauc_precision_at_3_diff1": 0.30547880581372333, + "nauc_precision_at_3_max": 0.28883169339921894, + "nauc_precision_at_3_std": 0.06523362871977251, + "nauc_precision_at_5_diff1": 0.24306006898153978, + "nauc_precision_at_5_max": 0.17547730987570465, + "nauc_precision_at_5_std": 0.09577625017832547, + "nauc_recall_at_1000_diff1": 0.09545559731138092, + "nauc_recall_at_1000_max": -0.04514392731817744, + "nauc_recall_at_1000_std": 0.26635796057028654, + "nauc_recall_at_100_diff1": 0.07745769090182385, + "nauc_recall_at_100_max": -0.00980034984722344, + "nauc_recall_at_100_std": 0.26763228591492033, + "nauc_recall_at_10_diff1": 0.2814022527196166, + "nauc_recall_at_10_max": 0.1376622720948982, + "nauc_recall_at_10_std": 0.1430028148202162, + "nauc_recall_at_1_diff1": 0.4732459963619434, + "nauc_recall_at_1_max": 0.40147711944185954, + "nauc_recall_at_1_std": 0.11598894600228143, + "nauc_recall_at_20_diff1": 0.20841212058971403, + "nauc_recall_at_20_max": 0.08726705842309825, + "nauc_recall_at_20_std": 0.19607799846822052, + "nauc_recall_at_3_diff1": 0.33321973530179383, + "nauc_recall_at_3_max": 0.28415100519609954, + "nauc_recall_at_3_std": 0.039616671531695, + "nauc_recall_at_5_diff1": 0.2678834696934614, + "nauc_recall_at_5_max": 0.20063281275236816, + "nauc_recall_at_5_std": 0.053998235228928625, + "ndcg_at_1": 0.02077, + "ndcg_at_10": 0.0239, + "ndcg_at_100": 0.03912, + "ndcg_at_1000": 0.05866, + "ndcg_at_20": 0.02766, + "ndcg_at_3": 0.01989, + "ndcg_at_5": 0.02196, + "precision_at_1": 0.02077, + "precision_at_10": 0.00767, + "precision_at_100": 0.00256, + "precision_at_1000": 0.0006, + "precision_at_20": 0.00559, + "precision_at_3": 0.01305, + "precision_at_5": 0.0115, + "recall_at_1": 0.01134, + "recall_at_10": 0.02898, + "recall_at_100": 0.08692, + "recall_at_1000": 0.21272, + "recall_at_20": 0.0393, + "recall_at_3": 0.01833, + "recall_at_5": 0.02313 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLReranking.json b/results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLReranking.json new file mode 100644 index 000000000..b6d671dfb --- /dev/null +++ b/results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 332.03379106521606, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.03249, + "MAP@10(MIRACL)": 0.08859, + "MAP@100(MIRACL)": 0.12204, + "MAP@1000(MIRACL)": 0.12204, + "MAP@20(MIRACL)": 0.10096, + "MAP@3(MIRACL)": 0.06157, + "MAP@5(MIRACL)": 0.07433, + "NDCG@1(MIRACL)": 0.06576, + "NDCG@10(MIRACL)": 0.13774, + "NDCG@100(MIRACL)": 0.28298, + "NDCG@1000(MIRACL)": 0.28298, + "NDCG@20(MIRACL)": 0.17613, + "NDCG@3(MIRACL)": 0.08785, + "NDCG@5(MIRACL)": 0.10658, + "P@1(MIRACL)": 0.06576, + "P@10(MIRACL)": 0.04932, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.03966, + "P@3(MIRACL)": 0.06415, + "P@5(MIRACL)": 0.0587, + "Recall@1(MIRACL)": 0.03249, + "Recall@10(MIRACL)": 0.22317, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.34267, + "Recall@3(MIRACL)": 0.09457, + "Recall@5(MIRACL)": 0.14143, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.13774, + "nAUC_MAP@1000_diff1(MIRACL)": 0.012444774333158702, + "nAUC_MAP@1000_max(MIRACL)": 0.20148023251309904, + "nAUC_MAP@1000_std(MIRACL)": -0.04156066916150299, + "nAUC_MAP@100_diff1(MIRACL)": 0.012444774333158702, + "nAUC_MAP@100_max(MIRACL)": 0.20148023251309904, + "nAUC_MAP@100_std(MIRACL)": -0.04156066916150299, + "nAUC_MAP@10_diff1(MIRACL)": 0.013992649907279574, + "nAUC_MAP@10_max(MIRACL)": 0.19274838745069975, + "nAUC_MAP@10_std(MIRACL)": -0.03641176671308699, + "nAUC_MAP@1_diff1(MIRACL)": 0.036991733901303364, + "nAUC_MAP@1_max(MIRACL)": 0.19769368417973682, + "nAUC_MAP@1_std(MIRACL)": 0.021028022025521315, + "nAUC_MAP@20_diff1(MIRACL)": 0.017024409795056216, + "nAUC_MAP@20_max(MIRACL)": 0.18947657031822984, + "nAUC_MAP@20_std(MIRACL)": -0.03930373709578228, + "nAUC_MAP@3_diff1(MIRACL)": 0.02984350365575744, + "nAUC_MAP@3_max(MIRACL)": 0.21111825128022568, + "nAUC_MAP@3_std(MIRACL)": -0.027746362781322313, + "nAUC_MAP@5_diff1(MIRACL)": 0.030732493989563483, + "nAUC_MAP@5_max(MIRACL)": 0.19069173430647865, + "nAUC_MAP@5_std(MIRACL)": -0.027796652833142126, + "nAUC_NDCG@1000_diff1(MIRACL)": -0.002221654174595374, + "nAUC_NDCG@1000_max(MIRACL)": 0.2069793634660494, + "nAUC_NDCG@1000_std(MIRACL)": -0.07640165284010486, + "nAUC_NDCG@100_diff1(MIRACL)": -0.002221654174595374, + "nAUC_NDCG@100_max(MIRACL)": 0.2069793634660494, + "nAUC_NDCG@100_std(MIRACL)": -0.07640165284010486, + "nAUC_NDCG@10_diff1(MIRACL)": 0.00411733073703301, + "nAUC_NDCG@10_max(MIRACL)": 0.18578737409173893, + "nAUC_NDCG@10_std(MIRACL)": -0.058351281716300316, + "nAUC_NDCG@1_diff1(MIRACL)": 0.005026196958887933, + "nAUC_NDCG@1_max(MIRACL)": 0.17289147245478406, + "nAUC_NDCG@1_std(MIRACL)": -0.005073001583829222, + "nAUC_NDCG@20_diff1(MIRACL)": 0.008634395886982874, + "nAUC_NDCG@20_max(MIRACL)": 0.1771821899378481, + "nAUC_NDCG@20_std(MIRACL)": -0.0660471466562408, + "nAUC_NDCG@3_diff1(MIRACL)": 0.01012547385819658, + "nAUC_NDCG@3_max(MIRACL)": 0.20342983882260823, + "nAUC_NDCG@3_std(MIRACL)": -0.038212881540235194, + "nAUC_NDCG@5_diff1(MIRACL)": 0.024516986306320047, + "nAUC_NDCG@5_max(MIRACL)": 0.1834319306673172, + "nAUC_NDCG@5_std(MIRACL)": -0.044974011518910166, + "nAUC_P@1000_diff1(MIRACL)": -0.03236483459933086, + "nAUC_P@1000_max(MIRACL)": 0.15789731792927758, + "nAUC_P@1000_std(MIRACL)": -0.08015135746130718, + "nAUC_P@100_diff1(MIRACL)": -0.03236483459933046, + "nAUC_P@100_max(MIRACL)": 0.1578973179292782, + "nAUC_P@100_std(MIRACL)": -0.08015135746130667, + "nAUC_P@10_diff1(MIRACL)": -0.02862403549773072, + "nAUC_P@10_max(MIRACL)": 0.20744154929288758, + "nAUC_P@10_std(MIRACL)": -0.08286308469354495, + "nAUC_P@1_diff1(MIRACL)": 0.005026196958887933, + "nAUC_P@1_max(MIRACL)": 0.17289147245478406, + "nAUC_P@1_std(MIRACL)": -0.005073001583829222, + "nAUC_P@20_diff1(MIRACL)": -0.021013449506557435, + "nAUC_P@20_max(MIRACL)": 0.1824328319833457, + "nAUC_P@20_std(MIRACL)": -0.08125930948625942, + "nAUC_P@3_diff1(MIRACL)": -0.006476238475580832, + "nAUC_P@3_max(MIRACL)": 0.20144608407911233, + "nAUC_P@3_std(MIRACL)": -0.04959330747886969, + "nAUC_P@5_diff1(MIRACL)": 0.014476528876276993, + "nAUC_P@5_max(MIRACL)": 0.1944672985077437, + "nAUC_P@5_std(MIRACL)": -0.05343824906218265, + "nAUC_Recall@1000_diff1(MIRACL)": -0.03670548424456901, + "nAUC_Recall@1000_max(MIRACL)": 0.2113762146324399, + "nAUC_Recall@1000_std(MIRACL)": -0.14608750509598692, + "nAUC_Recall@100_diff1(MIRACL)": -0.03670548424456901, + "nAUC_Recall@100_max(MIRACL)": 0.2113762146324399, + "nAUC_Recall@100_std(MIRACL)": -0.14608750509598692, + "nAUC_Recall@10_diff1(MIRACL)": -0.007253217905559515, + "nAUC_Recall@10_max(MIRACL)": 0.148520117082639, + "nAUC_Recall@10_std(MIRACL)": -0.06861403684335979, + "nAUC_Recall@1_diff1(MIRACL)": 0.036991733901303364, + "nAUC_Recall@1_max(MIRACL)": 0.19769368417973682, + "nAUC_Recall@1_std(MIRACL)": 0.021028022025521315, + "nAUC_Recall@20_diff1(MIRACL)": 0.006145608930069141, + "nAUC_Recall@20_max(MIRACL)": 0.12287852188860986, + "nAUC_Recall@20_std(MIRACL)": -0.07665126050832406, + "nAUC_Recall@3_diff1(MIRACL)": 0.017907996059126418, + "nAUC_Recall@3_max(MIRACL)": 0.20215321399986189, + "nAUC_Recall@3_std(MIRACL)": -0.06080556283841131, + "nAUC_Recall@5_diff1(MIRACL)": 0.033201118834581815, + "nAUC_Recall@5_max(MIRACL)": 0.15612469845385965, + "nAUC_Recall@5_std(MIRACL)": -0.05732379232701142 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLRetrieval.json b/results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLRetrieval.json new file mode 100644 index 000000000..42c796c70 --- /dev/null +++ b/results/DeepPavlov__rubert-base-cased-sentence/78b5122d6365337dd4114281b0d08cd1edbb3bc8/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 21002.245460510254, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.01923, + "map_at_1": 0.00745, + "map_at_10": 0.0126, + "map_at_100": 0.01507, + "map_at_1000": 0.01562, + "map_at_20": 0.01401, + "map_at_3": 0.00939, + "map_at_5": 0.01138, + "mrr_at_1": 0.01597444089456869, + "mrr_at_10": 0.0248263096505908, + "mrr_at_100": 0.028704343525750877, + "mrr_at_1000": 0.029319069224043506, + "mrr_at_20": 0.027031580718035336, + "mrr_at_3": 0.019968051118210862, + "mrr_at_5": 0.022843450479233224, + "nauc_map_at_1000_diff1": 0.1808503520567857, + "nauc_map_at_1000_max": 0.4049798935333121, + "nauc_map_at_1000_std": 0.1331532091901272, + "nauc_map_at_100_diff1": 0.18335852500457933, + "nauc_map_at_100_max": 0.4076565743178648, + "nauc_map_at_100_std": 0.13066039729013346, + "nauc_map_at_10_diff1": 0.20583209643441155, + "nauc_map_at_10_max": 0.4338996506695928, + "nauc_map_at_10_std": 0.10013807228680226, + "nauc_map_at_1_diff1": 0.3844981112490685, + "nauc_map_at_1_max": 0.5453673040847261, + "nauc_map_at_1_std": 0.08490609560478464, + "nauc_map_at_20_diff1": 0.1901623774905697, + "nauc_map_at_20_max": 0.41021176191278347, + "nauc_map_at_20_std": 0.11627137683211432, + "nauc_map_at_3_diff1": 0.29515563891248536, + "nauc_map_at_3_max": 0.4816837417300507, + "nauc_map_at_3_std": 0.10232759214069907, + "nauc_map_at_5_diff1": 0.2114691980400611, + "nauc_map_at_5_max": 0.44503165055006305, + "nauc_map_at_5_std": 0.08566374431658773, + "nauc_mrr_at_1000_diff1": 0.21255999727745692, + "nauc_mrr_at_1000_max": 0.3349232239056018, + "nauc_mrr_at_1000_std": 0.1201210368323592, + "nauc_mrr_at_100_diff1": 0.2141040675250439, + "nauc_mrr_at_100_max": 0.33701892064890293, + "nauc_mrr_at_100_std": 0.11960503202860295, + "nauc_mrr_at_10_diff1": 0.2336757900897331, + "nauc_mrr_at_10_max": 0.3453530559661451, + "nauc_mrr_at_10_std": 0.09470594834119643, + "nauc_mrr_at_1_diff1": 0.37664451387960196, + "nauc_mrr_at_1_max": 0.424190882303245, + "nauc_mrr_at_1_std": 0.07584762337302695, + "nauc_mrr_at_20_diff1": 0.22242586878661974, + "nauc_mrr_at_20_max": 0.33700374937646177, + "nauc_mrr_at_20_std": 0.10933619808933061, + "nauc_mrr_at_3_diff1": 0.29103444008683316, + "nauc_mrr_at_3_max": 0.3593694680746989, + "nauc_mrr_at_3_std": 0.0922418033802797, + "nauc_mrr_at_5_diff1": 0.24720920022944193, + "nauc_mrr_at_5_max": 0.3414556436531041, + "nauc_mrr_at_5_std": 0.09316383421215321, + "nauc_ndcg_at_1000_diff1": 0.10325334202786589, + "nauc_ndcg_at_1000_max": 0.31122638940341246, + "nauc_ndcg_at_1000_std": 0.1901147184928521, + "nauc_ndcg_at_100_diff1": 0.11822493456131096, + "nauc_ndcg_at_100_max": 0.3314686830336522, + "nauc_ndcg_at_100_std": 0.1763232662562098, + "nauc_ndcg_at_10_diff1": 0.15973959589081294, + "nauc_ndcg_at_10_max": 0.3749472301955911, + "nauc_ndcg_at_10_std": 0.1063517357493114, + "nauc_ndcg_at_1_diff1": 0.37664451387960196, + "nauc_ndcg_at_1_max": 0.424190882303245, + "nauc_ndcg_at_1_std": 0.07584762337302695, + "nauc_ndcg_at_20_diff1": 0.13605125151204894, + "nauc_ndcg_at_20_max": 0.33797789809375167, + "nauc_ndcg_at_20_std": 0.13555167863103187, + "nauc_ndcg_at_3_diff1": 0.28077569887426795, + "nauc_ndcg_at_3_max": 0.42284770462910176, + "nauc_ndcg_at_3_std": 0.10507039379799196, + "nauc_ndcg_at_5_diff1": 0.1721870344544443, + "nauc_ndcg_at_5_max": 0.3772497938284512, + "nauc_ndcg_at_5_std": 0.08357553894164373, + "nauc_precision_at_1000_diff1": 0.0816674093313524, + "nauc_precision_at_1000_max": 0.27661008789956565, + "nauc_precision_at_1000_std": 0.2479988961701787, + "nauc_precision_at_100_diff1": 0.10222304622803763, + "nauc_precision_at_100_max": 0.317625314567852, + "nauc_precision_at_100_std": 0.2647968604993878, + "nauc_precision_at_10_diff1": 0.12823778828702606, + "nauc_precision_at_10_max": 0.3016023608759314, + "nauc_precision_at_10_std": 0.14605361659131877, + "nauc_precision_at_1_diff1": 0.37664451387960196, + "nauc_precision_at_1_max": 0.424190882303245, + "nauc_precision_at_1_std": 0.07584762337302695, + "nauc_precision_at_20_diff1": 0.11530170432745876, + "nauc_precision_at_20_max": 0.28240785135489466, + "nauc_precision_at_20_std": 0.20604106170033853, + "nauc_precision_at_3_diff1": 0.22760539357716666, + "nauc_precision_at_3_max": 0.34113586164506426, + "nauc_precision_at_3_std": 0.14712563140922208, + "nauc_precision_at_5_diff1": 0.10947570694661184, + "nauc_precision_at_5_max": 0.2713848531155379, + "nauc_precision_at_5_std": 0.1304028806358822, + "nauc_recall_at_1000_diff1": 0.04497035268229982, + "nauc_recall_at_1000_max": 0.2392505472640779, + "nauc_recall_at_1000_std": 0.19421922993521232, + "nauc_recall_at_100_diff1": 0.0492434703134225, + "nauc_recall_at_100_max": 0.2766887065471701, + "nauc_recall_at_100_std": 0.1959711088998611, + "nauc_recall_at_10_diff1": 0.06387915724531845, + "nauc_recall_at_10_max": 0.3486089975910857, + "nauc_recall_at_10_std": 0.09950664592267136, + "nauc_recall_at_1_diff1": 0.3844981112490685, + "nauc_recall_at_1_max": 0.5453673040847261, + "nauc_recall_at_1_std": 0.08490609560478464, + "nauc_recall_at_20_diff1": 0.051018068791782153, + "nauc_recall_at_20_max": 0.2892968458226205, + "nauc_recall_at_20_std": 0.13904869553012572, + "nauc_recall_at_3_diff1": 0.22184585354807296, + "nauc_recall_at_3_max": 0.4033802933601568, + "nauc_recall_at_3_std": 0.13068013058844516, + "nauc_recall_at_5_diff1": 0.055573409438299186, + "nauc_recall_at_5_max": 0.355859291659614, + "nauc_recall_at_5_std": 0.07264826510604884, + "ndcg_at_1": 0.01597, + "ndcg_at_10": 0.01923, + "ndcg_at_100": 0.0334, + "ndcg_at_1000": 0.05122, + "ndcg_at_20": 0.02446, + "ndcg_at_3": 0.01334, + "ndcg_at_5": 0.01633, + "precision_at_1": 0.01597, + "precision_at_10": 0.00639, + "precision_at_100": 0.00216, + "precision_at_1000": 0.00054, + "precision_at_20": 0.00527, + "precision_at_3": 0.00879, + "precision_at_5": 0.00879, + "recall_at_1": 0.00745, + "recall_at_10": 0.02712, + "recall_at_100": 0.08062, + "recall_at_1000": 0.19121, + "recall_at_20": 0.04322, + "recall_at_3": 0.01165, + "recall_at_5": 0.01906 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLReranking.json b/results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLReranking.json new file mode 100644 index 000000000..4cc531bd0 --- /dev/null +++ b/results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 365.3101854324341, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.03523, + "MAP@10(MIRACL)": 0.08521, + "MAP@100(MIRACL)": 0.12003, + "MAP@1000(MIRACL)": 0.12003, + "MAP@20(MIRACL)": 0.09867, + "MAP@3(MIRACL)": 0.06076, + "MAP@5(MIRACL)": 0.07137, + "NDCG@1(MIRACL)": 0.06656, + "NDCG@10(MIRACL)": 0.13275, + "NDCG@100(MIRACL)": 0.28144, + "NDCG@1000(MIRACL)": 0.28144, + "NDCG@20(MIRACL)": 0.17554, + "NDCG@3(MIRACL)": 0.08512, + "NDCG@5(MIRACL)": 0.10088, + "P@1(MIRACL)": 0.06656, + "P@10(MIRACL)": 0.04683, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.03974, + "P@3(MIRACL)": 0.05934, + "P@5(MIRACL)": 0.05405, + "Recall@1(MIRACL)": 0.03523, + "Recall@10(MIRACL)": 0.21447, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.34866, + "Recall@3(MIRACL)": 0.09156, + "Recall@5(MIRACL)": 0.1313, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.13275, + "nAUC_MAP@1000_diff1(MIRACL)": 0.01839045382022375, + "nAUC_MAP@1000_max(MIRACL)": 0.06521477526780851, + "nAUC_MAP@1000_std(MIRACL)": 0.048264654362493824, + "nAUC_MAP@100_diff1(MIRACL)": 0.01839045382022375, + "nAUC_MAP@100_max(MIRACL)": 0.06521477526780851, + "nAUC_MAP@100_std(MIRACL)": 0.048264654362493824, + "nAUC_MAP@10_diff1(MIRACL)": 0.020647993466903512, + "nAUC_MAP@10_max(MIRACL)": 0.05819788720614898, + "nAUC_MAP@10_std(MIRACL)": 0.032900284030292944, + "nAUC_MAP@1_diff1(MIRACL)": 0.07488942402363453, + "nAUC_MAP@1_max(MIRACL)": 0.06805354070611272, + "nAUC_MAP@1_std(MIRACL)": 0.0030171777778926143, + "nAUC_MAP@20_diff1(MIRACL)": 0.01898383342839747, + "nAUC_MAP@20_max(MIRACL)": 0.06135053353426655, + "nAUC_MAP@20_std(MIRACL)": 0.0488098406004644, + "nAUC_MAP@3_diff1(MIRACL)": 0.011950241748932429, + "nAUC_MAP@3_max(MIRACL)": 0.04625105501744333, + "nAUC_MAP@3_std(MIRACL)": 0.013037435281920497, + "nAUC_MAP@5_diff1(MIRACL)": 0.013976644310013045, + "nAUC_MAP@5_max(MIRACL)": 0.049182055775398664, + "nAUC_MAP@5_std(MIRACL)": 0.029182473293902472, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.01671057518048647, + "nAUC_NDCG@1000_max(MIRACL)": 0.09025970564454235, + "nAUC_NDCG@1000_std(MIRACL)": 0.05538314414404315, + "nAUC_NDCG@100_diff1(MIRACL)": 0.01671057518048647, + "nAUC_NDCG@100_max(MIRACL)": 0.09025970564454235, + "nAUC_NDCG@100_std(MIRACL)": 0.05538314414404315, + "nAUC_NDCG@10_diff1(MIRACL)": 0.016283368161920466, + "nAUC_NDCG@10_max(MIRACL)": 0.06348896189943598, + "nAUC_NDCG@10_std(MIRACL)": 0.03997987079864151, + "nAUC_NDCG@1_diff1(MIRACL)": 0.05079559985096985, + "nAUC_NDCG@1_max(MIRACL)": 0.08080072396582971, + "nAUC_NDCG@1_std(MIRACL)": 0.022086014800257778, + "nAUC_NDCG@20_diff1(MIRACL)": 0.012660827026717619, + "nAUC_NDCG@20_max(MIRACL)": 0.06601237812488703, + "nAUC_NDCG@20_std(MIRACL)": 0.06679548499242714, + "nAUC_NDCG@3_diff1(MIRACL)": -0.004633514217262749, + "nAUC_NDCG@3_max(MIRACL)": 0.049968292612775425, + "nAUC_NDCG@3_std(MIRACL)": 0.02075859198956818, + "nAUC_NDCG@5_diff1(MIRACL)": 0.0031518861428202877, + "nAUC_NDCG@5_max(MIRACL)": 0.0501210947839302, + "nAUC_NDCG@5_std(MIRACL)": 0.03838338138367369, + "nAUC_P@1000_diff1(MIRACL)": -0.006994741556038145, + "nAUC_P@1000_max(MIRACL)": 0.07968492279295374, + "nAUC_P@1000_std(MIRACL)": -0.0030949620790654206, + "nAUC_P@100_diff1(MIRACL)": -0.006994741556037683, + "nAUC_P@100_max(MIRACL)": 0.07968492279295382, + "nAUC_P@100_std(MIRACL)": -0.003094962079065289, + "nAUC_P@10_diff1(MIRACL)": 0.016581400680281164, + "nAUC_P@10_max(MIRACL)": 0.0852829494941327, + "nAUC_P@10_std(MIRACL)": 0.04256349447233155, + "nAUC_P@1_diff1(MIRACL)": 0.05079559985096985, + "nAUC_P@1_max(MIRACL)": 0.08080072396582971, + "nAUC_P@1_std(MIRACL)": 0.022086014800257778, + "nAUC_P@20_diff1(MIRACL)": 0.010002923698363288, + "nAUC_P@20_max(MIRACL)": 0.07628760282068327, + "nAUC_P@20_std(MIRACL)": 0.070023588830929, + "nAUC_P@3_diff1(MIRACL)": -0.032530161667330304, + "nAUC_P@3_max(MIRACL)": 0.04915224226141074, + "nAUC_P@3_std(MIRACL)": 0.04401195784488937, + "nAUC_P@5_diff1(MIRACL)": -0.01744319979756836, + "nAUC_P@5_max(MIRACL)": 0.05361558630490634, + "nAUC_P@5_std(MIRACL)": 0.05797671658033097, + "nAUC_Recall@1000_diff1(MIRACL)": 0.04768505596178666, + "nAUC_Recall@1000_max(MIRACL)": 0.18895880548241759, + "nAUC_Recall@1000_std(MIRACL)": 0.0985764222781227, + "nAUC_Recall@100_diff1(MIRACL)": 0.04768505596178666, + "nAUC_Recall@100_max(MIRACL)": 0.18895880548241759, + "nAUC_Recall@100_std(MIRACL)": 0.0985764222781227, + "nAUC_Recall@10_diff1(MIRACL)": 0.014816877330868816, + "nAUC_Recall@10_max(MIRACL)": 0.06134637050381316, + "nAUC_Recall@10_std(MIRACL)": 0.051513554648132294, + "nAUC_Recall@1_diff1(MIRACL)": 0.07488942402363453, + "nAUC_Recall@1_max(MIRACL)": 0.06805354070611272, + "nAUC_Recall@1_std(MIRACL)": 0.0030171777778926143, + "nAUC_Recall@20_diff1(MIRACL)": 0.008914003765195185, + "nAUC_Recall@20_max(MIRACL)": 0.06569847806747695, + "nAUC_Recall@20_std(MIRACL)": 0.10081401454605385, + "nAUC_Recall@3_diff1(MIRACL)": -0.014018446414698994, + "nAUC_Recall@3_max(MIRACL)": 0.03552355882552499, + "nAUC_Recall@3_std(MIRACL)": 0.019436560766768633, + "nAUC_Recall@5_diff1(MIRACL)": -0.004942245478783206, + "nAUC_Recall@5_max(MIRACL)": 0.039544221432146864, + "nAUC_Recall@5_std(MIRACL)": 0.05196144241520297 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLRetrieval.json b/results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLRetrieval.json new file mode 100644 index 000000000..00951e04c --- /dev/null +++ b/results/DeepPavlov__rubert-base-cased/4036cab694767a299f2b9e6492909664d9414229/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 20509.529305934906, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.00884, + "map_at_1": 0.00253, + "map_at_10": 0.00556, + "map_at_100": 0.00672, + "map_at_1000": 0.00729, + "map_at_20": 0.00605, + "map_at_3": 0.00414, + "map_at_5": 0.00467, + "mrr_at_1": 0.005591054313099041, + "mrr_at_10": 0.010698121101475734, + "mrr_at_100": 0.012838845293827096, + "mrr_at_1000": 0.013529608727618088, + "mrr_at_20": 0.011762671367983984, + "mrr_at_3": 0.008785942492012781, + "mrr_at_5": 0.009744408945686902, + "nauc_map_at_1000_diff1": -0.19921356103823729, + "nauc_map_at_1000_max": 0.2222258564143542, + "nauc_map_at_1000_std": -0.30751973193773696, + "nauc_map_at_100_diff1": -0.2123368636415114, + "nauc_map_at_100_max": 0.24332381326953975, + "nauc_map_at_100_std": -0.313547357881313, + "nauc_map_at_10_diff1": -0.23027110298899656, + "nauc_map_at_10_max": 0.26674650424384005, + "nauc_map_at_10_std": -0.3161268628517557, + "nauc_map_at_1_diff1": -0.20534679868576974, + "nauc_map_at_1_max": 0.48288783030708476, + "nauc_map_at_1_std": -0.41149427433029534, + "nauc_map_at_20_diff1": -0.2195826263121819, + "nauc_map_at_20_max": 0.2659420087945013, + "nauc_map_at_20_std": -0.31691871603736993, + "nauc_map_at_3_diff1": -0.2899448104442136, + "nauc_map_at_3_max": 0.32180570350060644, + "nauc_map_at_3_std": -0.3602905670030344, + "nauc_map_at_5_diff1": -0.25092136740627285, + "nauc_map_at_5_max": 0.29213928498680775, + "nauc_map_at_5_std": -0.345381108411095, + "nauc_mrr_at_1000_diff1": -0.1858173249222513, + "nauc_mrr_at_1000_max": 0.20576049140598773, + "nauc_mrr_at_1000_std": -0.35341076333602917, + "nauc_mrr_at_100_diff1": -0.1935500885493443, + "nauc_mrr_at_100_max": 0.21589497741861968, + "nauc_mrr_at_100_std": -0.36006688632737444, + "nauc_mrr_at_10_diff1": -0.22262502686028737, + "nauc_mrr_at_10_max": 0.24805485347749007, + "nauc_mrr_at_10_std": -0.37304695839688995, + "nauc_mrr_at_1_diff1": -0.25650884925962625, + "nauc_mrr_at_1_max": 0.43956198466451935, + "nauc_mrr_at_1_std": -0.47430106636960045, + "nauc_mrr_at_20_diff1": -0.213717728446171, + "nauc_mrr_at_20_max": 0.2347734524964038, + "nauc_mrr_at_20_std": -0.37178291114422846, + "nauc_mrr_at_3_diff1": -0.28516056182456356, + "nauc_mrr_at_3_max": 0.2739754623131517, + "nauc_mrr_at_3_std": -0.4069069570577781, + "nauc_mrr_at_5_diff1": -0.22365387168249598, + "nauc_mrr_at_5_max": 0.285117175868277, + "nauc_mrr_at_5_std": -0.3861983968838321, + "nauc_ndcg_at_1000_diff1": -0.0889295492709914, + "nauc_ndcg_at_1000_max": 0.038281879647060785, + "nauc_ndcg_at_1000_std": -0.23975402668428916, + "nauc_ndcg_at_100_diff1": -0.15040315349198352, + "nauc_ndcg_at_100_max": 0.14121401247390522, + "nauc_ndcg_at_100_std": -0.2944577185239924, + "nauc_ndcg_at_10_diff1": -0.20531721425096053, + "nauc_ndcg_at_10_max": 0.22450747319971998, + "nauc_ndcg_at_10_std": -0.306290806412707, + "nauc_ndcg_at_1_diff1": -0.25650884925962625, + "nauc_ndcg_at_1_max": 0.43956198466451935, + "nauc_ndcg_at_1_std": -0.47430106636960045, + "nauc_ndcg_at_20_diff1": -0.18248220162875217, + "nauc_ndcg_at_20_max": 0.22183237686449656, + "nauc_ndcg_at_20_std": -0.3122553444275293, + "nauc_ndcg_at_3_diff1": -0.30410337122708747, + "nauc_ndcg_at_3_max": 0.27587343566774664, + "nauc_ndcg_at_3_std": -0.3677691904664344, + "nauc_ndcg_at_5_diff1": -0.22623660286176833, + "nauc_ndcg_at_5_max": 0.274368712752, + "nauc_ndcg_at_5_std": -0.34166162087772767, + "nauc_precision_at_1000_diff1": -0.024381519527901133, + "nauc_precision_at_1000_max": -0.05653577532510617, + "nauc_precision_at_1000_std": -0.17889791805256836, + "nauc_precision_at_100_diff1": -0.07950925899259019, + "nauc_precision_at_100_max": 0.028676328337004918, + "nauc_precision_at_100_std": -0.26645291555500134, + "nauc_precision_at_10_diff1": -0.14548402176232972, + "nauc_precision_at_10_max": 0.17391203599722663, + "nauc_precision_at_10_std": -0.2802540658297276, + "nauc_precision_at_1_diff1": -0.25650884925962625, + "nauc_precision_at_1_max": 0.43956198466451935, + "nauc_precision_at_1_std": -0.47430106636960045, + "nauc_precision_at_20_diff1": -0.13048675349762734, + "nauc_precision_at_20_max": 0.12978535185457296, + "nauc_precision_at_20_std": -0.313883188584373, + "nauc_precision_at_3_diff1": -0.3104307309471353, + "nauc_precision_at_3_max": 0.2210747285346667, + "nauc_precision_at_3_std": -0.3723054320869596, + "nauc_precision_at_5_diff1": -0.16822796523780645, + "nauc_precision_at_5_max": 0.22347514023953083, + "nauc_precision_at_5_std": -0.32718303998386356, + "nauc_recall_at_1000_diff1": -0.0628684977014355, + "nauc_recall_at_1000_max": -0.027159552826940513, + "nauc_recall_at_1000_std": -0.19718244615782066, + "nauc_recall_at_100_diff1": -0.13340017087659634, + "nauc_recall_at_100_max": 0.08983191406959506, + "nauc_recall_at_100_std": -0.2769355913774109, + "nauc_recall_at_10_diff1": -0.18475376829529488, + "nauc_recall_at_10_max": 0.17005292012297682, + "nauc_recall_at_10_std": -0.2592801875863338, + "nauc_recall_at_1_diff1": -0.20534679868576974, + "nauc_recall_at_1_max": 0.48288783030708476, + "nauc_recall_at_1_std": -0.41149427433029534, + "nauc_recall_at_20_diff1": -0.1504032283634947, + "nauc_recall_at_20_max": 0.20410475783215515, + "nauc_recall_at_20_std": -0.28033858495668273, + "nauc_recall_at_3_diff1": -0.3421085662826202, + "nauc_recall_at_3_max": 0.27725497485968725, + "nauc_recall_at_3_std": -0.3313094277499435, + "nauc_recall_at_5_diff1": -0.22251826688283496, + "nauc_recall_at_5_max": 0.25720238414487817, + "nauc_recall_at_5_std": -0.3012613020023596, + "ndcg_at_1": 0.00559, + "ndcg_at_10": 0.00884, + "ndcg_at_100": 0.01717, + "ndcg_at_1000": 0.0364, + "ndcg_at_20": 0.01104, + "ndcg_at_3": 0.00608, + "ndcg_at_5": 0.00679, + "precision_at_1": 0.00559, + "precision_at_10": 0.00272, + "precision_at_100": 0.00126, + "precision_at_1000": 0.00048, + "precision_at_20": 0.00228, + "precision_at_3": 0.00426, + "precision_at_5": 0.00351, + "recall_at_1": 0.00253, + "recall_at_10": 0.01408, + "recall_at_100": 0.04632, + "recall_at_1000": 0.16823, + "recall_at_20": 0.02084, + "recall_at_3": 0.00642, + "recall_at_5": 0.00821 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLReranking.json b/results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLReranking.json new file mode 100644 index 000000000..ed470771b --- /dev/null +++ b/results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 363.7984347343445, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.10451, + "MAP@10(MIRACL)": 0.21397, + "MAP@100(MIRACL)": 0.24515, + "MAP@1000(MIRACL)": 0.24515, + "MAP@20(MIRACL)": 0.22903, + "MAP@3(MIRACL)": 0.16389, + "MAP@5(MIRACL)": 0.18833, + "NDCG@1(MIRACL)": 0.17161, + "NDCG@10(MIRACL)": 0.28858, + "NDCG@100(MIRACL)": 0.39368, + "NDCG@1000(MIRACL)": 0.39368, + "NDCG@20(MIRACL)": 0.32944, + "NDCG@3(MIRACL)": 0.21116, + "NDCG@5(MIRACL)": 0.24294, + "P@1(MIRACL)": 0.17161, + "P@10(MIRACL)": 0.09078, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.06183, + "P@3(MIRACL)": 0.13873, + "P@5(MIRACL)": 0.12029, + "Recall@1(MIRACL)": 0.10451, + "Recall@10(MIRACL)": 0.41931, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.54501, + "Recall@3(MIRACL)": 0.21986, + "Recall@5(MIRACL)": 0.30025, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.28858, + "nAUC_MAP@1000_diff1(MIRACL)": 0.1607390687465434, + "nAUC_MAP@1000_max(MIRACL)": 0.07092115713074475, + "nAUC_MAP@1000_std(MIRACL)": 0.11591297854918839, + "nAUC_MAP@100_diff1(MIRACL)": 0.1607390687465434, + "nAUC_MAP@100_max(MIRACL)": 0.07092115713074475, + "nAUC_MAP@100_std(MIRACL)": 0.11591297854918839, + "nAUC_MAP@10_diff1(MIRACL)": 0.16894200567747072, + "nAUC_MAP@10_max(MIRACL)": 0.053334485947664725, + "nAUC_MAP@10_std(MIRACL)": 0.1198555810431587, + "nAUC_MAP@1_diff1(MIRACL)": 0.2406568812154273, + "nAUC_MAP@1_max(MIRACL)": 0.027446248281086837, + "nAUC_MAP@1_std(MIRACL)": 0.09483134783738956, + "nAUC_MAP@20_diff1(MIRACL)": 0.1649881352064396, + "nAUC_MAP@20_max(MIRACL)": 0.06450289627294362, + "nAUC_MAP@20_std(MIRACL)": 0.12222711495389968, + "nAUC_MAP@3_diff1(MIRACL)": 0.19307558017652554, + "nAUC_MAP@3_max(MIRACL)": 0.03408118460497972, + "nAUC_MAP@3_std(MIRACL)": 0.10392154625521477, + "nAUC_MAP@5_diff1(MIRACL)": 0.18004294945280908, + "nAUC_MAP@5_max(MIRACL)": 0.03780160675935726, + "nAUC_MAP@5_std(MIRACL)": 0.11404671375014983, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.11853290344716626, + "nAUC_NDCG@1000_max(MIRACL)": 0.11908356154779412, + "nAUC_NDCG@1000_std(MIRACL)": 0.08955069696451529, + "nAUC_NDCG@100_diff1(MIRACL)": 0.11853290344716626, + "nAUC_NDCG@100_max(MIRACL)": 0.11908356154779412, + "nAUC_NDCG@100_std(MIRACL)": 0.08955069696451529, + "nAUC_NDCG@10_diff1(MIRACL)": 0.13852857448427391, + "nAUC_NDCG@10_max(MIRACL)": 0.0715953057693749, + "nAUC_NDCG@10_std(MIRACL)": 0.11326914700739751, + "nAUC_NDCG@1_diff1(MIRACL)": 0.16973301765262583, + "nAUC_NDCG@1_max(MIRACL)": 0.04180528628602951, + "nAUC_NDCG@1_std(MIRACL)": 0.06517467999300579, + "nAUC_NDCG@20_diff1(MIRACL)": 0.13171423724059175, + "nAUC_NDCG@20_max(MIRACL)": 0.0954427610862077, + "nAUC_NDCG@20_std(MIRACL)": 0.11453825750638774, + "nAUC_NDCG@3_diff1(MIRACL)": 0.15929205597257903, + "nAUC_NDCG@3_max(MIRACL)": 0.05027360465820666, + "nAUC_NDCG@3_std(MIRACL)": 0.09775697184845182, + "nAUC_NDCG@5_diff1(MIRACL)": 0.1489975859225961, + "nAUC_NDCG@5_max(MIRACL)": 0.04889191306587007, + "nAUC_NDCG@5_std(MIRACL)": 0.10277055905782381, + "nAUC_P@1000_diff1(MIRACL)": -0.05687422003149549, + "nAUC_P@1000_max(MIRACL)": 0.15803684408061777, + "nAUC_P@1000_std(MIRACL)": -0.014454778814518222, + "nAUC_P@100_diff1(MIRACL)": -0.05687422003149517, + "nAUC_P@100_max(MIRACL)": 0.15803684408061838, + "nAUC_P@100_std(MIRACL)": -0.014454778814517729, + "nAUC_P@10_diff1(MIRACL)": 0.0016512466585581606, + "nAUC_P@10_max(MIRACL)": 0.13009076827635588, + "nAUC_P@10_std(MIRACL)": 0.09500567949326245, + "nAUC_P@1_diff1(MIRACL)": 0.16973301765262583, + "nAUC_P@1_max(MIRACL)": 0.04180528628602951, + "nAUC_P@1_std(MIRACL)": 0.06517467999300579, + "nAUC_P@20_diff1(MIRACL)": -0.01617932124511627, + "nAUC_P@20_max(MIRACL)": 0.1648937031860745, + "nAUC_P@20_std(MIRACL)": 0.08401233007329864, + "nAUC_P@3_diff1(MIRACL)": 0.09300178200144689, + "nAUC_P@3_max(MIRACL)": 0.09110242815606388, + "nAUC_P@3_std(MIRACL)": 0.09792001636369037, + "nAUC_P@5_diff1(MIRACL)": 0.04560107863932591, + "nAUC_P@5_max(MIRACL)": 0.0996943537605223, + "nAUC_P@5_std(MIRACL)": 0.10745578602834144, + "nAUC_Recall@1000_diff1(MIRACL)": 0.00571195038839169, + "nAUC_Recall@1000_max(MIRACL)": 0.29384781415032, + "nAUC_Recall@1000_std(MIRACL)": 0.024019362155189846, + "nAUC_Recall@100_diff1(MIRACL)": 0.00571195038839169, + "nAUC_Recall@100_max(MIRACL)": 0.29384781415032, + "nAUC_Recall@100_std(MIRACL)": 0.024019362155189846, + "nAUC_Recall@10_diff1(MIRACL)": 0.09340481675768546, + "nAUC_Recall@10_max(MIRACL)": 0.07749196851450199, + "nAUC_Recall@10_std(MIRACL)": 0.118860508652886, + "nAUC_Recall@1_diff1(MIRACL)": 0.2406568812154273, + "nAUC_Recall@1_max(MIRACL)": 0.027446248281086837, + "nAUC_Recall@1_std(MIRACL)": 0.09483134783738956, + "nAUC_Recall@20_diff1(MIRACL)": 0.06778310497247497, + "nAUC_Recall@20_max(MIRACL)": 0.1318526818368991, + "nAUC_Recall@20_std(MIRACL)": 0.11597560617959335, + "nAUC_Recall@3_diff1(MIRACL)": 0.1556116198086097, + "nAUC_Recall@3_max(MIRACL)": 0.031494829298944434, + "nAUC_Recall@3_std(MIRACL)": 0.09288103202066446, + "nAUC_Recall@5_diff1(MIRACL)": 0.1212816230457404, + "nAUC_Recall@5_max(MIRACL)": 0.031155252419378072, + "nAUC_Recall@5_std(MIRACL)": 0.09562315851441967 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLRetrieval.json b/results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLRetrieval.json new file mode 100644 index 000000000..a0cb6c425 --- /dev/null +++ b/results/cointegrated__LaBSE-en-ru/cf0714e606d4af551e14ad69a7929cd6b0da7f7e/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 23193.26416158676, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.10576, + "map_at_1": 0.03644, + "map_at_10": 0.0715, + "map_at_100": 0.08195, + "map_at_1000": 0.08356, + "map_at_20": 0.07669, + "map_at_3": 0.05383, + "map_at_5": 0.06267, + "mrr_at_1": 0.07188498402555911, + "mrr_at_10": 0.12613500938181438, + "mrr_at_100": 0.13640840659922152, + "mrr_at_1000": 0.13727556645587122, + "mrr_at_20": 0.1320073706805091, + "mrr_at_3": 0.10383386581469642, + "mrr_at_5": 0.117052715654952, + "nauc_map_at_1000_diff1": 0.2512572009392238, + "nauc_map_at_1000_max": 0.17205325204002672, + "nauc_map_at_1000_std": 0.14039629168529785, + "nauc_map_at_100_diff1": 0.25162813144780055, + "nauc_map_at_100_max": 0.16994929424070968, + "nauc_map_at_100_std": 0.1367669504943269, + "nauc_map_at_10_diff1": 0.2601056598779861, + "nauc_map_at_10_max": 0.16192115749265382, + "nauc_map_at_10_std": 0.10193779587541951, + "nauc_map_at_1_diff1": 0.39704635391190385, + "nauc_map_at_1_max": 0.21135873872234623, + "nauc_map_at_1_std": 0.01568057453449297, + "nauc_map_at_20_diff1": 0.25700462808187396, + "nauc_map_at_20_max": 0.16630505296088827, + "nauc_map_at_20_std": 0.11990719102944798, + "nauc_map_at_3_diff1": 0.29199177578148033, + "nauc_map_at_3_max": 0.17358274770232346, + "nauc_map_at_3_std": 0.051819989630166804, + "nauc_map_at_5_diff1": 0.2786419037544095, + "nauc_map_at_5_max": 0.16258854084290317, + "nauc_map_at_5_std": 0.07480235108358689, + "nauc_mrr_at_1000_diff1": 0.20951597178681264, + "nauc_mrr_at_1000_max": 0.124176822665613, + "nauc_mrr_at_1000_std": 0.11798574005046951, + "nauc_mrr_at_100_diff1": 0.20949058028354023, + "nauc_mrr_at_100_max": 0.12409688092881681, + "nauc_mrr_at_100_std": 0.11814006403576938, + "nauc_mrr_at_10_diff1": 0.2134227839186618, + "nauc_mrr_at_10_max": 0.12014969300969583, + "nauc_mrr_at_10_std": 0.10590631964102942, + "nauc_mrr_at_1_diff1": 0.2708908410809206, + "nauc_mrr_at_1_max": 0.12374534761059741, + "nauc_mrr_at_1_std": 0.05805456126488014, + "nauc_mrr_at_20_diff1": 0.21192418620986853, + "nauc_mrr_at_20_max": 0.12170546689932722, + "nauc_mrr_at_20_std": 0.11390448924958053, + "nauc_mrr_at_3_diff1": 0.22232919190245748, + "nauc_mrr_at_3_max": 0.11049551954499721, + "nauc_mrr_at_3_std": 0.08623468587289258, + "nauc_mrr_at_5_diff1": 0.22101757358305782, + "nauc_mrr_at_5_max": 0.1088014874332789, + "nauc_mrr_at_5_std": 0.0907128802721397, + "nauc_ndcg_at_1000_diff1": 0.19787664370641367, + "nauc_ndcg_at_1000_max": 0.18911777692601775, + "nauc_ndcg_at_1000_std": 0.24522735282078698, + "nauc_ndcg_at_100_diff1": 0.2028158255136735, + "nauc_ndcg_at_100_max": 0.171363192061495, + "nauc_ndcg_at_100_std": 0.22325060868210958, + "nauc_ndcg_at_10_diff1": 0.22439181136094902, + "nauc_ndcg_at_10_max": 0.14826370917030995, + "nauc_ndcg_at_10_std": 0.13293784585319285, + "nauc_ndcg_at_1_diff1": 0.2708908410809206, + "nauc_ndcg_at_1_max": 0.12374534761059741, + "nauc_ndcg_at_1_std": 0.05805456126488014, + "nauc_ndcg_at_20_diff1": 0.2171764554126753, + "nauc_ndcg_at_20_max": 0.15636245394294954, + "nauc_ndcg_at_20_std": 0.17063564515967924, + "nauc_ndcg_at_3_diff1": 0.23788358032611873, + "nauc_ndcg_at_3_max": 0.13576105162126248, + "nauc_ndcg_at_3_std": 0.08743289022489725, + "nauc_ndcg_at_5_diff1": 0.2404264804529493, + "nauc_ndcg_at_5_max": 0.1341021696836697, + "nauc_ndcg_at_5_std": 0.10223524353315816, + "nauc_precision_at_1000_diff1": 0.022985637582131645, + "nauc_precision_at_1000_max": 0.16182707781224945, + "nauc_precision_at_1000_std": 0.3011418049185661, + "nauc_precision_at_100_diff1": 0.060456028260623705, + "nauc_precision_at_100_max": 0.15293981912211516, + "nauc_precision_at_100_std": 0.3250066451545661, + "nauc_precision_at_10_diff1": 0.1255811105249635, + "nauc_precision_at_10_max": 0.1331825582681074, + "nauc_precision_at_10_std": 0.2145759005208703, + "nauc_precision_at_1_diff1": 0.2708908410809206, + "nauc_precision_at_1_max": 0.12374534761059741, + "nauc_precision_at_1_std": 0.05805456126488014, + "nauc_precision_at_20_diff1": 0.11091460353856848, + "nauc_precision_at_20_max": 0.15041093253526722, + "nauc_precision_at_20_std": 0.26748545015523156, + "nauc_precision_at_3_diff1": 0.18439052947568327, + "nauc_precision_at_3_max": 0.10090168225510374, + "nauc_precision_at_3_std": 0.12241219718582981, + "nauc_precision_at_5_diff1": 0.16798807586987757, + "nauc_precision_at_5_max": 0.10339942077141671, + "nauc_precision_at_5_std": 0.15537705231960366, + "nauc_recall_at_1000_diff1": 0.10968578225418296, + "nauc_recall_at_1000_max": 0.21012588440294544, + "nauc_recall_at_1000_std": 0.37335751972847947, + "nauc_recall_at_100_diff1": 0.13663888379226063, + "nauc_recall_at_100_max": 0.1675247201188889, + "nauc_recall_at_100_std": 0.3165604963758389, + "nauc_recall_at_10_diff1": 0.1817609672128631, + "nauc_recall_at_10_max": 0.1371554131679401, + "nauc_recall_at_10_std": 0.16667966211490096, + "nauc_recall_at_1_diff1": 0.39704635391190385, + "nauc_recall_at_1_max": 0.21135873872234623, + "nauc_recall_at_1_std": 0.01568057453449297, + "nauc_recall_at_20_diff1": 0.1655297013937271, + "nauc_recall_at_20_max": 0.14559372085888742, + "nauc_recall_at_20_std": 0.23050629626110405, + "nauc_recall_at_3_diff1": 0.2396228841521712, + "nauc_recall_at_3_max": 0.14411066263755568, + "nauc_recall_at_3_std": 0.06909935341326734, + "nauc_recall_at_5_diff1": 0.23149003422182549, + "nauc_recall_at_5_max": 0.12195246232149018, + "nauc_recall_at_5_std": 0.10741670041320078, + "ndcg_at_1": 0.07188, + "ndcg_at_10": 0.10576, + "ndcg_at_100": 0.15817, + "ndcg_at_1000": 0.1958, + "ndcg_at_20": 0.12363, + "ndcg_at_3": 0.07667, + "ndcg_at_5": 0.08844, + "precision_at_1": 0.07188, + "precision_at_10": 0.03754, + "precision_at_100": 0.00947, + "precision_at_1000": 0.00161, + "precision_at_20": 0.02644, + "precision_at_3": 0.05485, + "precision_at_5": 0.04936, + "recall_at_1": 0.03644, + "recall_at_10": 0.14915, + "recall_at_100": 0.34513, + "recall_at_1000": 0.58018, + "recall_at_20": 0.20264, + "recall_at_3": 0.07134, + "recall_at_5": 0.1023 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLReranking.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLReranking.json new file mode 100644 index 000000000..e58a6abb5 --- /dev/null +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 560.3415849208832, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.21112, + "MAP@10(MIRACL)": 0.3816, + "MAP@100(MIRACL)": 0.40705, + "MAP@1000(MIRACL)": 0.40705, + "MAP@20(MIRACL)": 0.39761, + "MAP@3(MIRACL)": 0.31743, + "MAP@5(MIRACL)": 0.34929, + "NDCG@1(MIRACL)": 0.34403, + "NDCG@10(MIRACL)": 0.46747, + "NDCG@100(MIRACL)": 0.5285, + "NDCG@1000(MIRACL)": 0.5285, + "NDCG@20(MIRACL)": 0.50024, + "NDCG@3(MIRACL)": 0.39237, + "NDCG@5(MIRACL)": 0.42026, + "P@1(MIRACL)": 0.34403, + "P@10(MIRACL)": 0.13144, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.08035, + "P@3(MIRACL)": 0.24833, + "P@5(MIRACL)": 0.19198, + "Recall@1(MIRACL)": 0.21112, + "Recall@10(MIRACL)": 0.6061, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.69934, + "Recall@3(MIRACL)": 0.3995, + "Recall@5(MIRACL)": 0.48635, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.46747, + "nAUC_MAP@1000_diff1(MIRACL)": 0.1823923057793737, + "nAUC_MAP@1000_max(MIRACL)": 0.141933733291052, + "nAUC_MAP@1000_std(MIRACL)": 0.11095519644453063, + "nAUC_MAP@100_diff1(MIRACL)": 0.1823923057793737, + "nAUC_MAP@100_max(MIRACL)": 0.141933733291052, + "nAUC_MAP@100_std(MIRACL)": 0.11095519644453063, + "nAUC_MAP@10_diff1(MIRACL)": 0.19973808431259543, + "nAUC_MAP@10_max(MIRACL)": 0.11419621638854538, + "nAUC_MAP@10_std(MIRACL)": 0.09899522646401729, + "nAUC_MAP@1_diff1(MIRACL)": 0.3303948167768132, + "nAUC_MAP@1_max(MIRACL)": 0.02744715276586645, + "nAUC_MAP@1_std(MIRACL)": 0.031747140128193005, + "nAUC_MAP@20_diff1(MIRACL)": 0.1867101293845589, + "nAUC_MAP@20_max(MIRACL)": 0.12941967567205329, + "nAUC_MAP@20_std(MIRACL)": 0.1082587374122704, + "nAUC_MAP@3_diff1(MIRACL)": 0.2409784922414162, + "nAUC_MAP@3_max(MIRACL)": 0.06322589984119335, + "nAUC_MAP@3_std(MIRACL)": 0.05631687995486115, + "nAUC_MAP@5_diff1(MIRACL)": 0.22255967898022008, + "nAUC_MAP@5_max(MIRACL)": 0.08781862704539999, + "nAUC_MAP@5_std(MIRACL)": 0.07492614669478646, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.11305493848392806, + "nAUC_NDCG@1000_max(MIRACL)": 0.24613477569092226, + "nAUC_NDCG@1000_std(MIRACL)": 0.14790315805545287, + "nAUC_NDCG@100_diff1(MIRACL)": 0.11305493848392806, + "nAUC_NDCG@100_max(MIRACL)": 0.24613477569092226, + "nAUC_NDCG@100_std(MIRACL)": 0.14790315805545287, + "nAUC_NDCG@10_diff1(MIRACL)": 0.15235324460573782, + "nAUC_NDCG@10_max(MIRACL)": 0.1696991935908001, + "nAUC_NDCG@10_std(MIRACL)": 0.12743939217569245, + "nAUC_NDCG@1_diff1(MIRACL)": 0.23186506058437903, + "nAUC_NDCG@1_max(MIRACL)": 0.16491090330145136, + "nAUC_NDCG@1_std(MIRACL)": 0.09147635516063957, + "nAUC_NDCG@20_diff1(MIRACL)": 0.1280518111457371, + "nAUC_NDCG@20_max(MIRACL)": 0.20540177314188754, + "nAUC_NDCG@20_std(MIRACL)": 0.14585000900293443, + "nAUC_NDCG@3_diff1(MIRACL)": 0.18429527227758208, + "nAUC_NDCG@3_max(MIRACL)": 0.1262398976112893, + "nAUC_NDCG@3_std(MIRACL)": 0.08630425957389345, + "nAUC_NDCG@5_diff1(MIRACL)": 0.1737729208321552, + "nAUC_NDCG@5_max(MIRACL)": 0.13520814158454686, + "nAUC_NDCG@5_std(MIRACL)": 0.0994026239497414, + "nAUC_P@1000_diff1(MIRACL)": -0.1959784988626437, + "nAUC_P@1000_max(MIRACL)": 0.3202734885527638, + "nAUC_P@1000_std(MIRACL)": 0.15250061141046048, + "nAUC_P@100_diff1(MIRACL)": -0.19597849886264354, + "nAUC_P@100_max(MIRACL)": 0.32027348855276416, + "nAUC_P@100_std(MIRACL)": 0.15250061141046053, + "nAUC_P@10_diff1(MIRACL)": -0.14311094299741098, + "nAUC_P@10_max(MIRACL)": 0.2867370876219064, + "nAUC_P@10_std(MIRACL)": 0.20360592106048991, + "nAUC_P@1_diff1(MIRACL)": 0.23186506058437903, + "nAUC_P@1_max(MIRACL)": 0.16491090330145136, + "nAUC_P@1_std(MIRACL)": 0.09147635516063957, + "nAUC_P@20_diff1(MIRACL)": -0.1869835582909233, + "nAUC_P@20_max(MIRACL)": 0.302116695902364, + "nAUC_P@20_std(MIRACL)": 0.19865062455707078, + "nAUC_P@3_diff1(MIRACL)": 0.006753373092379826, + "nAUC_P@3_max(MIRACL)": 0.22179247147343825, + "nAUC_P@3_std(MIRACL)": 0.13353757115193057, + "nAUC_P@5_diff1(MIRACL)": -0.06804352878808012, + "nAUC_P@5_max(MIRACL)": 0.2595778305774872, + "nAUC_P@5_std(MIRACL)": 0.176712635896499, + "nAUC_Recall@1000_diff1(MIRACL)": -0.13333260691512308, + "nAUC_Recall@1000_max(MIRACL)": 0.5863964763269425, + "nAUC_Recall@1000_std(MIRACL)": 0.2544234893808267, + "nAUC_Recall@100_diff1(MIRACL)": -0.13333260691512308, + "nAUC_Recall@100_max(MIRACL)": 0.5863964763269425, + "nAUC_Recall@100_std(MIRACL)": 0.2544234893808267, + "nAUC_Recall@10_diff1(MIRACL)": 0.06257877631905441, + "nAUC_Recall@10_max(MIRACL)": 0.188665120712509, + "nAUC_Recall@10_std(MIRACL)": 0.15048763385054417, + "nAUC_Recall@1_diff1(MIRACL)": 0.3303948167768132, + "nAUC_Recall@1_max(MIRACL)": 0.02744715276586645, + "nAUC_Recall@1_std(MIRACL)": 0.031747140128193005, + "nAUC_Recall@20_diff1(MIRACL)": -0.019872349569906737, + "nAUC_Recall@20_max(MIRACL)": 0.3125012906578129, + "nAUC_Recall@20_std(MIRACL)": 0.21647681953646483, + "nAUC_Recall@3_diff1(MIRACL)": 0.18327904514610804, + "nAUC_Recall@3_max(MIRACL)": 0.06246668078286075, + "nAUC_Recall@3_std(MIRACL)": 0.05856620613793044, + "nAUC_Recall@5_diff1(MIRACL)": 0.1276251814256237, + "nAUC_Recall@5_max(MIRACL)": 0.10743300277604201, + "nAUC_Recall@5_std(MIRACL)": 0.08991908920730439 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLRetrieval.json b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLRetrieval.json new file mode 100644 index 000000000..7b3eccd9c --- /dev/null +++ b/results/deepvk__USER-base/436a489a2087d61aa670b3496a9915f84e46c861/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 27397.726912736893, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.35216, + "map_at_1": 0.14044, + "map_at_10": 0.26309, + "map_at_100": 0.28981, + "map_at_1000": 0.29178, + "map_at_20": 0.27807, + "map_at_3": 0.2155, + "map_at_5": 0.23917, + "mrr_at_1": 0.28594249201277955, + "mrr_at_10": 0.4109466124042799, + "mrr_at_100": 0.42077830360362506, + "mrr_at_1000": 0.42113716816178787, + "mrr_at_20": 0.4176780722978009, + "mrr_at_3": 0.3772630457933969, + "mrr_at_5": 0.3970713525026616, + "nauc_map_at_1000_diff1": 0.16481568994209758, + "nauc_map_at_1000_max": 0.13778244917939705, + "nauc_map_at_1000_std": 0.04034157349782001, + "nauc_map_at_100_diff1": 0.16517153022323655, + "nauc_map_at_100_max": 0.1360329155622734, + "nauc_map_at_100_std": 0.03872677269446993, + "nauc_map_at_10_diff1": 0.16937910311866264, + "nauc_map_at_10_max": 0.11622704479010856, + "nauc_map_at_10_std": 0.007703975112991193, + "nauc_map_at_1_diff1": 0.2228846096271204, + "nauc_map_at_1_max": 0.07765431366600557, + "nauc_map_at_1_std": -0.04089387584581535, + "nauc_map_at_20_diff1": 0.1650518990050143, + "nauc_map_at_20_max": 0.12767733816688695, + "nauc_map_at_20_std": 0.022903038006970706, + "nauc_map_at_3_diff1": 0.18715860335275827, + "nauc_map_at_3_max": 0.0919729239336384, + "nauc_map_at_3_std": -0.018578092608548556, + "nauc_map_at_5_diff1": 0.17824264455272681, + "nauc_map_at_5_max": 0.1050961042878537, + "nauc_map_at_5_std": -0.007053798354989882, + "nauc_mrr_at_1000_diff1": 0.1435798304083784, + "nauc_mrr_at_1000_max": 0.19479990493636898, + "nauc_mrr_at_1000_std": 0.1072567331574904, + "nauc_mrr_at_100_diff1": 0.14348547139617326, + "nauc_mrr_at_100_max": 0.19493692705901222, + "nauc_mrr_at_100_std": 0.10767814905190118, + "nauc_mrr_at_10_diff1": 0.14313944343725465, + "nauc_mrr_at_10_max": 0.19297074786708043, + "nauc_mrr_at_10_std": 0.10470901572340778, + "nauc_mrr_at_1_diff1": 0.1717974919463759, + "nauc_mrr_at_1_max": 0.18036177614568435, + "nauc_mrr_at_1_std": 0.06062602208644375, + "nauc_mrr_at_20_diff1": 0.14294385157334125, + "nauc_mrr_at_20_max": 0.19434834549819424, + "nauc_mrr_at_20_std": 0.10758721954050914, + "nauc_mrr_at_3_diff1": 0.1491852013469599, + "nauc_mrr_at_3_max": 0.18367271730534854, + "nauc_mrr_at_3_std": 0.08855199931404897, + "nauc_mrr_at_5_diff1": 0.14161939109263946, + "nauc_mrr_at_5_max": 0.1882628841049982, + "nauc_mrr_at_5_std": 0.10194120096163446, + "nauc_ndcg_at_1000_diff1": 0.1432075438376385, + "nauc_ndcg_at_1000_max": 0.20755126685248662, + "nauc_ndcg_at_1000_std": 0.14053295605990943, + "nauc_ndcg_at_100_diff1": 0.1450681578686777, + "nauc_ndcg_at_100_max": 0.19762138854219458, + "nauc_ndcg_at_100_std": 0.13850065768744688, + "nauc_ndcg_at_10_diff1": 0.14811269776863384, + "nauc_ndcg_at_10_max": 0.14858935275235824, + "nauc_ndcg_at_10_std": 0.055353893746812356, + "nauc_ndcg_at_1_diff1": 0.1717974919463759, + "nauc_ndcg_at_1_max": 0.18036177614568435, + "nauc_ndcg_at_1_std": 0.06062602208644375, + "nauc_ndcg_at_20_diff1": 0.14051177320452046, + "nauc_ndcg_at_20_max": 0.17032992847697223, + "nauc_ndcg_at_20_std": 0.08648598936236548, + "nauc_ndcg_at_3_diff1": 0.15848490503685106, + "nauc_ndcg_at_3_max": 0.1420363788575197, + "nauc_ndcg_at_3_std": 0.04378165071699611, + "nauc_ndcg_at_5_diff1": 0.15441586061122994, + "nauc_ndcg_at_5_max": 0.13943370801564334, + "nauc_ndcg_at_5_std": 0.04299078112088833, + "nauc_precision_at_1000_diff1": -0.0693850001257654, + "nauc_precision_at_1000_max": 0.23895457301265866, + "nauc_precision_at_1000_std": 0.3104748618015318, + "nauc_precision_at_100_diff1": -0.04124129473256874, + "nauc_precision_at_100_max": 0.2650630128799022, + "nauc_precision_at_100_std": 0.34597349056547744, + "nauc_precision_at_10_diff1": 0.011975278358837714, + "nauc_precision_at_10_max": 0.22743813089548828, + "nauc_precision_at_10_std": 0.1993056614622371, + "nauc_precision_at_1_diff1": 0.1717974919463759, + "nauc_precision_at_1_max": 0.18036177614568435, + "nauc_precision_at_1_std": 0.06062602208644375, + "nauc_precision_at_20_diff1": -0.01325046000697272, + "nauc_precision_at_20_max": 0.25277266222735995, + "nauc_precision_at_20_std": 0.26257037178413756, + "nauc_precision_at_3_diff1": 0.07952419005134455, + "nauc_precision_at_3_max": 0.20262624120983022, + "nauc_precision_at_3_std": 0.13721345332727053, + "nauc_precision_at_5_diff1": 0.050353434949167254, + "nauc_precision_at_5_max": 0.22305126160186645, + "nauc_precision_at_5_std": 0.16948618118197217, + "nauc_recall_at_1000_diff1": 0.05450086317600606, + "nauc_recall_at_1000_max": 0.5131843341034735, + "nauc_recall_at_1000_std": 0.6259159892171178, + "nauc_recall_at_100_diff1": 0.08555859909425792, + "nauc_recall_at_100_max": 0.24547678339071335, + "nauc_recall_at_100_std": 0.32674693031150087, + "nauc_recall_at_10_diff1": 0.0956565022136966, + "nauc_recall_at_10_max": 0.1153704756423791, + "nauc_recall_at_10_std": 0.052319521754105446, + "nauc_recall_at_1_diff1": 0.2228846096271204, + "nauc_recall_at_1_max": 0.07765431366600557, + "nauc_recall_at_1_std": -0.04089387584581535, + "nauc_recall_at_20_diff1": 0.07192642711241914, + "nauc_recall_at_20_max": 0.15817325629728196, + "nauc_recall_at_20_std": 0.11737077259493674, + "nauc_recall_at_3_diff1": 0.1566837237945779, + "nauc_recall_at_3_max": 0.07826704455497915, + "nauc_recall_at_3_std": -0.006212673105043959, + "nauc_recall_at_5_diff1": 0.1251760704584453, + "nauc_recall_at_5_max": 0.09749037650409106, + "nauc_recall_at_5_std": 0.023717947434930144, + "ndcg_at_1": 0.28594, + "ndcg_at_10": 0.35216, + "ndcg_at_100": 0.44159, + "ndcg_at_1000": 0.47062, + "ndcg_at_20": 0.39015, + "ndcg_at_3": 0.29528, + "ndcg_at_5": 0.31334, + "precision_at_1": 0.28594, + "precision_at_10": 0.11134, + "precision_at_100": 0.02098, + "precision_at_1000": 0.00263, + "precision_at_20": 0.073, + "precision_at_3": 0.20634, + "precision_at_5": 0.16102, + "recall_at_1": 0.14044, + "recall_at_10": 0.45461, + "recall_at_100": 0.76199, + "recall_at_1000": 0.92997, + "recall_at_20": 0.56236, + "recall_at_3": 0.27919, + "recall_at_5": 0.34925 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLReranking.json b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLReranking.json new file mode 100644 index 000000000..c3043261e --- /dev/null +++ b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 1337.5348699092865, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.37343, + "MAP@10(MIRACL)": 0.58028, + "MAP@100(MIRACL)": 0.59596, + "MAP@1000(MIRACL)": 0.59596, + "MAP@20(MIRACL)": 0.59212, + "MAP@3(MIRACL)": 0.51002, + "MAP@5(MIRACL)": 0.54835, + "NDCG@1(MIRACL)": 0.61187, + "NDCG@10(MIRACL)": 0.64352, + "NDCG@100(MIRACL)": 0.67274, + "NDCG@1000(MIRACL)": 0.67274, + "NDCG@20(MIRACL)": 0.66272, + "NDCG@3(MIRACL)": 0.59802, + "NDCG@5(MIRACL)": 0.61268, + "P@1(MIRACL)": 0.61187, + "P@10(MIRACL)": 0.16279, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.09102, + "P@3(MIRACL)": 0.36621, + "P@5(MIRACL)": 0.2672, + "Recall@1(MIRACL)": 0.37343, + "Recall@10(MIRACL)": 0.71532, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.76588, + "Recall@3(MIRACL)": 0.56204, + "Recall@5(MIRACL)": 0.6347, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.64352, + "nAUC_MAP@1000_diff1(MIRACL)": 0.16369580982116821, + "nAUC_MAP@1000_max(MIRACL)": 0.42239620598518807, + "nAUC_MAP@1000_std(MIRACL)": 0.16749837607576545, + "nAUC_MAP@100_diff1(MIRACL)": 0.16369580982116821, + "nAUC_MAP@100_max(MIRACL)": 0.42239620598518807, + "nAUC_MAP@100_std(MIRACL)": 0.16749837607576545, + "nAUC_MAP@10_diff1(MIRACL)": 0.1777354084810405, + "nAUC_MAP@10_max(MIRACL)": 0.40110524759880395, + "nAUC_MAP@10_std(MIRACL)": 0.16316774574850326, + "nAUC_MAP@1_diff1(MIRACL)": 0.34751213408864073, + "nAUC_MAP@1_max(MIRACL)": 0.21124484523163467, + "nAUC_MAP@1_std(MIRACL)": 0.06745736791545207, + "nAUC_MAP@20_diff1(MIRACL)": 0.1663518826651002, + "nAUC_MAP@20_max(MIRACL)": 0.4167337547244206, + "nAUC_MAP@20_std(MIRACL)": 0.16838164363990538, + "nAUC_MAP@3_diff1(MIRACL)": 0.23408176613171383, + "nAUC_MAP@3_max(MIRACL)": 0.31793438164303434, + "nAUC_MAP@3_std(MIRACL)": 0.11792421168822836, + "nAUC_MAP@5_diff1(MIRACL)": 0.21164394176000853, + "nAUC_MAP@5_max(MIRACL)": 0.36546488695604123, + "nAUC_MAP@5_std(MIRACL)": 0.13888031171581466, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.09054922137454599, + "nAUC_NDCG@1000_max(MIRACL)": 0.5020999012050297, + "nAUC_NDCG@1000_std(MIRACL)": 0.19504823492778492, + "nAUC_NDCG@100_diff1(MIRACL)": 0.09054922137454599, + "nAUC_NDCG@100_max(MIRACL)": 0.5020999012050297, + "nAUC_NDCG@100_std(MIRACL)": 0.19504823492778492, + "nAUC_NDCG@10_diff1(MIRACL)": 0.12012030798150602, + "nAUC_NDCG@10_max(MIRACL)": 0.4614166178163878, + "nAUC_NDCG@10_std(MIRACL)": 0.1930005379666468, + "nAUC_NDCG@1_diff1(MIRACL)": 0.21635416648720676, + "nAUC_NDCG@1_max(MIRACL)": 0.48987316016525373, + "nAUC_NDCG@1_std(MIRACL)": 0.17870774535876865, + "nAUC_NDCG@20_diff1(MIRACL)": 0.09920483781956703, + "nAUC_NDCG@20_max(MIRACL)": 0.48690054646646186, + "nAUC_NDCG@20_std(MIRACL)": 0.1986539242571865, + "nAUC_NDCG@3_diff1(MIRACL)": 0.16018379648197664, + "nAUC_NDCG@3_max(MIRACL)": 0.41880466938344874, + "nAUC_NDCG@3_std(MIRACL)": 0.16054351380660972, + "nAUC_NDCG@5_diff1(MIRACL)": 0.15663699892608932, + "nAUC_NDCG@5_max(MIRACL)": 0.43368181772414954, + "nAUC_NDCG@5_std(MIRACL)": 0.16793212849418865, + "nAUC_P@1000_diff1(MIRACL)": -0.24806959682156068, + "nAUC_P@1000_max(MIRACL)": 0.30211561782669605, + "nAUC_P@1000_std(MIRACL)": 0.11181259863421601, + "nAUC_P@100_diff1(MIRACL)": -0.24806959682156018, + "nAUC_P@100_max(MIRACL)": 0.30211561782669666, + "nAUC_P@100_std(MIRACL)": 0.11181259863421626, + "nAUC_P@10_diff1(MIRACL)": -0.22923642537662203, + "nAUC_P@10_max(MIRACL)": 0.33581426488360555, + "nAUC_P@10_std(MIRACL)": 0.1640192243717893, + "nAUC_P@1_diff1(MIRACL)": 0.21635416648720676, + "nAUC_P@1_max(MIRACL)": 0.48987316016525373, + "nAUC_P@1_std(MIRACL)": 0.17870774535876865, + "nAUC_P@20_diff1(MIRACL)": -0.2482204044648911, + "nAUC_P@20_max(MIRACL)": 0.3157053568013486, + "nAUC_P@20_std(MIRACL)": 0.14292999781296842, + "nAUC_P@3_diff1(MIRACL)": -0.1318222462176626, + "nAUC_P@3_max(MIRACL)": 0.3879173415208174, + "nAUC_P@3_std(MIRACL)": 0.16118644883569527, + "nAUC_P@5_diff1(MIRACL)": -0.1748159906892032, + "nAUC_P@5_max(MIRACL)": 0.3788807688498715, + "nAUC_P@5_std(MIRACL)": 0.16082721688316387, + "nAUC_Recall@1000_diff1(MIRACL)": -0.1330629926866159, + "nAUC_Recall@1000_max(MIRACL)": 0.6545383504160608, + "nAUC_Recall@1000_std(MIRACL)": 0.22805798801939764, + "nAUC_Recall@100_diff1(MIRACL)": -0.1330629926866159, + "nAUC_Recall@100_max(MIRACL)": 0.6545383504160608, + "nAUC_Recall@100_std(MIRACL)": 0.22805798801939764, + "nAUC_Recall@10_diff1(MIRACL)": 0.0060777110423422065, + "nAUC_Recall@10_max(MIRACL)": 0.4792759189971433, + "nAUC_Recall@10_std(MIRACL)": 0.21573575482628432, + "nAUC_Recall@1_diff1(MIRACL)": 0.34751213408864073, + "nAUC_Recall@1_max(MIRACL)": 0.21124484523163467, + "nAUC_Recall@1_std(MIRACL)": 0.06745736791545207, + "nAUC_Recall@20_diff1(MIRACL)": -0.07347569466401233, + "nAUC_Recall@20_max(MIRACL)": 0.5688806666152781, + "nAUC_Recall@20_std(MIRACL)": 0.23749974664538245, + "nAUC_Recall@3_diff1(MIRACL)": 0.17144613501832698, + "nAUC_Recall@3_max(MIRACL)": 0.3062693844995822, + "nAUC_Recall@3_std(MIRACL)": 0.12833257606180778, + "nAUC_Recall@5_diff1(MIRACL)": 0.11584608926735515, + "nAUC_Recall@5_max(MIRACL)": 0.3830865024658133, + "nAUC_Recall@5_std(MIRACL)": 0.1598944180564982 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLRetrieval.json b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLRetrieval.json new file mode 100644 index 000000000..5235b772d --- /dev/null +++ b/results/deepvk__USER-bge-m3/0cc6cfe48e260fb0474c753087a69369e88709ae/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 79733.88897752762, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.67331, + "map_at_1": 0.33707, + "map_at_10": 0.5718, + "map_at_100": 0.59834, + "map_at_1000": 0.59905, + "map_at_20": 0.58952, + "map_at_3": 0.48975, + "map_at_5": 0.53302, + "mrr_at_1": 0.6549520766773163, + "mrr_at_10": 0.7575488741822615, + "mrr_at_100": 0.7602737679821753, + "mrr_at_1000": 0.7603338538394415, + "mrr_at_20": 0.7596905007927257, + "mrr_at_3": 0.741347177848776, + "mrr_at_5": 0.7521698615548466, + "nauc_map_at_1000_diff1": 0.31046412695726244, + "nauc_map_at_1000_max": 0.2535704433009994, + "nauc_map_at_1000_std": -0.00444676686164573, + "nauc_map_at_100_diff1": 0.31017908451995324, + "nauc_map_at_100_max": 0.2534098992353127, + "nauc_map_at_100_std": -0.004199635452287357, + "nauc_map_at_10_diff1": 0.3096021459306623, + "nauc_map_at_10_max": 0.2375278891857717, + "nauc_map_at_10_std": -0.024297805526740827, + "nauc_map_at_1_diff1": 0.3691246395779491, + "nauc_map_at_1_max": 0.11149879052782376, + "nauc_map_at_1_std": -0.09279204793878627, + "nauc_map_at_20_diff1": 0.3077814808128069, + "nauc_map_at_20_max": 0.246177959169062, + "nauc_map_at_20_std": -0.013263324870322776, + "nauc_map_at_3_diff1": 0.3123677849000194, + "nauc_map_at_3_max": 0.18120580921955476, + "nauc_map_at_3_std": -0.06986049660935907, + "nauc_map_at_5_diff1": 0.3100206503803961, + "nauc_map_at_5_max": 0.20745951229097526, + "nauc_map_at_5_std": -0.049538601440898826, + "nauc_mrr_at_1000_diff1": 0.41429473523589566, + "nauc_mrr_at_1000_max": 0.37426073951376115, + "nauc_mrr_at_1000_std": 0.08035573736397497, + "nauc_mrr_at_100_diff1": 0.41432052910356443, + "nauc_mrr_at_100_max": 0.3743409119656549, + "nauc_mrr_at_100_std": 0.08051136673986828, + "nauc_mrr_at_10_diff1": 0.4133668780627259, + "nauc_mrr_at_10_max": 0.37380743604650696, + "nauc_mrr_at_10_std": 0.08048670295985251, + "nauc_mrr_at_1_diff1": 0.442187005246103, + "nauc_mrr_at_1_max": 0.3573626137866292, + "nauc_mrr_at_1_std": 0.05856653637322582, + "nauc_mrr_at_20_diff1": 0.41380433303304887, + "nauc_mrr_at_20_max": 0.37486373326828054, + "nauc_mrr_at_20_std": 0.08116064198923009, + "nauc_mrr_at_3_diff1": 0.41123839386001626, + "nauc_mrr_at_3_max": 0.37740157533959834, + "nauc_mrr_at_3_std": 0.08295821757798297, + "nauc_mrr_at_5_diff1": 0.4125348040159775, + "nauc_mrr_at_5_max": 0.37444987585116873, + "nauc_mrr_at_5_std": 0.07381702658526894, + "nauc_ndcg_at_1000_diff1": 0.32863044006199094, + "nauc_ndcg_at_1000_max": 0.3149453033904987, + "nauc_ndcg_at_1000_std": 0.05401425025689069, + "nauc_ndcg_at_100_diff1": 0.326314924514221, + "nauc_ndcg_at_100_max": 0.31711746911675737, + "nauc_ndcg_at_100_std": 0.06279795347970916, + "nauc_ndcg_at_10_diff1": 0.31856209311019285, + "nauc_ndcg_at_10_max": 0.2778332627728469, + "nauc_ndcg_at_10_std": 0.008124242774154512, + "nauc_ndcg_at_1_diff1": 0.442187005246103, + "nauc_ndcg_at_1_max": 0.3573626137866292, + "nauc_ndcg_at_1_std": 0.05856653637322582, + "nauc_ndcg_at_20_diff1": 0.31505957820335256, + "nauc_ndcg_at_20_max": 0.29473175040306426, + "nauc_ndcg_at_20_std": 0.03496496968007336, + "nauc_ndcg_at_3_diff1": 0.307724732413406, + "nauc_ndcg_at_3_max": 0.2776033509455335, + "nauc_ndcg_at_3_std": 0.011863583891098982, + "nauc_ndcg_at_5_diff1": 0.3142167161637325, + "nauc_ndcg_at_5_max": 0.2646608431322972, + "nauc_ndcg_at_5_std": -0.007012089616605423, + "nauc_precision_at_1000_diff1": -0.10648939536633904, + "nauc_precision_at_1000_max": 0.15479111001172807, + "nauc_precision_at_1000_std": 0.21970700850385896, + "nauc_precision_at_100_diff1": -0.10333628689000973, + "nauc_precision_at_100_max": 0.18491309988652063, + "nauc_precision_at_100_std": 0.2513800783981376, + "nauc_precision_at_10_diff1": -0.06117893658350001, + "nauc_precision_at_10_max": 0.24552564064003551, + "nauc_precision_at_10_std": 0.21623632919307056, + "nauc_precision_at_1_diff1": 0.442187005246103, + "nauc_precision_at_1_max": 0.3573626137866292, + "nauc_precision_at_1_std": 0.05856653637322582, + "nauc_precision_at_20_diff1": -0.09586764567451068, + "nauc_precision_at_20_max": 0.20858579626903279, + "nauc_precision_at_20_std": 0.2312304145798955, + "nauc_precision_at_3_diff1": 0.039964968810316, + "nauc_precision_at_3_max": 0.29881812977337596, + "nauc_precision_at_3_std": 0.14839246356433067, + "nauc_precision_at_5_diff1": -0.011865474100239136, + "nauc_precision_at_5_max": 0.28011540747629904, + "nauc_precision_at_5_std": 0.18394881324999224, + "nauc_recall_at_1000_diff1": 0.159694964244455, + "nauc_recall_at_1000_max": 0.6084738771190085, + "nauc_recall_at_1000_std": 0.4885908670612438, + "nauc_recall_at_100_diff1": 0.21977752857669278, + "nauc_recall_at_100_max": 0.42940189103214593, + "nauc_recall_at_100_std": 0.37625038294149143, + "nauc_recall_at_10_diff1": 0.23007666976624366, + "nauc_recall_at_10_max": 0.20659064378740344, + "nauc_recall_at_10_std": -0.018388774154385605, + "nauc_recall_at_1_diff1": 0.3691246395779491, + "nauc_recall_at_1_max": 0.11149879052782376, + "nauc_recall_at_1_std": -0.09279204793878627, + "nauc_recall_at_20_diff1": 0.1902768879510536, + "nauc_recall_at_20_max": 0.23987230457258862, + "nauc_recall_at_20_std": 0.07100196013337941, + "nauc_recall_at_3_diff1": 0.25280103717947044, + "nauc_recall_at_3_max": 0.1517102658040892, + "nauc_recall_at_3_std": -0.07582894775856397, + "nauc_recall_at_5_diff1": 0.22946057031694822, + "nauc_recall_at_5_max": 0.17066685250268454, + "nauc_recall_at_5_std": -0.06887384902131653, + "ndcg_at_1": 0.65495, + "ndcg_at_10": 0.67331, + "ndcg_at_100": 0.73314, + "ndcg_at_1000": 0.7407, + "ndcg_at_20": 0.70594, + "ndcg_at_3": 0.62033, + "ndcg_at_5": 0.63385, + "precision_at_1": 0.65495, + "precision_at_10": 0.19585, + "precision_at_100": 0.02653, + "precision_at_1000": 0.0028, + "precision_at_20": 0.11378, + "precision_at_3": 0.4196, + "precision_at_5": 0.30958, + "recall_at_1": 0.33707, + "recall_at_10": 0.76141, + "recall_at_100": 0.94659, + "recall_at_1000": 0.98648, + "recall_at_20": 0.85046, + "recall_at_3": 0.55863, + "recall_at_5": 0.65185 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLReranking.json b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLReranking.json new file mode 100644 index 000000000..aeb2b67cf --- /dev/null +++ b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 420.34707498550415, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.31812, + "MAP@10(MIRACL)": 0.50242, + "MAP@100(MIRACL)": 0.52271, + "MAP@1000(MIRACL)": 0.52271, + "MAP@20(MIRACL)": 0.5162, + "MAP@3(MIRACL)": 0.43843, + "MAP@5(MIRACL)": 0.47254, + "NDCG@1(MIRACL)": 0.51724, + "NDCG@10(MIRACL)": 0.57445, + "NDCG@100(MIRACL)": 0.61833, + "NDCG@1000(MIRACL)": 0.61833, + "NDCG@20(MIRACL)": 0.60024, + "NDCG@3(MIRACL)": 0.52066, + "NDCG@5(MIRACL)": 0.53896, + "P@1(MIRACL)": 0.51724, + "P@10(MIRACL)": 0.14739, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.08585, + "P@3(MIRACL)": 0.31836, + "P@5(MIRACL)": 0.23512, + "Recall@1(MIRACL)": 0.31812, + "Recall@10(MIRACL)": 0.66634, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.73754, + "Recall@3(MIRACL)": 0.49989, + "Recall@5(MIRACL)": 0.57316, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.57445, + "nAUC_MAP@1000_diff1(MIRACL)": 0.19161527398962017, + "nAUC_MAP@1000_max(MIRACL)": 0.34754941603032463, + "nAUC_MAP@1000_std(MIRACL)": 0.12688987020000986, + "nAUC_MAP@100_diff1(MIRACL)": 0.19161527398962017, + "nAUC_MAP@100_max(MIRACL)": 0.34754941603032463, + "nAUC_MAP@100_std(MIRACL)": 0.12688987020000986, + "nAUC_MAP@10_diff1(MIRACL)": 0.20915964124896091, + "nAUC_MAP@10_max(MIRACL)": 0.32960046582497143, + "nAUC_MAP@10_std(MIRACL)": 0.11836631299553067, + "nAUC_MAP@1_diff1(MIRACL)": 0.3411971555656886, + "nAUC_MAP@1_max(MIRACL)": 0.2284786913613299, + "nAUC_MAP@1_std(MIRACL)": 0.05147408603278471, + "nAUC_MAP@20_diff1(MIRACL)": 0.19629983102068446, + "nAUC_MAP@20_max(MIRACL)": 0.34225951157839335, + "nAUC_MAP@20_std(MIRACL)": 0.129607802458707, + "nAUC_MAP@3_diff1(MIRACL)": 0.24710311272902033, + "nAUC_MAP@3_max(MIRACL)": 0.2848510199390606, + "nAUC_MAP@3_std(MIRACL)": 0.08153450194477747, + "nAUC_MAP@5_diff1(MIRACL)": 0.2260726950706647, + "nAUC_MAP@5_max(MIRACL)": 0.3025845211881447, + "nAUC_MAP@5_std(MIRACL)": 0.09498634611411115, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.11937584381632427, + "nAUC_NDCG@1000_max(MIRACL)": 0.4163327936865998, + "nAUC_NDCG@1000_std(MIRACL)": 0.15427474631530055, + "nAUC_NDCG@100_diff1(MIRACL)": 0.11937584381632427, + "nAUC_NDCG@100_max(MIRACL)": 0.4163327936865998, + "nAUC_NDCG@100_std(MIRACL)": 0.15427474631530055, + "nAUC_NDCG@10_diff1(MIRACL)": 0.1586925301907808, + "nAUC_NDCG@10_max(MIRACL)": 0.37467843899933145, + "nAUC_NDCG@10_std(MIRACL)": 0.14713203794705193, + "nAUC_NDCG@1_diff1(MIRACL)": 0.24607593037412248, + "nAUC_NDCG@1_max(MIRACL)": 0.4151906967074807, + "nAUC_NDCG@1_std(MIRACL)": 0.11023479920637448, + "nAUC_NDCG@20_diff1(MIRACL)": 0.13326418834341675, + "nAUC_NDCG@20_max(MIRACL)": 0.39970036928783076, + "nAUC_NDCG@20_std(MIRACL)": 0.16496011284213272, + "nAUC_NDCG@3_diff1(MIRACL)": 0.18931294124824874, + "nAUC_NDCG@3_max(MIRACL)": 0.34685036722959733, + "nAUC_NDCG@3_std(MIRACL)": 0.11081515298708684, + "nAUC_NDCG@5_diff1(MIRACL)": 0.176748733440649, + "nAUC_NDCG@5_max(MIRACL)": 0.3446812690898362, + "nAUC_NDCG@5_std(MIRACL)": 0.11737104082665309, + "nAUC_P@1000_diff1(MIRACL)": -0.2295525933818072, + "nAUC_P@1000_max(MIRACL)": 0.2447030410124217, + "nAUC_P@1000_std(MIRACL)": 0.09739642402985318, + "nAUC_P@100_diff1(MIRACL)": -0.22955259338180706, + "nAUC_P@100_max(MIRACL)": 0.2447030410124224, + "nAUC_P@100_std(MIRACL)": 0.09739642402985393, + "nAUC_P@10_diff1(MIRACL)": -0.17061617165628162, + "nAUC_P@10_max(MIRACL)": 0.2739843511865561, + "nAUC_P@10_std(MIRACL)": 0.15692178866000603, + "nAUC_P@1_diff1(MIRACL)": 0.24607593037412248, + "nAUC_P@1_max(MIRACL)": 0.4151906967074807, + "nAUC_P@1_std(MIRACL)": 0.11023479920637448, + "nAUC_P@20_diff1(MIRACL)": -0.21508711852036375, + "nAUC_P@20_max(MIRACL)": 0.26509005981779976, + "nAUC_P@20_std(MIRACL)": 0.16274372922403277, + "nAUC_P@3_diff1(MIRACL)": -0.05758830629926193, + "nAUC_P@3_max(MIRACL)": 0.3305735012798927, + "nAUC_P@3_std(MIRACL)": 0.1311520957805537, + "nAUC_P@5_diff1(MIRACL)": -0.11704797111270464, + "nAUC_P@5_max(MIRACL)": 0.28954793610273477, + "nAUC_P@5_std(MIRACL)": 0.1364467728125387, + "nAUC_Recall@1000_diff1(MIRACL)": -0.12985756693851075, + "nAUC_Recall@1000_max(MIRACL)": 0.5694435013457453, + "nAUC_Recall@1000_std(MIRACL)": 0.21984693147494017, + "nAUC_Recall@100_diff1(MIRACL)": -0.12985756693851075, + "nAUC_Recall@100_max(MIRACL)": 0.5694435013457453, + "nAUC_Recall@100_std(MIRACL)": 0.21984693147494017, + "nAUC_Recall@10_diff1(MIRACL)": 0.0560080749860404, + "nAUC_Recall@10_max(MIRACL)": 0.36752859117402303, + "nAUC_Recall@10_std(MIRACL)": 0.18121135197598398, + "nAUC_Recall@1_diff1(MIRACL)": 0.3411971555656886, + "nAUC_Recall@1_max(MIRACL)": 0.2284786913613299, + "nAUC_Recall@1_std(MIRACL)": 0.05147408603278471, + "nAUC_Recall@20_diff1(MIRACL)": -0.036700066551683244, + "nAUC_Recall@20_max(MIRACL)": 0.45277920055175724, + "nAUC_Recall@20_std(MIRACL)": 0.24649502116921354, + "nAUC_Recall@3_diff1(MIRACL)": 0.17766964031644797, + "nAUC_Recall@3_max(MIRACL)": 0.26779723025441554, + "nAUC_Recall@3_std(MIRACL)": 0.08511580925216117, + "nAUC_Recall@5_diff1(MIRACL)": 0.1193479983642539, + "nAUC_Recall@5_max(MIRACL)": 0.2879524981032353, + "nAUC_Recall@5_std(MIRACL)": 0.10907047769211439 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLRetrieval.json b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLRetrieval.json new file mode 100644 index 000000000..53db70ef2 --- /dev/null +++ b/results/sergeyzh__LaBSE-ru-turbo/1940b046c6b5e125df11722b899130329d0a46da/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 20807.88755607605, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.5597, + "map_at_1": 0.26764, + "map_at_10": 0.4564, + "map_at_100": 0.48435, + "map_at_1000": 0.48551, + "map_at_20": 0.47338, + "map_at_3": 0.38636, + "map_at_5": 0.42381, + "mrr_at_1": 0.5287539936102237, + "mrr_at_10": 0.6471303057964402, + "mrr_at_100": 0.6517004625880416, + "mrr_at_1000": 0.6518242780365923, + "mrr_at_20": 0.6503906253486804, + "mrr_at_3": 0.6210063897763579, + "mrr_at_5": 0.6386980830670933, + "nauc_map_at_1000_diff1": 0.27416529440521775, + "nauc_map_at_1000_max": 0.29155018321625303, + "nauc_map_at_1000_std": 0.03636611003745805, + "nauc_map_at_100_diff1": 0.27393282881164016, + "nauc_map_at_100_max": 0.29120652504641886, + "nauc_map_at_100_std": 0.036273442802439715, + "nauc_map_at_10_diff1": 0.27615820355507653, + "nauc_map_at_10_max": 0.27251195552071134, + "nauc_map_at_10_std": 0.007665500803553428, + "nauc_map_at_1_diff1": 0.329241708615118, + "nauc_map_at_1_max": 0.19936429283846985, + "nauc_map_at_1_std": -0.06789474202337108, + "nauc_map_at_20_diff1": 0.2729485151571175, + "nauc_map_at_20_max": 0.2849490221592377, + "nauc_map_at_20_std": 0.02495135823142479, + "nauc_map_at_3_diff1": 0.28065407754658217, + "nauc_map_at_3_max": 0.2398458845326663, + "nauc_map_at_3_std": -0.0331693875753278, + "nauc_map_at_5_diff1": 0.27884181216910897, + "nauc_map_at_5_max": 0.2574540393882961, + "nauc_map_at_5_std": -0.01188149677416825, + "nauc_mrr_at_1000_diff1": 0.31529173219564594, + "nauc_mrr_at_1000_max": 0.3655366952949916, + "nauc_mrr_at_1000_std": 0.13157926798174252, + "nauc_mrr_at_100_diff1": 0.3152577258960199, + "nauc_mrr_at_100_max": 0.36570923363725033, + "nauc_mrr_at_100_std": 0.13182529045402172, + "nauc_mrr_at_10_diff1": 0.3148595532586969, + "nauc_mrr_at_10_max": 0.364541241694381, + "nauc_mrr_at_10_std": 0.132040781362547, + "nauc_mrr_at_1_diff1": 0.33194554878185606, + "nauc_mrr_at_1_max": 0.3386528893202122, + "nauc_mrr_at_1_std": 0.08523989518916594, + "nauc_mrr_at_20_diff1": 0.3151631076593011, + "nauc_mrr_at_20_max": 0.36623252152229596, + "nauc_mrr_at_20_std": 0.13193824975245297, + "nauc_mrr_at_3_diff1": 0.30903543188915344, + "nauc_mrr_at_3_max": 0.3614961851591655, + "nauc_mrr_at_3_std": 0.12165786138587273, + "nauc_mrr_at_5_diff1": 0.3119282830148585, + "nauc_mrr_at_5_max": 0.3621324420711308, + "nauc_mrr_at_5_std": 0.127611070240037, + "nauc_ndcg_at_1000_diff1": 0.2780074499955654, + "nauc_ndcg_at_1000_max": 0.34680559388928683, + "nauc_ndcg_at_1000_std": 0.11710333288626233, + "nauc_ndcg_at_100_diff1": 0.2742648507131034, + "nauc_ndcg_at_100_max": 0.3499297709978287, + "nauc_ndcg_at_100_std": 0.12488704833000548, + "nauc_ndcg_at_10_diff1": 0.2747236043794353, + "nauc_ndcg_at_10_max": 0.30686543840388336, + "nauc_ndcg_at_10_std": 0.05332530103820933, + "nauc_ndcg_at_1_diff1": 0.33194554878185606, + "nauc_ndcg_at_1_max": 0.3386528893202122, + "nauc_ndcg_at_1_std": 0.08523989518916594, + "nauc_ndcg_at_20_diff1": 0.2693264925599876, + "nauc_ndcg_at_20_max": 0.3332019744670605, + "nauc_ndcg_at_20_std": 0.08964271831822929, + "nauc_ndcg_at_3_diff1": 0.26889242653685386, + "nauc_ndcg_at_3_max": 0.3029353973521449, + "nauc_ndcg_at_3_std": 0.04763730282326251, + "nauc_ndcg_at_5_diff1": 0.2743375105843297, + "nauc_ndcg_at_5_max": 0.2985765931228452, + "nauc_ndcg_at_5_std": 0.037343488725734, + "nauc_precision_at_1000_diff1": -0.08946391373910205, + "nauc_precision_at_1000_max": 0.12753282696973994, + "nauc_precision_at_1000_std": 0.26336031213691846, + "nauc_precision_at_100_diff1": -0.07828114415002212, + "nauc_precision_at_100_max": 0.1741024647302496, + "nauc_precision_at_100_std": 0.3049459059430319, + "nauc_precision_at_10_diff1": -0.004309228688413683, + "nauc_precision_at_10_max": 0.2313793882764895, + "nauc_precision_at_10_std": 0.23815659052688296, + "nauc_precision_at_1_diff1": 0.33194554878185606, + "nauc_precision_at_1_max": 0.3386528893202122, + "nauc_precision_at_1_std": 0.08523989518916594, + "nauc_precision_at_20_diff1": -0.049357064838689645, + "nauc_precision_at_20_max": 0.2217017871653108, + "nauc_precision_at_20_std": 0.28723601200790044, + "nauc_precision_at_3_diff1": 0.09992786657722827, + "nauc_precision_at_3_max": 0.30500627418514165, + "nauc_precision_at_3_std": 0.17674125775966207, + "nauc_precision_at_5_diff1": 0.05286852184493803, + "nauc_precision_at_5_max": 0.270225740894786, + "nauc_precision_at_5_std": 0.21283510882680823, + "nauc_recall_at_1000_diff1": 0.1452347005770385, + "nauc_recall_at_1000_max": 0.5852168018951652, + "nauc_recall_at_1000_std": 0.7140301031521733, + "nauc_recall_at_100_diff1": 0.15319059168849597, + "nauc_recall_at_100_max": 0.431624317426493, + "nauc_recall_at_100_std": 0.3989330894214971, + "nauc_recall_at_10_diff1": 0.1980789509314901, + "nauc_recall_at_10_max": 0.24701306755555247, + "nauc_recall_at_10_std": 0.038789923903056266, + "nauc_recall_at_1_diff1": 0.329241708615118, + "nauc_recall_at_1_max": 0.19936429283846985, + "nauc_recall_at_1_std": -0.06789474202337108, + "nauc_recall_at_20_diff1": 0.16657208299277043, + "nauc_recall_at_20_max": 0.3155803759625278, + "nauc_recall_at_20_std": 0.13530826004339108, + "nauc_recall_at_3_diff1": 0.2289774371065448, + "nauc_recall_at_3_max": 0.21534015978919546, + "nauc_recall_at_3_std": -0.04153668050963815, + "nauc_recall_at_5_diff1": 0.2146910577599793, + "nauc_recall_at_5_max": 0.22287617485339403, + "nauc_recall_at_5_std": -0.015756066025045038, + "ndcg_at_1": 0.52875, + "ndcg_at_10": 0.5597, + "ndcg_at_100": 0.63278, + "ndcg_at_1000": 0.64752, + "ndcg_at_20": 0.59491, + "ndcg_at_3": 0.50047, + "ndcg_at_5": 0.52019, + "precision_at_1": 0.52875, + "precision_at_10": 0.16581, + "precision_at_100": 0.02504, + "precision_at_1000": 0.00277, + "precision_at_20": 0.10012, + "precision_at_3": 0.33573, + "precision_at_5": 0.25719, + "recall_at_1": 0.26764, + "recall_at_10": 0.65609, + "recall_at_100": 0.89086, + "recall_at_1000": 0.97556, + "recall_at_20": 0.75186, + "recall_at_3": 0.45635, + "recall_at_5": 0.54513 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file diff --git a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLReranking.json b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLReranking.json new file mode 100644 index 000000000..a1e10627e --- /dev/null +++ b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLReranking.json @@ -0,0 +1,130 @@ +{ + "dataset_revision": "6d1962c527217f8927fca80f890f14f36b2802af", + "evaluation_time": 68.77151584625244, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "MAP@1(MIRACL)": 0.23849, + "MAP@10(MIRACL)": 0.39491, + "MAP@100(MIRACL)": 0.41972, + "MAP@1000(MIRACL)": 0.41972, + "MAP@20(MIRACL)": 0.41071, + "MAP@3(MIRACL)": 0.32898, + "MAP@5(MIRACL)": 0.36361, + "NDCG@1(MIRACL)": 0.40016, + "NDCG@10(MIRACL)": 0.47726, + "NDCG@100(MIRACL)": 0.53921, + "NDCG@1000(MIRACL)": 0.53921, + "NDCG@20(MIRACL)": 0.50991, + "NDCG@3(MIRACL)": 0.40474, + "NDCG@5(MIRACL)": 0.43392, + "P@1(MIRACL)": 0.40016, + "P@10(MIRACL)": 0.13119, + "P@100(MIRACL)": 0.01953, + "P@1000(MIRACL)": 0.00195, + "P@20(MIRACL)": 0.08007, + "P@3(MIRACL)": 0.25368, + "P@5(MIRACL)": 0.19759, + "Recall@1(MIRACL)": 0.23849, + "Recall@10(MIRACL)": 0.59627, + "Recall@100(MIRACL)": 0.79952, + "Recall@1000(MIRACL)": 0.79952, + "Recall@20(MIRACL)": 0.68982, + "Recall@3(MIRACL)": 0.38858, + "Recall@5(MIRACL)": 0.48188, + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.47726, + "nAUC_MAP@1000_diff1(MIRACL)": 0.2872676892101099, + "nAUC_MAP@1000_max(MIRACL)": 0.27336971161029494, + "nAUC_MAP@1000_std(MIRACL)": 0.11176597984333532, + "nAUC_MAP@100_diff1(MIRACL)": 0.2872676892101099, + "nAUC_MAP@100_max(MIRACL)": 0.27336971161029494, + "nAUC_MAP@100_std(MIRACL)": 0.11176597984333532, + "nAUC_MAP@10_diff1(MIRACL)": 0.30114498595957584, + "nAUC_MAP@10_max(MIRACL)": 0.250244401885258, + "nAUC_MAP@10_std(MIRACL)": 0.10758785387583467, + "nAUC_MAP@1_diff1(MIRACL)": 0.38160439906321814, + "nAUC_MAP@1_max(MIRACL)": 0.1348757799161937, + "nAUC_MAP@1_std(MIRACL)": 0.07146554479094082, + "nAUC_MAP@20_diff1(MIRACL)": 0.2922977382703431, + "nAUC_MAP@20_max(MIRACL)": 0.26533313969146693, + "nAUC_MAP@20_std(MIRACL)": 0.11612225387009131, + "nAUC_MAP@3_diff1(MIRACL)": 0.32625326348599987, + "nAUC_MAP@3_max(MIRACL)": 0.1981248147039294, + "nAUC_MAP@3_std(MIRACL)": 0.07912239429341339, + "nAUC_MAP@5_diff1(MIRACL)": 0.3172814216173415, + "nAUC_MAP@5_max(MIRACL)": 0.22515815828980001, + "nAUC_MAP@5_std(MIRACL)": 0.0927272478285517, + "nAUC_NDCG@1000_diff1(MIRACL)": 0.23484682563547074, + "nAUC_NDCG@1000_max(MIRACL)": 0.3556893582666105, + "nAUC_NDCG@1000_std(MIRACL)": 0.11017139045829125, + "nAUC_NDCG@100_diff1(MIRACL)": 0.23484682563547074, + "nAUC_NDCG@100_max(MIRACL)": 0.3556893582666105, + "nAUC_NDCG@100_std(MIRACL)": 0.11017139045829125, + "nAUC_NDCG@10_diff1(MIRACL)": 0.2700317713684702, + "nAUC_NDCG@10_max(MIRACL)": 0.29957322866749264, + "nAUC_NDCG@10_std(MIRACL)": 0.12265387154098055, + "nAUC_NDCG@1_diff1(MIRACL)": 0.2984148080969219, + "nAUC_NDCG@1_max(MIRACL)": 0.2592793666928669, + "nAUC_NDCG@1_std(MIRACL)": 0.1255454845343917, + "nAUC_NDCG@20_diff1(MIRACL)": 0.25379431649764045, + "nAUC_NDCG@20_max(MIRACL)": 0.3298006599397996, + "nAUC_NDCG@20_std(MIRACL)": 0.13268963495407754, + "nAUC_NDCG@3_diff1(MIRACL)": 0.27951266148108067, + "nAUC_NDCG@3_max(MIRACL)": 0.26668618078373213, + "nAUC_NDCG@3_std(MIRACL)": 0.09624880428546952, + "nAUC_NDCG@5_diff1(MIRACL)": 0.2852720756117471, + "nAUC_NDCG@5_max(MIRACL)": 0.27713548289317713, + "nAUC_NDCG@5_std(MIRACL)": 0.10817608012312932, + "nAUC_P@1000_diff1(MIRACL)": -0.14059724600496942, + "nAUC_P@1000_max(MIRACL)": 0.3199669356764759, + "nAUC_P@1000_std(MIRACL)": 0.03835612009647427, + "nAUC_P@100_diff1(MIRACL)": -0.1405972460049691, + "nAUC_P@100_max(MIRACL)": 0.31996693567647627, + "nAUC_P@100_std(MIRACL)": 0.03835612009647461, + "nAUC_P@10_diff1(MIRACL)": -0.05134718252244719, + "nAUC_P@10_max(MIRACL)": 0.3469368554279621, + "nAUC_P@10_std(MIRACL)": 0.13132562573634232, + "nAUC_P@1_diff1(MIRACL)": 0.2984148080969219, + "nAUC_P@1_max(MIRACL)": 0.2592793666928669, + "nAUC_P@1_std(MIRACL)": 0.1255454845343917, + "nAUC_P@20_diff1(MIRACL)": -0.09761748560964138, + "nAUC_P@20_max(MIRACL)": 0.3391383413150524, + "nAUC_P@20_std(MIRACL)": 0.12527657004318224, + "nAUC_P@3_diff1(MIRACL)": 0.09580998381734214, + "nAUC_P@3_max(MIRACL)": 0.3339603177251361, + "nAUC_P@3_std(MIRACL)": 0.10865724713059256, + "nAUC_P@5_diff1(MIRACL)": 0.03606693383184591, + "nAUC_P@5_max(MIRACL)": 0.36154128687849146, + "nAUC_P@5_std(MIRACL)": 0.12387522243056538, + "nAUC_Recall@1000_diff1(MIRACL)": 0.04431296733879144, + "nAUC_Recall@1000_max(MIRACL)": 0.6193303016280437, + "nAUC_Recall@1000_std(MIRACL)": 0.04668758203112529, + "nAUC_Recall@100_diff1(MIRACL)": 0.04431296733879144, + "nAUC_Recall@100_max(MIRACL)": 0.6193303016280437, + "nAUC_Recall@100_std(MIRACL)": 0.04668758203112529, + "nAUC_Recall@10_diff1(MIRACL)": 0.20650810402384154, + "nAUC_Recall@10_max(MIRACL)": 0.3090827883493747, + "nAUC_Recall@10_std(MIRACL)": 0.12622764224025904, + "nAUC_Recall@1_diff1(MIRACL)": 0.38160439906321814, + "nAUC_Recall@1_max(MIRACL)": 0.1348757799161937, + "nAUC_Recall@1_std(MIRACL)": 0.07146554479094082, + "nAUC_Recall@20_diff1(MIRACL)": 0.15882219127599978, + "nAUC_Recall@20_max(MIRACL)": 0.41374324653328715, + "nAUC_Recall@20_std(MIRACL)": 0.15887926808674105, + "nAUC_Recall@3_diff1(MIRACL)": 0.279474649080437, + "nAUC_Recall@3_max(MIRACL)": 0.209618936355193, + "nAUC_Recall@3_std(MIRACL)": 0.07066732709709174, + "nAUC_Recall@5_diff1(MIRACL)": 0.25417661088035237, + "nAUC_Recall@5_max(MIRACL)": 0.24809018429951632, + "nAUC_Recall@5_std(MIRACL)": 0.09409100631734804 + } + ] + }, + "task_name": "MIRACLReranking" +} \ No newline at end of file diff --git a/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLRetrieval.json b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLRetrieval.json new file mode 100644 index 000000000..ebcf8c590 --- /dev/null +++ b/results/sergeyzh__rubert-tiny-turbo/8ce0cf757446ce9bb2d5f5a4ac8103c7a1049054/MIRACLRetrieval.json @@ -0,0 +1,158 @@ +{ + "dataset_revision": "main", + "evaluation_time": 3594.588621854782, + "kg_co2_emissions": null, + "mteb_version": "1.12.94", + "scores": { + "dev": [ + { + "hf_subset": "ru", + "languages": [ + "rus-Cyrl" + ], + "main_score": 0.37072, + "map_at_1": 0.15892, + "map_at_10": 0.28316, + "map_at_100": 0.30785, + "map_at_1000": 0.30956, + "map_at_20": 0.29733, + "map_at_3": 0.23297, + "map_at_5": 0.25879, + "mrr_at_1": 0.3266773162939297, + "mrr_at_10": 0.44059473097012986, + "mrr_at_100": 0.4481469339149334, + "mrr_at_1000": 0.4485610962261281, + "mrr_at_20": 0.44547837280826674, + "mrr_at_3": 0.4088125665601699, + "mrr_at_5": 0.42921991480298116, + "nauc_map_at_1000_diff1": 0.2934728880317595, + "nauc_map_at_1000_max": 0.2803889908494092, + "nauc_map_at_1000_std": 0.17497630914740578, + "nauc_map_at_100_diff1": 0.29337045965082836, + "nauc_map_at_100_max": 0.27998691270147263, + "nauc_map_at_100_std": 0.17462210070279177, + "nauc_map_at_10_diff1": 0.2981869936945455, + "nauc_map_at_10_max": 0.25986399757092, + "nauc_map_at_10_std": 0.14922839340242242, + "nauc_map_at_1_diff1": 0.3695938095593721, + "nauc_map_at_1_max": 0.16935584805075934, + "nauc_map_at_1_std": 0.04808416555473584, + "nauc_map_at_20_diff1": 0.2944108738720747, + "nauc_map_at_20_max": 0.271607391979753, + "nauc_map_at_20_std": 0.16217825416728787, + "nauc_map_at_3_diff1": 0.3150447868956032, + "nauc_map_at_3_max": 0.21946592329964754, + "nauc_map_at_3_std": 0.10631144014373939, + "nauc_map_at_5_diff1": 0.3042104116039605, + "nauc_map_at_5_max": 0.23847326640787797, + "nauc_map_at_5_std": 0.12184949672037304, + "nauc_mrr_at_1000_diff1": 0.30577766018148594, + "nauc_mrr_at_1000_max": 0.32160376957350306, + "nauc_mrr_at_1000_std": 0.18697782876244146, + "nauc_mrr_at_100_diff1": 0.3058418134662923, + "nauc_mrr_at_100_max": 0.32178471670080927, + "nauc_mrr_at_100_std": 0.1872469736971952, + "nauc_mrr_at_10_diff1": 0.3044296919147239, + "nauc_mrr_at_10_max": 0.31925368830161066, + "nauc_mrr_at_10_std": 0.18613122020197054, + "nauc_mrr_at_1_diff1": 0.33868604792897716, + "nauc_mrr_at_1_max": 0.2799830265068991, + "nauc_mrr_at_1_std": 0.12440405825733825, + "nauc_mrr_at_20_diff1": 0.30468914512889766, + "nauc_mrr_at_20_max": 0.3210491768284289, + "nauc_mrr_at_20_std": 0.18646569866910595, + "nauc_mrr_at_3_diff1": 0.30844489781487905, + "nauc_mrr_at_3_max": 0.3143040497478699, + "nauc_mrr_at_3_std": 0.17348815023544506, + "nauc_mrr_at_5_diff1": 0.30465721840475, + "nauc_mrr_at_5_max": 0.3184358007448806, + "nauc_mrr_at_5_std": 0.1828759284758846, + "nauc_ndcg_at_1000_diff1": 0.2822071320012406, + "nauc_ndcg_at_1000_max": 0.33858559389182524, + "nauc_ndcg_at_1000_std": 0.2513679016442538, + "nauc_ndcg_at_100_diff1": 0.2790018855704696, + "nauc_ndcg_at_100_max": 0.3405965217901526, + "nauc_ndcg_at_100_std": 0.2559354827563565, + "nauc_ndcg_at_10_diff1": 0.2831834682994349, + "nauc_ndcg_at_10_max": 0.29273860223996323, + "nauc_ndcg_at_10_std": 0.1923841803921502, + "nauc_ndcg_at_1_diff1": 0.33868604792897716, + "nauc_ndcg_at_1_max": 0.2799830265068991, + "nauc_ndcg_at_1_std": 0.12440405825733825, + "nauc_ndcg_at_20_diff1": 0.2786617978219243, + "nauc_ndcg_at_20_max": 0.3159664687960494, + "nauc_ndcg_at_20_std": 0.21715215364971913, + "nauc_ndcg_at_3_diff1": 0.2912787044544828, + "nauc_ndcg_at_3_max": 0.2822328642038921, + "nauc_ndcg_at_3_std": 0.15740917569269725, + "nauc_ndcg_at_5_diff1": 0.28273558859698683, + "nauc_ndcg_at_5_max": 0.28031444532811123, + "nauc_ndcg_at_5_std": 0.161413643817168, + "nauc_precision_at_1000_diff1": -0.02527989304903982, + "nauc_precision_at_1000_max": 0.24839658356397035, + "nauc_precision_at_1000_std": 0.23684404558221928, + "nauc_precision_at_100_diff1": 0.0005267616154763404, + "nauc_precision_at_100_max": 0.3127401001375868, + "nauc_precision_at_100_std": 0.3151595874475618, + "nauc_precision_at_10_diff1": 0.08489203117719982, + "nauc_precision_at_10_max": 0.34794254200161595, + "nauc_precision_at_10_std": 0.2849141048048217, + "nauc_precision_at_1_diff1": 0.33868604792897716, + "nauc_precision_at_1_max": 0.2799830265068991, + "nauc_precision_at_1_std": 0.12440405825733825, + "nauc_precision_at_20_diff1": 0.04737701832057733, + "nauc_precision_at_20_max": 0.3436379974672856, + "nauc_precision_at_20_std": 0.2963095418416236, + "nauc_precision_at_3_diff1": 0.18533743974899164, + "nauc_precision_at_3_max": 0.3344372864659781, + "nauc_precision_at_3_std": 0.21631018190355145, + "nauc_precision_at_5_diff1": 0.13017132305234802, + "nauc_precision_at_5_max": 0.34337454780038285, + "nauc_precision_at_5_std": 0.23301868255615032, + "nauc_recall_at_1000_diff1": 0.18924851692447336, + "nauc_recall_at_1000_max": 0.4348796532107701, + "nauc_recall_at_1000_std": 0.6147539790585745, + "nauc_recall_at_100_diff1": 0.18302095539300786, + "nauc_recall_at_100_max": 0.3898941154709321, + "nauc_recall_at_100_std": 0.4440805328409773, + "nauc_recall_at_10_diff1": 0.20861183802587943, + "nauc_recall_at_10_max": 0.2490488916221133, + "nauc_recall_at_10_std": 0.21368075955550989, + "nauc_recall_at_1_diff1": 0.3695938095593721, + "nauc_recall_at_1_max": 0.16935584805075934, + "nauc_recall_at_1_std": 0.04808416555473584, + "nauc_recall_at_20_diff1": 0.19139778912421246, + "nauc_recall_at_20_max": 0.3018547993867756, + "nauc_recall_at_20_std": 0.2740092020780387, + "nauc_recall_at_3_diff1": 0.26608881499217385, + "nauc_recall_at_3_max": 0.2082352720200464, + "nauc_recall_at_3_std": 0.1303527737238042, + "nauc_recall_at_5_diff1": 0.22744008970703994, + "nauc_recall_at_5_max": 0.22007503867600614, + "nauc_recall_at_5_std": 0.1517071031897581, + "ndcg_at_1": 0.32668, + "ndcg_at_10": 0.37072, + "ndcg_at_100": 0.44836, + "ndcg_at_1000": 0.47562, + "ndcg_at_20": 0.40445, + "ndcg_at_3": 0.31845, + "ndcg_at_5": 0.33585, + "precision_at_1": 0.32668, + "precision_at_10": 0.11925, + "precision_at_100": 0.02062, + "precision_at_1000": 0.00253, + "precision_at_20": 0.0744, + "precision_at_3": 0.22364, + "precision_at_5": 0.177, + "recall_at_1": 0.15892, + "recall_at_10": 0.45738, + "recall_at_100": 0.72056, + "recall_at_1000": 0.88482, + "recall_at_20": 0.55446, + "recall_at_3": 0.28798, + "recall_at_5": 0.35837 + } + ] + }, + "task_name": "MIRACLRetrieval" +} \ No newline at end of file