From bc1319777506d4a9be02cc0bc0c2d4a7bd3a1edc Mon Sep 17 00:00:00 2001 From: Kenneth Enevoldsen Date: Mon, 19 Oct 2026 10:40:18 +0200 Subject: [PATCH] add results for ColUltraVBERT closes https://github.com/embeddings-benchmark/mteb/issues/5694 --- .../Vidore3NuclearRetrieval.json | 1090 +++++++++++++++++ .../Vidore3NuclearRetrieval.v2.json | 1090 +++++++++++++++++ .../Vidore3TelecomRetrieval.json | 1090 +++++++++++++++++ .../Vidore3TelecomRetrieval.v2.json | 1090 +++++++++++++++++ .../run_settings.jsonl | 4 + 5 files changed, 4364 insertions(+) create mode 100644 results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3NuclearRetrieval.json create mode 100644 results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3NuclearRetrieval.v2.json create mode 100644 results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3TelecomRetrieval.json create mode 100644 results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3TelecomRetrieval.v2.json diff --git a/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3NuclearRetrieval.json b/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3NuclearRetrieval.json new file mode 100644 index 0000000000..65ba3bd0bd --- /dev/null +++ b/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3NuclearRetrieval.json @@ -0,0 +1,1090 @@ +{ + "dataset_revision": "a463fc67fefc01152153101e88a32d5f9515e3e3", + "task_name": "Vidore3NuclearRetrieval", + "mteb_version": "2.26.10", + "scores": { + "test": [ + { + "ndcg_at_1": 0.34187, + "ndcg_at_3": 0.36697, + "ndcg_at_5": 0.37436, + "ndcg_at_10": 0.40707, + "ndcg_at_20": 0.44522, + "ndcg_at_100": 0.49923, + "ndcg_at_1000": 0.54172, + "map_at_1": 0.16327, + "map_at_3": 0.24631, + "map_at_5": 0.27383, + "map_at_10": 0.30255, + "map_at_20": 0.32442, + "map_at_100": 0.34005, + "map_at_1000": 0.34366, + "recall_at_1": 0.16327, + "recall_at_3": 0.30517, + "recall_at_5": 0.3696, + "recall_at_10": 0.47318, + "recall_at_20": 0.57822, + "recall_at_100": 0.76336, + "recall_at_1000": 0.96738, + "accuracy": 0.16327, + "precision_at_1": 0.35542, + "precision_at_3": 0.27209, + "precision_at_5": 0.21265, + "precision_at_10": 0.15241, + "precision_at_20": 0.10361, + "precision_at_100": 0.03123, + "precision_at_1000": 0.00448, + "mrr_at_1": 0.355422, + "mrr_at_3": 0.461847, + "mrr_at_5": 0.476456, + "mrr_at_10": 0.49086, + "mrr_at_20": 0.496002, + "mrr_at_100": 0.498305, + "mrr_at_1000": 0.498605, + "nauc_ndcg_at_1_max": 0.191633, + "nauc_ndcg_at_1_std": 0.018322, + "nauc_ndcg_at_1_diff1": 0.466011, + "nauc_ndcg_at_3_max": 0.201866, + "nauc_ndcg_at_3_std": 0.035026, + "nauc_ndcg_at_3_diff1": 0.419368, + "nauc_ndcg_at_5_max": 0.194152, + "nauc_ndcg_at_5_std": 0.02281, + "nauc_ndcg_at_5_diff1": 0.417327, + "nauc_ndcg_at_10_max": 0.197393, + "nauc_ndcg_at_10_std": 0.018314, + "nauc_ndcg_at_10_diff1": 0.434747, + "nauc_ndcg_at_20_max": 0.197172, + "nauc_ndcg_at_20_std": 0.033035, + "nauc_ndcg_at_20_diff1": 0.42009, + "nauc_ndcg_at_100_max": 0.214372, + "nauc_ndcg_at_100_std": 0.049932, + "nauc_ndcg_at_100_diff1": 0.41079, + "nauc_ndcg_at_1000_max": 0.223469, + "nauc_ndcg_at_1000_std": 0.061788, + "nauc_ndcg_at_1000_diff1": 0.413775, + "nauc_map_at_1_max": 0.17475, + "nauc_map_at_1_std": 0.034879, + "nauc_map_at_1_diff1": 0.418128, + "nauc_map_at_3_max": 0.201539, + "nauc_map_at_3_std": 0.035604, + "nauc_map_at_3_diff1": 0.413591, + "nauc_map_at_5_max": 0.195162, + "nauc_map_at_5_std": 0.023648, + "nauc_map_at_5_diff1": 0.399228, + "nauc_map_at_10_max": 0.20537, + "nauc_map_at_10_std": 0.026177, + "nauc_map_at_10_diff1": 0.41043, + "nauc_map_at_20_max": 0.206304, + "nauc_map_at_20_std": 0.03742, + "nauc_map_at_20_diff1": 0.401691, + "nauc_map_at_100_max": 0.214519, + "nauc_map_at_100_std": 0.044181, + "nauc_map_at_100_diff1": 0.399057, + "nauc_map_at_1000_max": 0.216499, + "nauc_map_at_1000_std": 0.046805, + "nauc_map_at_1000_diff1": 0.399552, + "nauc_recall_at_1_max": 0.17475, + "nauc_recall_at_1_std": 0.034879, + "nauc_recall_at_1_diff1": 0.418128, + "nauc_recall_at_3_max": 0.188112, + "nauc_recall_at_3_std": 0.036942, + "nauc_recall_at_3_diff1": 0.378317, + "nauc_recall_at_5_max": 0.156128, + "nauc_recall_at_5_std": 0.006242, + "nauc_recall_at_5_diff1": 0.349147, + "nauc_recall_at_10_max": 0.160338, + "nauc_recall_at_10_std": -0.00103, + "nauc_recall_at_10_diff1": 0.357275, + "nauc_recall_at_20_max": 0.147369, + "nauc_recall_at_20_std": 0.024122, + "nauc_recall_at_20_diff1": 0.327005, + "nauc_recall_at_100_max": 0.200556, + "nauc_recall_at_100_std": 0.061978, + "nauc_recall_at_100_diff1": 0.266613, + "nauc_recall_at_1000_max": 0.406309, + "nauc_recall_at_1000_std": 0.177586, + "nauc_recall_at_1000_diff1": 0.097801, + "nauc_precision_at_1_max": 0.207392, + "nauc_precision_at_1_std": 0.027091, + "nauc_precision_at_1_diff1": 0.452338, + "nauc_precision_at_3_max": 0.174289, + "nauc_precision_at_3_std": 0.030549, + "nauc_precision_at_3_diff1": 0.307706, + "nauc_precision_at_5_max": 0.14839, + "nauc_precision_at_5_std": 0.022589, + "nauc_precision_at_5_diff1": 0.234794, + "nauc_precision_at_10_max": 0.148419, + "nauc_precision_at_10_std": 0.05405, + "nauc_precision_at_10_diff1": 0.156966, + "nauc_precision_at_20_max": 0.118722, + "nauc_precision_at_20_std": 0.093649, + "nauc_precision_at_20_diff1": 0.058242, + "nauc_precision_at_100_max": 0.092105, + "nauc_precision_at_100_std": 0.116368, + "nauc_precision_at_100_diff1": -0.056784, + "nauc_precision_at_1000_max": 0.039928, + "nauc_precision_at_1000_std": 0.137875, + "nauc_precision_at_1000_diff1": -0.153653, + "nauc_mrr_at_1_max": 0.207392, + "nauc_mrr_at_1_std": 0.027091, + "nauc_mrr_at_1_diff1": 0.452338, + "nauc_mrr_at_3_max": 0.216721, + "nauc_mrr_at_3_std": 0.040564, + "nauc_mrr_at_3_diff1": 0.45581, + "nauc_mrr_at_5_max": 0.206777, + "nauc_mrr_at_5_std": 0.037031, + "nauc_mrr_at_5_diff1": 0.453225, + "nauc_mrr_at_10_max": 0.211538, + "nauc_mrr_at_10_std": 0.038621, + "nauc_mrr_at_10_diff1": 0.445897, + "nauc_mrr_at_20_max": 0.211242, + "nauc_mrr_at_20_std": 0.036089, + "nauc_mrr_at_20_diff1": 0.447041, + "nauc_mrr_at_100_max": 0.210732, + "nauc_mrr_at_100_std": 0.035783, + "nauc_mrr_at_100_diff1": 0.447208, + "nauc_mrr_at_1000_max": 0.210796, + "nauc_mrr_at_1000_std": 0.03576, + "nauc_mrr_at_1000_diff1": 0.44719, + "hit_rate_at_1": 0.35542, + "hit_rate_at_3": 0.5994, + "hit_rate_at_5": 0.66265, + "hit_rate_at_10": 0.76506, + "hit_rate_at_20": 0.83735, + "hit_rate_at_100": 0.92771, + "hit_rate_at_1000": 0.99398, + "main_score": 0.40707, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.39006, + "ndcg_at_3": 0.38329, + "ndcg_at_5": 0.39547, + "ndcg_at_10": 0.43112, + "ndcg_at_20": 0.46034, + "ndcg_at_100": 0.51882, + "ndcg_at_1000": 0.56048, + "map_at_1": 0.17485, + "map_at_3": 0.2555, + "map_at_5": 0.28669, + "map_at_10": 0.32343, + "map_at_20": 0.34156, + "map_at_100": 0.35831, + "map_at_1000": 0.36212, + "recall_at_1": 0.17485, + "recall_at_3": 0.30909, + "recall_at_5": 0.37823, + "recall_at_10": 0.49493, + "recall_at_20": 0.58166, + "recall_at_100": 0.78012, + "recall_at_1000": 0.9734, + "accuracy": 0.17485, + "precision_at_1": 0.40361, + "precision_at_3": 0.27309, + "precision_at_5": 0.22349, + "precision_at_10": 0.16355, + "precision_at_20": 0.10407, + "precision_at_100": 0.03178, + "precision_at_1000": 0.00454, + "mrr_at_1": 0.403614, + "mrr_at_3": 0.490462, + "mrr_at_5": 0.509137, + "mrr_at_10": 0.520823, + "mrr_at_20": 0.525492, + "mrr_at_100": 0.528274, + "mrr_at_1000": 0.528592, + "nauc_ndcg_at_1_max": 0.24441, + "nauc_ndcg_at_1_std": 0.050721, + "nauc_ndcg_at_1_diff1": 0.474734, + "nauc_ndcg_at_3_max": 0.208053, + "nauc_ndcg_at_3_std": 0.050806, + "nauc_ndcg_at_3_diff1": 0.404388, + "nauc_ndcg_at_5_max": 0.213925, + "nauc_ndcg_at_5_std": 0.045755, + "nauc_ndcg_at_5_diff1": 0.418429, + "nauc_ndcg_at_10_max": 0.229034, + "nauc_ndcg_at_10_std": 0.056955, + "nauc_ndcg_at_10_diff1": 0.439906, + "nauc_ndcg_at_20_max": 0.244387, + "nauc_ndcg_at_20_std": 0.072301, + "nauc_ndcg_at_20_diff1": 0.44117, + "nauc_ndcg_at_100_max": 0.254987, + "nauc_ndcg_at_100_std": 0.083897, + "nauc_ndcg_at_100_diff1": 0.444385, + "nauc_ndcg_at_1000_max": 0.246572, + "nauc_ndcg_at_1000_std": 0.080841, + "nauc_ndcg_at_1000_diff1": 0.43915, + "nauc_map_at_1_max": 0.199386, + "nauc_map_at_1_std": 0.075943, + "nauc_map_at_1_diff1": 0.436632, + "nauc_map_at_3_max": 0.194817, + "nauc_map_at_3_std": 0.048658, + "nauc_map_at_3_diff1": 0.408736, + "nauc_map_at_5_max": 0.201196, + "nauc_map_at_5_std": 0.040218, + "nauc_map_at_5_diff1": 0.406157, + "nauc_map_at_10_max": 0.211079, + "nauc_map_at_10_std": 0.043106, + "nauc_map_at_10_diff1": 0.412175, + "nauc_map_at_20_max": 0.219876, + "nauc_map_at_20_std": 0.053359, + "nauc_map_at_20_diff1": 0.414407, + "nauc_map_at_100_max": 0.225705, + "nauc_map_at_100_std": 0.058869, + "nauc_map_at_100_diff1": 0.412667, + "nauc_map_at_1000_max": 0.225728, + "nauc_map_at_1000_std": 0.059636, + "nauc_map_at_1000_diff1": 0.412497, + "nauc_recall_at_1_max": 0.199386, + "nauc_recall_at_1_std": 0.075943, + "nauc_recall_at_1_diff1": 0.436632, + "nauc_recall_at_3_max": 0.163239, + "nauc_recall_at_3_std": 0.038586, + "nauc_recall_at_3_diff1": 0.346092, + "nauc_recall_at_5_max": 0.171412, + "nauc_recall_at_5_std": 0.027351, + "nauc_recall_at_5_diff1": 0.33958, + "nauc_recall_at_10_max": 0.216867, + "nauc_recall_at_10_std": 0.057856, + "nauc_recall_at_10_diff1": 0.369226, + "nauc_recall_at_20_max": 0.251449, + "nauc_recall_at_20_std": 0.097249, + "nauc_recall_at_20_diff1": 0.366461, + "nauc_recall_at_100_max": 0.273996, + "nauc_recall_at_100_std": 0.121835, + "nauc_recall_at_100_diff1": 0.381645, + "nauc_recall_at_1000_max": 0.175379, + "nauc_recall_at_1000_std": 0.086296, + "nauc_recall_at_1000_diff1": 0.450555, + "nauc_precision_at_1_max": 0.243777, + "nauc_precision_at_1_std": 0.043972, + "nauc_precision_at_1_diff1": 0.480372, + "nauc_precision_at_3_max": 0.159546, + "nauc_precision_at_3_std": 0.019251, + "nauc_precision_at_3_diff1": 0.283527, + "nauc_precision_at_5_max": 0.154373, + "nauc_precision_at_5_std": 0.013129, + "nauc_precision_at_5_diff1": 0.21524, + "nauc_precision_at_10_max": 0.141599, + "nauc_precision_at_10_std": 0.038452, + "nauc_precision_at_10_diff1": 0.139459, + "nauc_precision_at_20_max": 0.13368, + "nauc_precision_at_20_std": 0.068218, + "nauc_precision_at_20_diff1": 0.095231, + "nauc_precision_at_100_max": 0.080505, + "nauc_precision_at_100_std": 0.082404, + "nauc_precision_at_100_diff1": -0.028634, + "nauc_precision_at_1000_max": -0.017487, + "nauc_precision_at_1000_std": 0.053201, + "nauc_precision_at_1000_diff1": -0.147736, + "nauc_mrr_at_1_max": 0.243777, + "nauc_mrr_at_1_std": 0.043972, + "nauc_mrr_at_1_diff1": 0.480372, + "nauc_mrr_at_3_max": 0.255689, + "nauc_mrr_at_3_std": 0.075432, + "nauc_mrr_at_3_diff1": 0.476675, + "nauc_mrr_at_5_max": 0.262381, + "nauc_mrr_at_5_std": 0.077512, + "nauc_mrr_at_5_diff1": 0.482283, + "nauc_mrr_at_10_max": 0.262845, + "nauc_mrr_at_10_std": 0.077492, + "nauc_mrr_at_10_diff1": 0.485334, + "nauc_mrr_at_20_max": 0.265211, + "nauc_mrr_at_20_std": 0.078563, + "nauc_mrr_at_20_diff1": 0.482253, + "nauc_mrr_at_100_max": 0.263071, + "nauc_mrr_at_100_std": 0.076227, + "nauc_mrr_at_100_diff1": 0.483264, + "nauc_mrr_at_1000_max": 0.262985, + "nauc_mrr_at_1000_std": 0.076398, + "nauc_mrr_at_1000_diff1": 0.483056, + "hit_rate_at_1": 0.40361, + "hit_rate_at_3": 0.5994, + "hit_rate_at_5": 0.68072, + "hit_rate_at_10": 0.76807, + "hit_rate_at_20": 0.83133, + "hit_rate_at_100": 0.93675, + "hit_rate_at_1000": 0.99398, + "main_score": 0.43112, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.41717, + "ndcg_at_3": 0.4376, + "ndcg_at_5": 0.46003, + "ndcg_at_10": 0.4927, + "ndcg_at_20": 0.52773, + "ndcg_at_100": 0.58172, + "ndcg_at_1000": 0.61477, + "map_at_1": 0.21178, + "map_at_3": 0.30548, + "map_at_5": 0.34937, + "map_at_10": 0.38419, + "map_at_20": 0.40749, + "map_at_100": 0.42539, + "map_at_1000": 0.42878, + "recall_at_1": 0.21178, + "recall_at_3": 0.35657, + "recall_at_5": 0.4508, + "recall_at_10": 0.55854, + "recall_at_20": 0.65231, + "recall_at_100": 0.83271, + "recall_at_1000": 0.97895, + "accuracy": 0.21178, + "precision_at_1": 0.43072, + "precision_at_3": 0.31827, + "precision_at_5": 0.26325, + "precision_at_10": 0.18584, + "precision_at_20": 0.12108, + "precision_at_100": 0.03491, + "precision_at_1000": 0.00459, + "mrr_at_1": 0.430723, + "mrr_at_3": 0.539157, + "mrr_at_5": 0.560542, + "mrr_at_10": 0.570182, + "mrr_at_20": 0.573056, + "mrr_at_100": 0.575329, + "mrr_at_1000": 0.575473, + "nauc_ndcg_at_1_max": 0.184195, + "nauc_ndcg_at_1_std": 0.000544, + "nauc_ndcg_at_1_diff1": 0.45453, + "nauc_ndcg_at_3_max": 0.168644, + "nauc_ndcg_at_3_std": -0.017969, + "nauc_ndcg_at_3_diff1": 0.369387, + "nauc_ndcg_at_5_max": 0.176282, + "nauc_ndcg_at_5_std": -0.052259, + "nauc_ndcg_at_5_diff1": 0.396675, + "nauc_ndcg_at_10_max": 0.199336, + "nauc_ndcg_at_10_std": -0.042408, + "nauc_ndcg_at_10_diff1": 0.400027, + "nauc_ndcg_at_20_max": 0.208294, + "nauc_ndcg_at_20_std": -0.035435, + "nauc_ndcg_at_20_diff1": 0.402226, + "nauc_ndcg_at_100_max": 0.1997, + "nauc_ndcg_at_100_std": -0.03832, + "nauc_ndcg_at_100_diff1": 0.401118, + "nauc_ndcg_at_1000_max": 0.187696, + "nauc_ndcg_at_1000_std": -0.030493, + "nauc_ndcg_at_1000_diff1": 0.396848, + "nauc_map_at_1_max": 0.127005, + "nauc_map_at_1_std": -0.004698, + "nauc_map_at_1_diff1": 0.412356, + "nauc_map_at_3_max": 0.159002, + "nauc_map_at_3_std": -0.011508, + "nauc_map_at_3_diff1": 0.402773, + "nauc_map_at_5_max": 0.170232, + "nauc_map_at_5_std": -0.034711, + "nauc_map_at_5_diff1": 0.404372, + "nauc_map_at_10_max": 0.18378, + "nauc_map_at_10_std": -0.031184, + "nauc_map_at_10_diff1": 0.395154, + "nauc_map_at_20_max": 0.185373, + "nauc_map_at_20_std": -0.02945, + "nauc_map_at_20_diff1": 0.389328, + "nauc_map_at_100_max": 0.186525, + "nauc_map_at_100_std": -0.029469, + "nauc_map_at_100_diff1": 0.388297, + "nauc_map_at_1000_max": 0.185101, + "nauc_map_at_1000_std": -0.028998, + "nauc_map_at_1000_diff1": 0.387584, + "nauc_recall_at_1_max": 0.127005, + "nauc_recall_at_1_std": -0.004698, + "nauc_recall_at_1_diff1": 0.412356, + "nauc_recall_at_3_max": 0.176536, + "nauc_recall_at_3_std": 0.007696, + "nauc_recall_at_3_diff1": 0.357214, + "nauc_recall_at_5_max": 0.197987, + "nauc_recall_at_5_std": -0.04877, + "nauc_recall_at_5_diff1": 0.37489, + "nauc_recall_at_10_max": 0.246208, + "nauc_recall_at_10_std": -0.026391, + "nauc_recall_at_10_diff1": 0.347721, + "nauc_recall_at_20_max": 0.264125, + "nauc_recall_at_20_std": -0.029606, + "nauc_recall_at_20_diff1": 0.343319, + "nauc_recall_at_100_max": 0.268075, + "nauc_recall_at_100_std": -0.050363, + "nauc_recall_at_100_diff1": 0.358792, + "nauc_recall_at_1000_max": 0.330596, + "nauc_recall_at_1000_std": 0.13262, + "nauc_recall_at_1000_diff1": 0.359092, + "nauc_precision_at_1_max": 0.189263, + "nauc_precision_at_1_std": -0.008074, + "nauc_precision_at_1_diff1": 0.448049, + "nauc_precision_at_3_max": 0.119824, + "nauc_precision_at_3_std": -0.05034, + "nauc_precision_at_3_diff1": 0.189035, + "nauc_precision_at_5_max": 0.101665, + "nauc_precision_at_5_std": -0.082172, + "nauc_precision_at_5_diff1": 0.12977, + "nauc_precision_at_10_max": 0.080993, + "nauc_precision_at_10_std": -0.039194, + "nauc_precision_at_10_diff1": 0.024271, + "nauc_precision_at_20_max": 0.04703, + "nauc_precision_at_20_std": -0.015348, + "nauc_precision_at_20_diff1": -0.040852, + "nauc_precision_at_100_max": -0.036587, + "nauc_precision_at_100_std": -0.008768, + "nauc_precision_at_100_diff1": -0.147375, + "nauc_precision_at_1000_max": -0.109466, + "nauc_precision_at_1000_std": 0.032314, + "nauc_precision_at_1000_diff1": -0.22857, + "nauc_mrr_at_1_max": 0.189263, + "nauc_mrr_at_1_std": -0.008074, + "nauc_mrr_at_1_diff1": 0.448049, + "nauc_mrr_at_3_max": 0.183241, + "nauc_mrr_at_3_std": -0.02625, + "nauc_mrr_at_3_diff1": 0.412404, + "nauc_mrr_at_5_max": 0.181305, + "nauc_mrr_at_5_std": -0.037563, + "nauc_mrr_at_5_diff1": 0.42029, + "nauc_mrr_at_10_max": 0.186599, + "nauc_mrr_at_10_std": -0.033688, + "nauc_mrr_at_10_diff1": 0.419347, + "nauc_mrr_at_20_max": 0.188669, + "nauc_mrr_at_20_std": -0.031623, + "nauc_mrr_at_20_diff1": 0.420282, + "nauc_mrr_at_100_max": 0.186452, + "nauc_mrr_at_100_std": -0.032401, + "nauc_mrr_at_100_diff1": 0.420213, + "nauc_mrr_at_1000_max": 0.186568, + "nauc_mrr_at_1000_std": -0.032223, + "nauc_mrr_at_1000_diff1": 0.42026, + "hit_rate_at_1": 0.43072, + "hit_rate_at_3": 0.6747, + "hit_rate_at_5": 0.76807, + "hit_rate_at_10": 0.84337, + "hit_rate_at_20": 0.88253, + "hit_rate_at_100": 0.96687, + "hit_rate_at_1000": 0.99398, + "main_score": 0.4927, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.31175, + "ndcg_at_3": 0.32455, + "ndcg_at_5": 0.33146, + "ndcg_at_10": 0.3659, + "ndcg_at_20": 0.40163, + "ndcg_at_100": 0.45899, + "ndcg_at_1000": 0.50662, + "map_at_1": 0.14358, + "map_at_3": 0.21319, + "map_at_5": 0.23827, + "map_at_10": 0.26782, + "map_at_20": 0.2866, + "map_at_100": 0.30251, + "map_at_1000": 0.30645, + "recall_at_1": 0.14358, + "recall_at_3": 0.26212, + "recall_at_5": 0.32044, + "recall_at_10": 0.42468, + "recall_at_20": 0.52419, + "recall_at_100": 0.72148, + "recall_at_1000": 0.95464, + "accuracy": 0.14358, + "precision_at_1": 0.32831, + "precision_at_3": 0.24398, + "precision_at_5": 0.19036, + "precision_at_10": 0.13946, + "precision_at_20": 0.09413, + "precision_at_100": 0.02949, + "precision_at_1000": 0.00442, + "mrr_at_1": 0.328313, + "mrr_at_3": 0.424699, + "mrr_at_5": 0.437199, + "mrr_at_10": 0.453258, + "mrr_at_20": 0.45941, + "mrr_at_100": 0.462215, + "mrr_at_1000": 0.462553, + "nauc_ndcg_at_1_max": 0.212428, + "nauc_ndcg_at_1_std": 0.085288, + "nauc_ndcg_at_1_diff1": 0.349804, + "nauc_ndcg_at_3_max": 0.21994, + "nauc_ndcg_at_3_std": 0.113579, + "nauc_ndcg_at_3_diff1": 0.328593, + "nauc_ndcg_at_5_max": 0.197386, + "nauc_ndcg_at_5_std": 0.084436, + "nauc_ndcg_at_5_diff1": 0.313198, + "nauc_ndcg_at_10_max": 0.2191, + "nauc_ndcg_at_10_std": 0.099403, + "nauc_ndcg_at_10_diff1": 0.309748, + "nauc_ndcg_at_20_max": 0.23807, + "nauc_ndcg_at_20_std": 0.113879, + "nauc_ndcg_at_20_diff1": 0.318853, + "nauc_ndcg_at_100_max": 0.24935, + "nauc_ndcg_at_100_std": 0.131888, + "nauc_ndcg_at_100_diff1": 0.314865, + "nauc_ndcg_at_1000_max": 0.255625, + "nauc_ndcg_at_1000_std": 0.139451, + "nauc_ndcg_at_1000_diff1": 0.308478, + "nauc_map_at_1_max": 0.193482, + "nauc_map_at_1_std": 0.114918, + "nauc_map_at_1_diff1": 0.32973, + "nauc_map_at_3_max": 0.21165, + "nauc_map_at_3_std": 0.109984, + "nauc_map_at_3_diff1": 0.313342, + "nauc_map_at_5_max": 0.188907, + "nauc_map_at_5_std": 0.078169, + "nauc_map_at_5_diff1": 0.302107, + "nauc_map_at_10_max": 0.203179, + "nauc_map_at_10_std": 0.089106, + "nauc_map_at_10_diff1": 0.300686, + "nauc_map_at_20_max": 0.216583, + "nauc_map_at_20_std": 0.102107, + "nauc_map_at_20_diff1": 0.306618, + "nauc_map_at_100_max": 0.223829, + "nauc_map_at_100_std": 0.111624, + "nauc_map_at_100_diff1": 0.305343, + "nauc_map_at_1000_max": 0.22584, + "nauc_map_at_1000_std": 0.113687, + "nauc_map_at_1000_diff1": 0.304598, + "nauc_recall_at_1_max": 0.193482, + "nauc_recall_at_1_std": 0.114918, + "nauc_recall_at_1_diff1": 0.32973, + "nauc_recall_at_3_max": 0.200557, + "nauc_recall_at_3_std": 0.0969, + "nauc_recall_at_3_diff1": 0.299481, + "nauc_recall_at_5_max": 0.158091, + "nauc_recall_at_5_std": 0.048136, + "nauc_recall_at_5_diff1": 0.263495, + "nauc_recall_at_10_max": 0.188296, + "nauc_recall_at_10_std": 0.087841, + "nauc_recall_at_10_diff1": 0.24722, + "nauc_recall_at_20_max": 0.215425, + "nauc_recall_at_20_std": 0.10503, + "nauc_recall_at_20_diff1": 0.263494, + "nauc_recall_at_100_max": 0.230393, + "nauc_recall_at_100_std": 0.157721, + "nauc_recall_at_100_diff1": 0.259307, + "nauc_recall_at_1000_max": 0.342147, + "nauc_recall_at_1000_std": 0.463191, + "nauc_recall_at_1000_diff1": 0.326435, + "nauc_precision_at_1_max": 0.206225, + "nauc_precision_at_1_std": 0.079659, + "nauc_precision_at_1_diff1": 0.340005, + "nauc_precision_at_3_max": 0.188362, + "nauc_precision_at_3_std": 0.074283, + "nauc_precision_at_3_diff1": 0.248041, + "nauc_precision_at_5_max": 0.142943, + "nauc_precision_at_5_std": 0.028641, + "nauc_precision_at_5_diff1": 0.19666, + "nauc_precision_at_10_max": 0.174166, + "nauc_precision_at_10_std": 0.075896, + "nauc_precision_at_10_diff1": 0.12865, + "nauc_precision_at_20_max": 0.208973, + "nauc_precision_at_20_std": 0.135778, + "nauc_precision_at_20_diff1": 0.106984, + "nauc_precision_at_100_max": 0.156929, + "nauc_precision_at_100_std": 0.132253, + "nauc_precision_at_100_diff1": -0.005037, + "nauc_precision_at_1000_max": 0.066531, + "nauc_precision_at_1000_std": 0.093499, + "nauc_precision_at_1000_diff1": -0.13804, + "nauc_mrr_at_1_max": 0.206225, + "nauc_mrr_at_1_std": 0.079659, + "nauc_mrr_at_1_diff1": 0.340005, + "nauc_mrr_at_3_max": 0.221865, + "nauc_mrr_at_3_std": 0.094582, + "nauc_mrr_at_3_diff1": 0.343531, + "nauc_mrr_at_5_max": 0.231499, + "nauc_mrr_at_5_std": 0.102958, + "nauc_mrr_at_5_diff1": 0.342199, + "nauc_mrr_at_10_max": 0.240525, + "nauc_mrr_at_10_std": 0.110737, + "nauc_mrr_at_10_diff1": 0.345132, + "nauc_mrr_at_20_max": 0.237482, + "nauc_mrr_at_20_std": 0.108378, + "nauc_mrr_at_20_diff1": 0.344748, + "nauc_mrr_at_100_max": 0.235321, + "nauc_mrr_at_100_std": 0.106919, + "nauc_mrr_at_100_diff1": 0.344194, + "nauc_mrr_at_1000_max": 0.235156, + "nauc_mrr_at_1000_std": 0.106853, + "nauc_mrr_at_1000_diff1": 0.344089, + "hit_rate_at_1": 0.32831, + "hit_rate_at_3": 0.54518, + "hit_rate_at_5": 0.5994, + "hit_rate_at_10": 0.71386, + "hit_rate_at_20": 0.79819, + "hit_rate_at_100": 0.91566, + "hit_rate_at_1000": 0.98795, + "main_score": 0.3659, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.36747, + "ndcg_at_3": 0.35599, + "ndcg_at_5": 0.37023, + "ndcg_at_10": 0.39876, + "ndcg_at_20": 0.43794, + "ndcg_at_100": 0.49368, + "ndcg_at_1000": 0.53609, + "map_at_1": 0.16756, + "map_at_3": 0.23633, + "map_at_5": 0.26592, + "map_at_10": 0.2965, + "map_at_20": 0.31664, + "map_at_100": 0.3322, + "map_at_1000": 0.33592, + "recall_at_1": 0.16756, + "recall_at_3": 0.28075, + "recall_at_5": 0.35497, + "recall_at_10": 0.45713, + "recall_at_20": 0.56794, + "recall_at_100": 0.75781, + "recall_at_1000": 0.96001, + "accuracy": 0.16756, + "precision_at_1": 0.38554, + "precision_at_3": 0.25502, + "precision_at_5": 0.21084, + "precision_at_10": 0.1491, + "precision_at_20": 0.10045, + "precision_at_100": 0.03093, + "precision_at_1000": 0.00443, + "mrr_at_1": 0.385542, + "mrr_at_3": 0.467871, + "mrr_at_5": 0.487902, + "mrr_at_10": 0.498206, + "mrr_at_20": 0.504188, + "mrr_at_100": 0.506571, + "mrr_at_1000": 0.50696, + "nauc_ndcg_at_1_max": 0.193644, + "nauc_ndcg_at_1_std": 0.026284, + "nauc_ndcg_at_1_diff1": 0.394654, + "nauc_ndcg_at_3_max": 0.173334, + "nauc_ndcg_at_3_std": 0.015804, + "nauc_ndcg_at_3_diff1": 0.31524, + "nauc_ndcg_at_5_max": 0.170496, + "nauc_ndcg_at_5_std": 0.019589, + "nauc_ndcg_at_5_diff1": 0.31853, + "nauc_ndcg_at_10_max": 0.190245, + "nauc_ndcg_at_10_std": 0.035132, + "nauc_ndcg_at_10_diff1": 0.304997, + "nauc_ndcg_at_20_max": 0.205277, + "nauc_ndcg_at_20_std": 0.052973, + "nauc_ndcg_at_20_diff1": 0.29742, + "nauc_ndcg_at_100_max": 0.212073, + "nauc_ndcg_at_100_std": 0.057644, + "nauc_ndcg_at_100_diff1": 0.311209, + "nauc_ndcg_at_1000_max": 0.210245, + "nauc_ndcg_at_1000_std": 0.061628, + "nauc_ndcg_at_1000_diff1": 0.314438, + "nauc_map_at_1_max": 0.172838, + "nauc_map_at_1_std": 0.071278, + "nauc_map_at_1_diff1": 0.330186, + "nauc_map_at_3_max": 0.171015, + "nauc_map_at_3_std": 0.047931, + "nauc_map_at_3_diff1": 0.299627, + "nauc_map_at_5_max": 0.172881, + "nauc_map_at_5_std": 0.0437, + "nauc_map_at_5_diff1": 0.307274, + "nauc_map_at_10_max": 0.189496, + "nauc_map_at_10_std": 0.050957, + "nauc_map_at_10_diff1": 0.299153, + "nauc_map_at_20_max": 0.198528, + "nauc_map_at_20_std": 0.062285, + "nauc_map_at_20_diff1": 0.294181, + "nauc_map_at_100_max": 0.204969, + "nauc_map_at_100_std": 0.066631, + "nauc_map_at_100_diff1": 0.294717, + "nauc_map_at_1000_max": 0.204823, + "nauc_map_at_1000_std": 0.066892, + "nauc_map_at_1000_diff1": 0.294972, + "nauc_recall_at_1_max": 0.172838, + "nauc_recall_at_1_std": 0.071278, + "nauc_recall_at_1_diff1": 0.330186, + "nauc_recall_at_3_max": 0.144529, + "nauc_recall_at_3_std": 0.018775, + "nauc_recall_at_3_diff1": 0.248753, + "nauc_recall_at_5_max": 0.138672, + "nauc_recall_at_5_std": 0.018185, + "nauc_recall_at_5_diff1": 0.249036, + "nauc_recall_at_10_max": 0.174196, + "nauc_recall_at_10_std": 0.039013, + "nauc_recall_at_10_diff1": 0.213582, + "nauc_recall_at_20_max": 0.19456, + "nauc_recall_at_20_std": 0.064369, + "nauc_recall_at_20_diff1": 0.171328, + "nauc_recall_at_100_max": 0.221461, + "nauc_recall_at_100_std": 0.080982, + "nauc_recall_at_100_diff1": 0.238171, + "nauc_recall_at_1000_max": 0.214584, + "nauc_recall_at_1000_std": 0.208479, + "nauc_recall_at_1000_diff1": 0.363816, + "nauc_precision_at_1_max": 0.202277, + "nauc_precision_at_1_std": 0.02198, + "nauc_precision_at_1_diff1": 0.387758, + "nauc_precision_at_3_max": 0.157049, + "nauc_precision_at_3_std": -0.010394, + "nauc_precision_at_3_diff1": 0.26302, + "nauc_precision_at_5_max": 0.141871, + "nauc_precision_at_5_std": -0.005933, + "nauc_precision_at_5_diff1": 0.215649, + "nauc_precision_at_10_max": 0.170833, + "nauc_precision_at_10_std": 0.032247, + "nauc_precision_at_10_diff1": 0.119388, + "nauc_precision_at_20_max": 0.161903, + "nauc_precision_at_20_std": 0.072839, + "nauc_precision_at_20_diff1": 0.043633, + "nauc_precision_at_100_max": 0.114569, + "nauc_precision_at_100_std": 0.087464, + "nauc_precision_at_100_diff1": -0.02123, + "nauc_precision_at_1000_max": 0.021277, + "nauc_precision_at_1000_std": 0.076727, + "nauc_precision_at_1000_diff1": -0.110719, + "nauc_mrr_at_1_max": 0.202277, + "nauc_mrr_at_1_std": 0.02198, + "nauc_mrr_at_1_diff1": 0.387758, + "nauc_mrr_at_3_max": 0.203517, + "nauc_mrr_at_3_std": 0.013379, + "nauc_mrr_at_3_diff1": 0.381303, + "nauc_mrr_at_5_max": 0.204178, + "nauc_mrr_at_5_std": 0.011997, + "nauc_mrr_at_5_diff1": 0.376283, + "nauc_mrr_at_10_max": 0.214394, + "nauc_mrr_at_10_std": 0.023146, + "nauc_mrr_at_10_diff1": 0.371923, + "nauc_mrr_at_20_max": 0.213881, + "nauc_mrr_at_20_std": 0.024152, + "nauc_mrr_at_20_diff1": 0.371943, + "nauc_mrr_at_100_max": 0.212017, + "nauc_mrr_at_100_std": 0.021997, + "nauc_mrr_at_100_diff1": 0.373122, + "nauc_mrr_at_1000_max": 0.211999, + "nauc_mrr_at_1000_std": 0.02211, + "nauc_mrr_at_1000_diff1": 0.373323, + "hit_rate_at_1": 0.38554, + "hit_rate_at_3": 0.5753, + "hit_rate_at_5": 0.66566, + "hit_rate_at_10": 0.74096, + "hit_rate_at_20": 0.82831, + "hit_rate_at_100": 0.9247, + "hit_rate_at_1000": 0.99096, + "main_score": 0.39876, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.37801, + "ndcg_at_3": 0.37932, + "ndcg_at_5": 0.38866, + "ndcg_at_10": 0.42152, + "ndcg_at_20": 0.45894, + "ndcg_at_100": 0.51456, + "ndcg_at_1000": 0.55397, + "map_at_1": 0.15962, + "map_at_3": 0.244, + "map_at_5": 0.27477, + "map_at_10": 0.30872, + "map_at_20": 0.33074, + "map_at_100": 0.34728, + "map_at_1000": 0.35082, + "recall_at_1": 0.15962, + "recall_at_3": 0.30413, + "recall_at_5": 0.37775, + "recall_at_10": 0.4917, + "recall_at_20": 0.5999, + "recall_at_100": 0.78115, + "recall_at_1000": 0.96973, + "accuracy": 0.15962, + "precision_at_1": 0.39759, + "precision_at_3": 0.27811, + "precision_at_5": 0.22349, + "precision_at_10": 0.15964, + "precision_at_20": 0.10693, + "precision_at_100": 0.03241, + "precision_at_1000": 0.00449, + "mrr_at_1": 0.39759, + "mrr_at_3": 0.490964, + "mrr_at_5": 0.508584, + "mrr_at_10": 0.521435, + "mrr_at_20": 0.526045, + "mrr_at_100": 0.528252, + "mrr_at_1000": 0.528558, + "nauc_ndcg_at_1_max": 0.104867, + "nauc_ndcg_at_1_std": 0.000257, + "nauc_ndcg_at_1_diff1": 0.387671, + "nauc_ndcg_at_3_max": 0.153654, + "nauc_ndcg_at_3_std": 0.03599, + "nauc_ndcg_at_3_diff1": 0.356648, + "nauc_ndcg_at_5_max": 0.158758, + "nauc_ndcg_at_5_std": 0.038919, + "nauc_ndcg_at_5_diff1": 0.373754, + "nauc_ndcg_at_10_max": 0.145437, + "nauc_ndcg_at_10_std": 0.017367, + "nauc_ndcg_at_10_diff1": 0.380998, + "nauc_ndcg_at_20_max": 0.171977, + "nauc_ndcg_at_20_std": 0.045288, + "nauc_ndcg_at_20_diff1": 0.372838, + "nauc_ndcg_at_100_max": 0.193288, + "nauc_ndcg_at_100_std": 0.070092, + "nauc_ndcg_at_100_diff1": 0.386226, + "nauc_ndcg_at_1000_max": 0.186833, + "nauc_ndcg_at_1000_std": 0.065245, + "nauc_ndcg_at_1000_diff1": 0.395172, + "nauc_map_at_1_max": 0.108923, + "nauc_map_at_1_std": 0.035268, + "nauc_map_at_1_diff1": 0.45834, + "nauc_map_at_3_max": 0.145954, + "nauc_map_at_3_std": 0.033186, + "nauc_map_at_3_diff1": 0.399173, + "nauc_map_at_5_max": 0.148227, + "nauc_map_at_5_std": 0.033419, + "nauc_map_at_5_diff1": 0.391753, + "nauc_map_at_10_max": 0.140791, + "nauc_map_at_10_std": 0.019507, + "nauc_map_at_10_diff1": 0.387559, + "nauc_map_at_20_max": 0.158113, + "nauc_map_at_20_std": 0.038023, + "nauc_map_at_20_diff1": 0.382567, + "nauc_map_at_100_max": 0.169668, + "nauc_map_at_100_std": 0.048561, + "nauc_map_at_100_diff1": 0.386361, + "nauc_map_at_1000_max": 0.170232, + "nauc_map_at_1000_std": 0.04953, + "nauc_map_at_1000_diff1": 0.388056, + "nauc_recall_at_1_max": 0.108923, + "nauc_recall_at_1_std": 0.035268, + "nauc_recall_at_1_diff1": 0.45834, + "nauc_recall_at_3_max": 0.164876, + "nauc_recall_at_3_std": 0.044792, + "nauc_recall_at_3_diff1": 0.312378, + "nauc_recall_at_5_max": 0.152799, + "nauc_recall_at_5_std": 0.045508, + "nauc_recall_at_5_diff1": 0.287963, + "nauc_recall_at_10_max": 0.130269, + "nauc_recall_at_10_std": 0.008091, + "nauc_recall_at_10_diff1": 0.263761, + "nauc_recall_at_20_max": 0.178807, + "nauc_recall_at_20_std": 0.060721, + "nauc_recall_at_20_diff1": 0.247314, + "nauc_recall_at_100_max": 0.264243, + "nauc_recall_at_100_std": 0.163722, + "nauc_recall_at_100_diff1": 0.25526, + "nauc_recall_at_1000_max": 0.291625, + "nauc_recall_at_1000_std": 0.127882, + "nauc_recall_at_1000_diff1": 0.33893, + "nauc_precision_at_1_max": 0.113172, + "nauc_precision_at_1_std": -0.000311, + "nauc_precision_at_1_diff1": 0.377148, + "nauc_precision_at_3_max": 0.140793, + "nauc_precision_at_3_std": 0.03432, + "nauc_precision_at_3_diff1": 0.217103, + "nauc_precision_at_5_max": 0.150104, + "nauc_precision_at_5_std": 0.052239, + "nauc_precision_at_5_diff1": 0.17321, + "nauc_precision_at_10_max": 0.103796, + "nauc_precision_at_10_std": 0.018405, + "nauc_precision_at_10_diff1": 0.105478, + "nauc_precision_at_20_max": 0.130604, + "nauc_precision_at_20_std": 0.07498, + "nauc_precision_at_20_diff1": 0.041104, + "nauc_precision_at_100_max": 0.09676, + "nauc_precision_at_100_std": 0.097362, + "nauc_precision_at_100_diff1": -0.035763, + "nauc_precision_at_1000_max": 0.025042, + "nauc_precision_at_1000_std": 0.077688, + "nauc_precision_at_1000_diff1": -0.111734, + "nauc_mrr_at_1_max": 0.113172, + "nauc_mrr_at_1_std": -0.000311, + "nauc_mrr_at_1_diff1": 0.377148, + "nauc_mrr_at_3_max": 0.158558, + "nauc_mrr_at_3_std": 0.023134, + "nauc_mrr_at_3_diff1": 0.36239, + "nauc_mrr_at_5_max": 0.165724, + "nauc_mrr_at_5_std": 0.033082, + "nauc_mrr_at_5_diff1": 0.363309, + "nauc_mrr_at_10_max": 0.166614, + "nauc_mrr_at_10_std": 0.03108, + "nauc_mrr_at_10_diff1": 0.364533, + "nauc_mrr_at_20_max": 0.164925, + "nauc_mrr_at_20_std": 0.029011, + "nauc_mrr_at_20_diff1": 0.367544, + "nauc_mrr_at_100_max": 0.164372, + "nauc_mrr_at_100_std": 0.028573, + "nauc_mrr_at_100_diff1": 0.368087, + "nauc_mrr_at_1000_max": 0.164148, + "nauc_mrr_at_1000_std": 0.028494, + "nauc_mrr_at_1000_diff1": 0.368163, + "hit_rate_at_1": 0.39759, + "hit_rate_at_3": 0.61145, + "hit_rate_at_5": 0.68976, + "hit_rate_at_10": 0.78614, + "hit_rate_at_20": 0.85542, + "hit_rate_at_100": 0.93373, + "hit_rate_at_1000": 0.99398, + "main_score": 0.42152, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.10" + } + ] + }, + "evaluation_time": 1775.8365016169846, + "kg_co2_emissions": null, + "date": 1792351844.314174, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 1.0616029612720013e-05, + "end": 25.142561199958436, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 25.14261385903228, + "end": 25.142623513005674, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries", + "start": 25.168895483016968, + "end": 26.283364980947226, + "split": "test", + "subset": "french" + }, + { + "name": "Searching corpus", + "start": 26.290145509992726, + "end": 287.46217071195133, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 287.52189908805303, + "end": 287.7950119760353, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 287.8350543800043, + "end": 288.3182904389687, + "split": "test", + "subset": "spanish" + }, + { + "name": "Searching corpus", + "start": 288.3213452900527, + "end": 569.1154392729513, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 569.1698817060096, + "end": 569.4570768070407, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 569.5107621720526, + "end": 570.0343357389793, + "split": "test", + "subset": "english" + }, + { + "name": "Searching corpus", + "start": 570.0373416759539, + "end": 881.1456898039905, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 881.2241539449897, + "end": 881.5111507850233, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 881.5501911620377, + "end": 883.1838013259694, + "split": "test", + "subset": "german" + }, + { + "name": "Searching corpus", + "start": 883.1881311630132, + "end": 1183.669485711027, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 1183.733382440987, + "end": 1184.004850495956, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 1184.0458207640331, + "end": 1184.4526937470073, + "split": "test", + "subset": "italian" + }, + { + "name": "Searching corpus", + "start": 1184.4555829579476, + "end": 1491.3901096460177, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 1491.4438862740062, + "end": 1491.7109482580563, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 1491.7483254129766, + "end": 1492.7764558560448, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Searching corpus", + "start": 1492.7860299720196, + "end": 1775.4621318239952, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 1775.5322135270108, + "end": 1775.8189915140392, + "split": "test", + "subset": "portuguese" + } + ] +} \ No newline at end of file diff --git a/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3NuclearRetrieval.v2.json b/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3NuclearRetrieval.v2.json new file mode 100644 index 0000000000..89827bcd46 --- /dev/null +++ b/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3NuclearRetrieval.v2.json @@ -0,0 +1,1090 @@ +{ + "dataset_revision": "826e3bc829e01f5a088d0435114200bce6058af8", + "task_name": "Vidore3NuclearRetrieval.v2", + "mteb_version": "2.26.10", + "scores": { + "test": [ + { + "ndcg_at_1": 0.38102, + "ndcg_at_3": 0.40941, + "ndcg_at_5": 0.41805, + "ndcg_at_10": 0.44033, + "ndcg_at_20": 0.47427, + "ndcg_at_100": 0.52824, + "ndcg_at_1000": 0.56833, + "map_at_1": 0.17245, + "map_at_3": 0.27074, + "map_at_5": 0.30453, + "map_at_10": 0.33312, + "map_at_20": 0.3541, + "map_at_100": 0.37007, + "map_at_1000": 0.37363, + "recall_at_1": 0.17245, + "recall_at_3": 0.33348, + "recall_at_5": 0.40765, + "recall_at_10": 0.49848, + "recall_at_20": 0.5933, + "recall_at_100": 0.77869, + "recall_at_1000": 0.96764, + "accuracy": 0.17245, + "precision_at_1": 0.40361, + "precision_at_3": 0.30321, + "precision_at_5": 0.24337, + "precision_at_10": 0.16476, + "precision_at_20": 0.10798, + "precision_at_100": 0.03169, + "precision_at_1000": 0.00448, + "mrr_at_1": 0.403614, + "mrr_at_3": 0.51506, + "mrr_at_5": 0.529217, + "mrr_at_10": 0.54071, + "mrr_at_20": 0.543939, + "mrr_at_100": 0.546034, + "mrr_at_1000": 0.5464, + "nauc_ndcg_at_1_max": 0.120739, + "nauc_ndcg_at_1_std": -0.018406, + "nauc_ndcg_at_1_diff1": 0.410451, + "nauc_ndcg_at_3_max": 0.142579, + "nauc_ndcg_at_3_std": -0.013904, + "nauc_ndcg_at_3_diff1": 0.332627, + "nauc_ndcg_at_5_max": 0.156473, + "nauc_ndcg_at_5_std": -0.008882, + "nauc_ndcg_at_5_diff1": 0.33763, + "nauc_ndcg_at_10_max": 0.14017, + "nauc_ndcg_at_10_std": -0.038798, + "nauc_ndcg_at_10_diff1": 0.343875, + "nauc_ndcg_at_20_max": 0.139484, + "nauc_ndcg_at_20_std": -0.028283, + "nauc_ndcg_at_20_diff1": 0.345418, + "nauc_ndcg_at_100_max": 0.167803, + "nauc_ndcg_at_100_std": -0.007922, + "nauc_ndcg_at_100_diff1": 0.347636, + "nauc_ndcg_at_1000_max": 0.172132, + "nauc_ndcg_at_1000_std": 0.004906, + "nauc_ndcg_at_1000_diff1": 0.347116, + "nauc_map_at_1_max": 0.173216, + "nauc_map_at_1_std": 0.065688, + "nauc_map_at_1_diff1": 0.4141, + "nauc_map_at_3_max": 0.155093, + "nauc_map_at_3_std": 0.009851, + "nauc_map_at_3_diff1": 0.369489, + "nauc_map_at_5_max": 0.154798, + "nauc_map_at_5_std": 0.002243, + "nauc_map_at_5_diff1": 0.364735, + "nauc_map_at_10_max": 0.150491, + "nauc_map_at_10_std": -0.013324, + "nauc_map_at_10_diff1": 0.363262, + "nauc_map_at_20_max": 0.152735, + "nauc_map_at_20_std": -0.005156, + "nauc_map_at_20_diff1": 0.36239, + "nauc_map_at_100_max": 0.163901, + "nauc_map_at_100_std": 0.0033, + "nauc_map_at_100_diff1": 0.361472, + "nauc_map_at_1000_max": 0.164642, + "nauc_map_at_1000_std": 0.004791, + "nauc_map_at_1000_diff1": 0.36152, + "nauc_recall_at_1_max": 0.173216, + "nauc_recall_at_1_std": 0.065688, + "nauc_recall_at_1_diff1": 0.4141, + "nauc_recall_at_3_max": 0.124659, + "nauc_recall_at_3_std": -0.022536, + "nauc_recall_at_3_diff1": 0.280328, + "nauc_recall_at_5_max": 0.13223, + "nauc_recall_at_5_std": -0.028496, + "nauc_recall_at_5_diff1": 0.273577, + "nauc_recall_at_10_max": 0.089686, + "nauc_recall_at_10_std": -0.091292, + "nauc_recall_at_10_diff1": 0.254094, + "nauc_recall_at_20_max": 0.075624, + "nauc_recall_at_20_std": -0.079861, + "nauc_recall_at_20_diff1": 0.248909, + "nauc_recall_at_100_max": 0.151845, + "nauc_recall_at_100_std": -0.052888, + "nauc_recall_at_100_diff1": 0.252381, + "nauc_recall_at_1000_max": 0.26634, + "nauc_recall_at_1000_std": -0.018816, + "nauc_recall_at_1000_diff1": 0.312907, + "nauc_precision_at_1_max": 0.122183, + "nauc_precision_at_1_std": -0.020137, + "nauc_precision_at_1_diff1": 0.404475, + "nauc_precision_at_3_max": 0.109356, + "nauc_precision_at_3_std": -0.033239, + "nauc_precision_at_3_diff1": 0.231876, + "nauc_precision_at_5_max": 0.099127, + "nauc_precision_at_5_std": -0.025449, + "nauc_precision_at_5_diff1": 0.171882, + "nauc_precision_at_10_max": 0.06705, + "nauc_precision_at_10_std": -0.049148, + "nauc_precision_at_10_diff1": 0.087496, + "nauc_precision_at_20_max": 0.054173, + "nauc_precision_at_20_std": -0.002863, + "nauc_precision_at_20_diff1": 0.033617, + "nauc_precision_at_100_max": 0.081148, + "nauc_precision_at_100_std": 0.065278, + "nauc_precision_at_100_diff1": -0.057009, + "nauc_precision_at_1000_max": 0.03839, + "nauc_precision_at_1000_std": 0.112822, + "nauc_precision_at_1000_diff1": -0.15769, + "nauc_mrr_at_1_max": 0.122183, + "nauc_mrr_at_1_std": -0.020137, + "nauc_mrr_at_1_diff1": 0.404475, + "nauc_mrr_at_3_max": 0.144436, + "nauc_mrr_at_3_std": -0.023983, + "nauc_mrr_at_3_diff1": 0.353947, + "nauc_mrr_at_5_max": 0.152062, + "nauc_mrr_at_5_std": -0.019815, + "nauc_mrr_at_5_diff1": 0.358145, + "nauc_mrr_at_10_max": 0.15502, + "nauc_mrr_at_10_std": -0.018657, + "nauc_mrr_at_10_diff1": 0.361541, + "nauc_mrr_at_20_max": 0.151936, + "nauc_mrr_at_20_std": -0.020671, + "nauc_mrr_at_20_diff1": 0.361642, + "nauc_mrr_at_100_max": 0.151915, + "nauc_mrr_at_100_std": -0.02127, + "nauc_mrr_at_100_diff1": 0.36168, + "nauc_mrr_at_1000_max": 0.151588, + "nauc_mrr_at_1000_std": -0.021609, + "nauc_mrr_at_1000_diff1": 0.361824, + "hit_rate_at_1": 0.40361, + "hit_rate_at_3": 0.65361, + "hit_rate_at_5": 0.71386, + "hit_rate_at_10": 0.79518, + "hit_rate_at_20": 0.84337, + "hit_rate_at_100": 0.92169, + "hit_rate_at_1000": 0.99398, + "main_score": 0.44033, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.40964, + "ndcg_at_3": 0.41044, + "ndcg_at_5": 0.42553, + "ndcg_at_10": 0.45401, + "ndcg_at_20": 0.48709, + "ndcg_at_100": 0.53946, + "ndcg_at_1000": 0.57983, + "map_at_1": 0.20207, + "map_at_3": 0.28452, + "map_at_5": 0.31788, + "map_at_10": 0.35246, + "map_at_20": 0.37221, + "map_at_100": 0.38827, + "map_at_1000": 0.39186, + "recall_at_1": 0.20207, + "recall_at_3": 0.32428, + "recall_at_5": 0.40219, + "recall_at_10": 0.50661, + "recall_at_20": 0.60319, + "recall_at_100": 0.78147, + "recall_at_1000": 0.96982, + "accuracy": 0.20207, + "precision_at_1": 0.42771, + "precision_at_3": 0.29618, + "precision_at_5": 0.24398, + "precision_at_10": 0.17139, + "precision_at_20": 0.11009, + "precision_at_100": 0.03208, + "precision_at_1000": 0.00451, + "mrr_at_1": 0.427711, + "mrr_at_3": 0.510542, + "mrr_at_5": 0.534036, + "mrr_at_10": 0.546352, + "mrr_at_20": 0.550364, + "mrr_at_100": 0.552511, + "mrr_at_1000": 0.552848, + "nauc_ndcg_at_1_max": 0.229107, + "nauc_ndcg_at_1_std": 0.052462, + "nauc_ndcg_at_1_diff1": 0.373764, + "nauc_ndcg_at_3_max": 0.189631, + "nauc_ndcg_at_3_std": 0.011306, + "nauc_ndcg_at_3_diff1": 0.300121, + "nauc_ndcg_at_5_max": 0.193246, + "nauc_ndcg_at_5_std": -0.002264, + "nauc_ndcg_at_5_diff1": 0.292581, + "nauc_ndcg_at_10_max": 0.205821, + "nauc_ndcg_at_10_std": 0.002952, + "nauc_ndcg_at_10_diff1": 0.310255, + "nauc_ndcg_at_20_max": 0.216354, + "nauc_ndcg_at_20_std": 0.014088, + "nauc_ndcg_at_20_diff1": 0.321107, + "nauc_ndcg_at_100_max": 0.22515, + "nauc_ndcg_at_100_std": 0.026198, + "nauc_ndcg_at_100_diff1": 0.318819, + "nauc_ndcg_at_1000_max": 0.223389, + "nauc_ndcg_at_1000_std": 0.028356, + "nauc_ndcg_at_1000_diff1": 0.321813, + "nauc_map_at_1_max": 0.190505, + "nauc_map_at_1_std": 0.06887, + "nauc_map_at_1_diff1": 0.291918, + "nauc_map_at_3_max": 0.185489, + "nauc_map_at_3_std": 0.025156, + "nauc_map_at_3_diff1": 0.290079, + "nauc_map_at_5_max": 0.181966, + "nauc_map_at_5_std": 0.005855, + "nauc_map_at_5_diff1": 0.282854, + "nauc_map_at_10_max": 0.192606, + "nauc_map_at_10_std": 0.008813, + "nauc_map_at_10_diff1": 0.292315, + "nauc_map_at_20_max": 0.199215, + "nauc_map_at_20_std": 0.015212, + "nauc_map_at_20_diff1": 0.294722, + "nauc_map_at_100_max": 0.202309, + "nauc_map_at_100_std": 0.018136, + "nauc_map_at_100_diff1": 0.293789, + "nauc_map_at_1000_max": 0.202679, + "nauc_map_at_1000_std": 0.01895, + "nauc_map_at_1000_diff1": 0.293997, + "nauc_recall_at_1_max": 0.190505, + "nauc_recall_at_1_std": 0.06887, + "nauc_recall_at_1_diff1": 0.291918, + "nauc_recall_at_3_max": 0.17233, + "nauc_recall_at_3_std": -0.002287, + "nauc_recall_at_3_diff1": 0.258056, + "nauc_recall_at_5_max": 0.168445, + "nauc_recall_at_5_std": -0.025944, + "nauc_recall_at_5_diff1": 0.227474, + "nauc_recall_at_10_max": 0.196614, + "nauc_recall_at_10_std": -0.008724, + "nauc_recall_at_10_diff1": 0.255486, + "nauc_recall_at_20_max": 0.21024, + "nauc_recall_at_20_std": 0.005219, + "nauc_recall_at_20_diff1": 0.274025, + "nauc_recall_at_100_max": 0.23268, + "nauc_recall_at_100_std": 0.043701, + "nauc_recall_at_100_diff1": 0.229339, + "nauc_recall_at_1000_max": 0.226639, + "nauc_recall_at_1000_std": -0.046322, + "nauc_recall_at_1000_diff1": 0.360192, + "nauc_precision_at_1_max": 0.225975, + "nauc_precision_at_1_std": 0.042312, + "nauc_precision_at_1_diff1": 0.37266, + "nauc_precision_at_3_max": 0.131921, + "nauc_precision_at_3_std": -0.025942, + "nauc_precision_at_3_diff1": 0.2145, + "nauc_precision_at_5_max": 0.101972, + "nauc_precision_at_5_std": -0.057195, + "nauc_precision_at_5_diff1": 0.145108, + "nauc_precision_at_10_max": 0.096747, + "nauc_precision_at_10_std": -0.012969, + "nauc_precision_at_10_diff1": 0.095914, + "nauc_precision_at_20_max": 0.085955, + "nauc_precision_at_20_std": 0.014843, + "nauc_precision_at_20_diff1": 0.052957, + "nauc_precision_at_100_max": 0.047885, + "nauc_precision_at_100_std": 0.051687, + "nauc_precision_at_100_diff1": -0.02741, + "nauc_precision_at_1000_max": -0.01325, + "nauc_precision_at_1000_std": 0.077307, + "nauc_precision_at_1000_diff1": -0.097151, + "nauc_mrr_at_1_max": 0.225975, + "nauc_mrr_at_1_std": 0.042312, + "nauc_mrr_at_1_diff1": 0.37266, + "nauc_mrr_at_3_max": 0.222315, + "nauc_mrr_at_3_std": 0.02028, + "nauc_mrr_at_3_diff1": 0.342257, + "nauc_mrr_at_5_max": 0.234129, + "nauc_mrr_at_5_std": 0.031387, + "nauc_mrr_at_5_diff1": 0.329456, + "nauc_mrr_at_10_max": 0.243334, + "nauc_mrr_at_10_std": 0.038965, + "nauc_mrr_at_10_diff1": 0.34075, + "nauc_mrr_at_20_max": 0.240443, + "nauc_mrr_at_20_std": 0.03533, + "nauc_mrr_at_20_diff1": 0.343998, + "nauc_mrr_at_100_max": 0.239026, + "nauc_mrr_at_100_std": 0.034224, + "nauc_mrr_at_100_diff1": 0.343182, + "nauc_mrr_at_1000_max": 0.238948, + "nauc_mrr_at_1000_std": 0.034196, + "nauc_mrr_at_1000_diff1": 0.343165, + "hit_rate_at_1": 0.42771, + "hit_rate_at_3": 0.60843, + "hit_rate_at_5": 0.71084, + "hit_rate_at_10": 0.79819, + "hit_rate_at_20": 0.85542, + "hit_rate_at_100": 0.92771, + "hit_rate_at_1000": 0.99398, + "main_score": 0.45401, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.47289, + "ndcg_at_3": 0.46375, + "ndcg_at_5": 0.47734, + "ndcg_at_10": 0.50808, + "ndcg_at_20": 0.54492, + "ndcg_at_100": 0.59368, + "ndcg_at_1000": 0.62799, + "map_at_1": 0.21993, + "map_at_3": 0.31597, + "map_at_5": 0.35848, + "map_at_10": 0.39461, + "map_at_20": 0.41915, + "map_at_100": 0.43656, + "map_at_1000": 0.43989, + "recall_at_1": 0.21993, + "recall_at_3": 0.36765, + "recall_at_5": 0.45841, + "recall_at_10": 0.57207, + "recall_at_20": 0.67043, + "recall_at_100": 0.82994, + "recall_at_1000": 0.98433, + "accuracy": 0.21993, + "precision_at_1": 0.5, + "precision_at_3": 0.33534, + "precision_at_5": 0.27108, + "precision_at_10": 0.18886, + "precision_at_20": 0.1244, + "precision_at_100": 0.03467, + "precision_at_1000": 0.0046, + "mrr_at_1": 0.5, + "mrr_at_3": 0.59488, + "mrr_at_5": 0.612349, + "mrr_at_10": 0.623054, + "mrr_at_20": 0.625343, + "mrr_at_100": 0.626772, + "mrr_at_1000": 0.626887, + "nauc_ndcg_at_1_max": 0.15961, + "nauc_ndcg_at_1_std": -0.050216, + "nauc_ndcg_at_1_diff1": 0.314407, + "nauc_ndcg_at_3_max": 0.141005, + "nauc_ndcg_at_3_std": -0.04361, + "nauc_ndcg_at_3_diff1": 0.24038, + "nauc_ndcg_at_5_max": 0.165819, + "nauc_ndcg_at_5_std": -0.056904, + "nauc_ndcg_at_5_diff1": 0.281947, + "nauc_ndcg_at_10_max": 0.176413, + "nauc_ndcg_at_10_std": -0.06714, + "nauc_ndcg_at_10_diff1": 0.277198, + "nauc_ndcg_at_20_max": 0.186305, + "nauc_ndcg_at_20_std": -0.057347, + "nauc_ndcg_at_20_diff1": 0.287013, + "nauc_ndcg_at_100_max": 0.187655, + "nauc_ndcg_at_100_std": -0.053501, + "nauc_ndcg_at_100_diff1": 0.289813, + "nauc_ndcg_at_1000_max": 0.17327, + "nauc_ndcg_at_1000_std": -0.04681, + "nauc_ndcg_at_1000_diff1": 0.284603, + "nauc_map_at_1_max": 0.173132, + "nauc_map_at_1_std": 0.025988, + "nauc_map_at_1_diff1": 0.308586, + "nauc_map_at_3_max": 0.147582, + "nauc_map_at_3_std": -0.013263, + "nauc_map_at_3_diff1": 0.27444, + "nauc_map_at_5_max": 0.166363, + "nauc_map_at_5_std": -0.026575, + "nauc_map_at_5_diff1": 0.294576, + "nauc_map_at_10_max": 0.174015, + "nauc_map_at_10_std": -0.035015, + "nauc_map_at_10_diff1": 0.285111, + "nauc_map_at_20_max": 0.175173, + "nauc_map_at_20_std": -0.036154, + "nauc_map_at_20_diff1": 0.285707, + "nauc_map_at_100_max": 0.177634, + "nauc_map_at_100_std": -0.035078, + "nauc_map_at_100_diff1": 0.285917, + "nauc_map_at_1000_max": 0.176547, + "nauc_map_at_1000_std": -0.033936, + "nauc_map_at_1000_diff1": 0.285724, + "nauc_recall_at_1_max": 0.173132, + "nauc_recall_at_1_std": 0.025988, + "nauc_recall_at_1_diff1": 0.308586, + "nauc_recall_at_3_max": 0.131309, + "nauc_recall_at_3_std": -0.018084, + "nauc_recall_at_3_diff1": 0.21758, + "nauc_recall_at_5_max": 0.172411, + "nauc_recall_at_5_std": -0.039155, + "nauc_recall_at_5_diff1": 0.250275, + "nauc_recall_at_10_max": 0.190681, + "nauc_recall_at_10_std": -0.071111, + "nauc_recall_at_10_diff1": 0.23256, + "nauc_recall_at_20_max": 0.206911, + "nauc_recall_at_20_std": -0.070193, + "nauc_recall_at_20_diff1": 0.241164, + "nauc_recall_at_100_max": 0.252167, + "nauc_recall_at_100_std": -0.058318, + "nauc_recall_at_100_diff1": 0.270878, + "nauc_recall_at_1000_max": 0.153928, + "nauc_recall_at_1000_std": 0.019682, + "nauc_recall_at_1000_diff1": 0.31046, + "nauc_precision_at_1_max": 0.14391, + "nauc_precision_at_1_std": -0.069954, + "nauc_precision_at_1_diff1": 0.30581, + "nauc_precision_at_3_max": 0.07502, + "nauc_precision_at_3_std": -0.088971, + "nauc_precision_at_3_diff1": 0.146187, + "nauc_precision_at_5_max": 0.070866, + "nauc_precision_at_5_std": -0.112334, + "nauc_precision_at_5_diff1": 0.136503, + "nauc_precision_at_10_max": 0.054615, + "nauc_precision_at_10_std": -0.099206, + "nauc_precision_at_10_diff1": 0.044847, + "nauc_precision_at_20_max": 0.008609, + "nauc_precision_at_20_std": -0.063184, + "nauc_precision_at_20_diff1": -0.007004, + "nauc_precision_at_100_max": -0.040166, + "nauc_precision_at_100_std": -0.042782, + "nauc_precision_at_100_diff1": -0.071667, + "nauc_precision_at_1000_max": -0.109806, + "nauc_precision_at_1000_std": 0.025644, + "nauc_precision_at_1000_diff1": -0.149348, + "nauc_mrr_at_1_max": 0.14391, + "nauc_mrr_at_1_std": -0.069954, + "nauc_mrr_at_1_diff1": 0.30581, + "nauc_mrr_at_3_max": 0.140043, + "nauc_mrr_at_3_std": -0.082608, + "nauc_mrr_at_3_diff1": 0.290943, + "nauc_mrr_at_5_max": 0.141182, + "nauc_mrr_at_5_std": -0.089246, + "nauc_mrr_at_5_diff1": 0.282794, + "nauc_mrr_at_10_max": 0.150897, + "nauc_mrr_at_10_std": -0.08605, + "nauc_mrr_at_10_diff1": 0.28846, + "nauc_mrr_at_20_max": 0.150606, + "nauc_mrr_at_20_std": -0.084366, + "nauc_mrr_at_20_diff1": 0.287318, + "nauc_mrr_at_100_max": 0.14927, + "nauc_mrr_at_100_std": -0.084743, + "nauc_mrr_at_100_diff1": 0.287352, + "nauc_mrr_at_1000_max": 0.149336, + "nauc_mrr_at_1000_std": -0.084615, + "nauc_mrr_at_1000_diff1": 0.287447, + "hit_rate_at_1": 0.5, + "hit_rate_at_3": 0.71084, + "hit_rate_at_5": 0.78916, + "hit_rate_at_10": 0.86747, + "hit_rate_at_20": 0.90361, + "hit_rate_at_100": 0.96687, + "hit_rate_at_1000": 0.99699, + "main_score": 0.50808, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.3238, + "ndcg_at_3": 0.33348, + "ndcg_at_5": 0.34272, + "ndcg_at_10": 0.37327, + "ndcg_at_20": 0.40714, + "ndcg_at_100": 0.4613, + "ndcg_at_1000": 0.51089, + "map_at_1": 0.14828, + "map_at_3": 0.22287, + "map_at_5": 0.24823, + "map_at_10": 0.27969, + "map_at_20": 0.29703, + "map_at_100": 0.31156, + "map_at_1000": 0.3157, + "recall_at_1": 0.14828, + "recall_at_3": 0.2714, + "recall_at_5": 0.33367, + "recall_at_10": 0.43363, + "recall_at_20": 0.52723, + "recall_at_100": 0.71214, + "recall_at_1000": 0.95543, + "accuracy": 0.14828, + "precision_at_1": 0.34337, + "precision_at_3": 0.24699, + "precision_at_5": 0.19819, + "precision_at_10": 0.14398, + "precision_at_20": 0.09443, + "precision_at_100": 0.0291, + "precision_at_1000": 0.00443, + "mrr_at_1": 0.343373, + "mrr_at_3": 0.429217, + "mrr_at_5": 0.450753, + "mrr_at_10": 0.462441, + "mrr_at_20": 0.468496, + "mrr_at_100": 0.470844, + "mrr_at_1000": 0.471264, + "nauc_ndcg_at_1_max": 0.167475, + "nauc_ndcg_at_1_std": 0.05322, + "nauc_ndcg_at_1_diff1": 0.388, + "nauc_ndcg_at_3_max": 0.188916, + "nauc_ndcg_at_3_std": 0.046407, + "nauc_ndcg_at_3_diff1": 0.321693, + "nauc_ndcg_at_5_max": 0.176743, + "nauc_ndcg_at_5_std": 0.038478, + "nauc_ndcg_at_5_diff1": 0.313993, + "nauc_ndcg_at_10_max": 0.180703, + "nauc_ndcg_at_10_std": 0.031595, + "nauc_ndcg_at_10_diff1": 0.301337, + "nauc_ndcg_at_20_max": 0.200302, + "nauc_ndcg_at_20_std": 0.044141, + "nauc_ndcg_at_20_diff1": 0.307521, + "nauc_ndcg_at_100_max": 0.214311, + "nauc_ndcg_at_100_std": 0.057562, + "nauc_ndcg_at_100_diff1": 0.32265, + "nauc_ndcg_at_1000_max": 0.221449, + "nauc_ndcg_at_1000_std": 0.068592, + "nauc_ndcg_at_1000_diff1": 0.33279, + "nauc_map_at_1_max": 0.178438, + "nauc_map_at_1_std": 0.086927, + "nauc_map_at_1_diff1": 0.302263, + "nauc_map_at_3_max": 0.179288, + "nauc_map_at_3_std": 0.048039, + "nauc_map_at_3_diff1": 0.292644, + "nauc_map_at_5_max": 0.169578, + "nauc_map_at_5_std": 0.040649, + "nauc_map_at_5_diff1": 0.288726, + "nauc_map_at_10_max": 0.170839, + "nauc_map_at_10_std": 0.034647, + "nauc_map_at_10_diff1": 0.29076, + "nauc_map_at_20_max": 0.182164, + "nauc_map_at_20_std": 0.042653, + "nauc_map_at_20_diff1": 0.292381, + "nauc_map_at_100_max": 0.191887, + "nauc_map_at_100_std": 0.051012, + "nauc_map_at_100_diff1": 0.299343, + "nauc_map_at_1000_max": 0.193198, + "nauc_map_at_1000_std": 0.052267, + "nauc_map_at_1000_diff1": 0.30044, + "nauc_recall_at_1_max": 0.178438, + "nauc_recall_at_1_std": 0.086927, + "nauc_recall_at_1_diff1": 0.302263, + "nauc_recall_at_3_max": 0.158252, + "nauc_recall_at_3_std": 0.019462, + "nauc_recall_at_3_diff1": 0.252046, + "nauc_recall_at_5_max": 0.141019, + "nauc_recall_at_5_std": 0.013978, + "nauc_recall_at_5_diff1": 0.221547, + "nauc_recall_at_10_max": 0.133774, + "nauc_recall_at_10_std": -0.005741, + "nauc_recall_at_10_diff1": 0.200761, + "nauc_recall_at_20_max": 0.165919, + "nauc_recall_at_20_std": 0.014123, + "nauc_recall_at_20_diff1": 0.202679, + "nauc_recall_at_100_max": 0.189199, + "nauc_recall_at_100_std": 0.038644, + "nauc_recall_at_100_diff1": 0.227285, + "nauc_recall_at_1000_max": 0.369674, + "nauc_recall_at_1000_std": 0.228275, + "nauc_recall_at_1000_diff1": 0.349995, + "nauc_precision_at_1_max": 0.150963, + "nauc_precision_at_1_std": 0.041169, + "nauc_precision_at_1_diff1": 0.387782, + "nauc_precision_at_3_max": 0.16214, + "nauc_precision_at_3_std": 0.008256, + "nauc_precision_at_3_diff1": 0.266445, + "nauc_precision_at_5_max": 0.134413, + "nauc_precision_at_5_std": 0.008916, + "nauc_precision_at_5_diff1": 0.237066, + "nauc_precision_at_10_max": 0.119562, + "nauc_precision_at_10_std": 0.013847, + "nauc_precision_at_10_diff1": 0.18276, + "nauc_precision_at_20_max": 0.15399, + "nauc_precision_at_20_std": 0.055759, + "nauc_precision_at_20_diff1": 0.153922, + "nauc_precision_at_100_max": 0.122332, + "nauc_precision_at_100_std": 0.078851, + "nauc_precision_at_100_diff1": 0.076571, + "nauc_precision_at_1000_max": 0.047564, + "nauc_precision_at_1000_std": 0.087986, + "nauc_precision_at_1000_diff1": -0.034529, + "nauc_mrr_at_1_max": 0.150963, + "nauc_mrr_at_1_std": 0.041169, + "nauc_mrr_at_1_diff1": 0.387782, + "nauc_mrr_at_3_max": 0.191765, + "nauc_mrr_at_3_std": 0.044021, + "nauc_mrr_at_3_diff1": 0.365056, + "nauc_mrr_at_5_max": 0.193177, + "nauc_mrr_at_5_std": 0.03972, + "nauc_mrr_at_5_diff1": 0.364855, + "nauc_mrr_at_10_max": 0.19729, + "nauc_mrr_at_10_std": 0.04323, + "nauc_mrr_at_10_diff1": 0.364162, + "nauc_mrr_at_20_max": 0.197468, + "nauc_mrr_at_20_std": 0.044899, + "nauc_mrr_at_20_diff1": 0.367787, + "nauc_mrr_at_100_max": 0.195588, + "nauc_mrr_at_100_std": 0.043636, + "nauc_mrr_at_100_diff1": 0.367637, + "nauc_mrr_at_1000_max": 0.19535, + "nauc_mrr_at_1000_std": 0.043602, + "nauc_mrr_at_1000_diff1": 0.3676, + "hit_rate_at_1": 0.34337, + "hit_rate_at_3": 0.53916, + "hit_rate_at_5": 0.63253, + "hit_rate_at_10": 0.71988, + "hit_rate_at_20": 0.80723, + "hit_rate_at_100": 0.90361, + "hit_rate_at_1000": 0.99096, + "main_score": 0.37327, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.37801, + "ndcg_at_3": 0.37546, + "ndcg_at_5": 0.38409, + "ndcg_at_10": 0.41341, + "ndcg_at_20": 0.45262, + "ndcg_at_100": 0.50815, + "ndcg_at_1000": 0.54897, + "map_at_1": 0.17888, + "map_at_3": 0.2544, + "map_at_5": 0.28458, + "map_at_10": 0.31552, + "map_at_20": 0.33725, + "map_at_100": 0.35342, + "map_at_1000": 0.35683, + "recall_at_1": 0.17888, + "recall_at_3": 0.29889, + "recall_at_5": 0.36641, + "recall_at_10": 0.46893, + "recall_at_20": 0.57048, + "recall_at_100": 0.76283, + "recall_at_1000": 0.96162, + "accuracy": 0.17888, + "precision_at_1": 0.39759, + "precision_at_3": 0.26807, + "precision_at_5": 0.21506, + "precision_at_10": 0.15331, + "precision_at_20": 0.10497, + "precision_at_100": 0.03145, + "precision_at_1000": 0.00442, + "mrr_at_1": 0.39759, + "mrr_at_3": 0.48494, + "mrr_at_5": 0.501205, + "mrr_at_10": 0.511286, + "mrr_at_20": 0.516055, + "mrr_at_100": 0.518757, + "mrr_at_1000": 0.519118, + "nauc_ndcg_at_1_max": 0.171538, + "nauc_ndcg_at_1_std": 0.027087, + "nauc_ndcg_at_1_diff1": 0.350215, + "nauc_ndcg_at_3_max": 0.143537, + "nauc_ndcg_at_3_std": 0.002888, + "nauc_ndcg_at_3_diff1": 0.311835, + "nauc_ndcg_at_5_max": 0.170525, + "nauc_ndcg_at_5_std": 0.027977, + "nauc_ndcg_at_5_diff1": 0.312368, + "nauc_ndcg_at_10_max": 0.159277, + "nauc_ndcg_at_10_std": 0.010111, + "nauc_ndcg_at_10_diff1": 0.305009, + "nauc_ndcg_at_20_max": 0.166636, + "nauc_ndcg_at_20_std": 0.024127, + "nauc_ndcg_at_20_diff1": 0.315495, + "nauc_ndcg_at_100_max": 0.171903, + "nauc_ndcg_at_100_std": 0.017381, + "nauc_ndcg_at_100_diff1": 0.309177, + "nauc_ndcg_at_1000_max": 0.177369, + "nauc_ndcg_at_1000_std": 0.02991, + "nauc_ndcg_at_1000_diff1": 0.31295, + "nauc_map_at_1_max": 0.151666, + "nauc_map_at_1_std": 0.086473, + "nauc_map_at_1_diff1": 0.287728, + "nauc_map_at_3_max": 0.136024, + "nauc_map_at_3_std": 0.033742, + "nauc_map_at_3_diff1": 0.298953, + "nauc_map_at_5_max": 0.145144, + "nauc_map_at_5_std": 0.034814, + "nauc_map_at_5_diff1": 0.302564, + "nauc_map_at_10_max": 0.149826, + "nauc_map_at_10_std": 0.025725, + "nauc_map_at_10_diff1": 0.303347, + "nauc_map_at_20_max": 0.15817, + "nauc_map_at_20_std": 0.034341, + "nauc_map_at_20_diff1": 0.305769, + "nauc_map_at_100_max": 0.163232, + "nauc_map_at_100_std": 0.033621, + "nauc_map_at_100_diff1": 0.302941, + "nauc_map_at_1000_max": 0.163678, + "nauc_map_at_1000_std": 0.034539, + "nauc_map_at_1000_diff1": 0.303114, + "nauc_recall_at_1_max": 0.151666, + "nauc_recall_at_1_std": 0.086473, + "nauc_recall_at_1_diff1": 0.287728, + "nauc_recall_at_3_max": 0.125191, + "nauc_recall_at_3_std": 0.010548, + "nauc_recall_at_3_diff1": 0.262147, + "nauc_recall_at_5_max": 0.154792, + "nauc_recall_at_5_std": 0.034442, + "nauc_recall_at_5_diff1": 0.259584, + "nauc_recall_at_10_max": 0.153127, + "nauc_recall_at_10_std": 0.005379, + "nauc_recall_at_10_diff1": 0.240219, + "nauc_recall_at_20_max": 0.149251, + "nauc_recall_at_20_std": 0.016114, + "nauc_recall_at_20_diff1": 0.266107, + "nauc_recall_at_100_max": 0.133042, + "nauc_recall_at_100_std": -0.028966, + "nauc_recall_at_100_diff1": 0.24137, + "nauc_recall_at_1000_max": 0.173461, + "nauc_recall_at_1000_std": 0.016742, + "nauc_recall_at_1000_diff1": 0.396117, + "nauc_precision_at_1_max": 0.167583, + "nauc_precision_at_1_std": 0.019165, + "nauc_precision_at_1_diff1": 0.357653, + "nauc_precision_at_3_max": 0.119037, + "nauc_precision_at_3_std": -0.040404, + "nauc_precision_at_3_diff1": 0.287693, + "nauc_precision_at_5_max": 0.154214, + "nauc_precision_at_5_std": -0.003995, + "nauc_precision_at_5_diff1": 0.240287, + "nauc_precision_at_10_max": 0.111741, + "nauc_precision_at_10_std": -0.036427, + "nauc_precision_at_10_diff1": 0.149912, + "nauc_precision_at_20_max": 0.10878, + "nauc_precision_at_20_std": 0.008271, + "nauc_precision_at_20_diff1": 0.087796, + "nauc_precision_at_100_max": 0.077626, + "nauc_precision_at_100_std": 0.016088, + "nauc_precision_at_100_diff1": -0.036554, + "nauc_precision_at_1000_max": 0.024229, + "nauc_precision_at_1000_std": 0.066199, + "nauc_precision_at_1000_diff1": -0.107955, + "nauc_mrr_at_1_max": 0.167583, + "nauc_mrr_at_1_std": 0.019165, + "nauc_mrr_at_1_diff1": 0.357653, + "nauc_mrr_at_3_max": 0.169644, + "nauc_mrr_at_3_std": -0.008168, + "nauc_mrr_at_3_diff1": 0.345079, + "nauc_mrr_at_5_max": 0.185063, + "nauc_mrr_at_5_std": 0.003586, + "nauc_mrr_at_5_diff1": 0.346053, + "nauc_mrr_at_10_max": 0.18742, + "nauc_mrr_at_10_std": 0.004044, + "nauc_mrr_at_10_diff1": 0.347143, + "nauc_mrr_at_20_max": 0.184985, + "nauc_mrr_at_20_std": 0.005881, + "nauc_mrr_at_20_diff1": 0.346876, + "nauc_mrr_at_100_max": 0.183898, + "nauc_mrr_at_100_std": 0.004452, + "nauc_mrr_at_100_diff1": 0.346763, + "nauc_mrr_at_1000_max": 0.183834, + "nauc_mrr_at_1000_std": 0.004425, + "nauc_mrr_at_1000_diff1": 0.346895, + "hit_rate_at_1": 0.39759, + "hit_rate_at_3": 0.59639, + "hit_rate_at_5": 0.66867, + "hit_rate_at_10": 0.74699, + "hit_rate_at_20": 0.81627, + "hit_rate_at_100": 0.91566, + "hit_rate_at_1000": 0.99398, + "main_score": 0.41341, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.4262, + "ndcg_at_3": 0.40835, + "ndcg_at_5": 0.42338, + "ndcg_at_10": 0.45127, + "ndcg_at_20": 0.4857, + "ndcg_at_100": 0.53833, + "ndcg_at_1000": 0.57705, + "map_at_1": 0.19109, + "map_at_3": 0.26837, + "map_at_5": 0.30647, + "map_at_10": 0.34076, + "map_at_20": 0.36204, + "map_at_100": 0.37792, + "map_at_1000": 0.38149, + "recall_at_1": 0.19109, + "recall_at_3": 0.31545, + "recall_at_5": 0.4002, + "recall_at_10": 0.50915, + "recall_at_20": 0.60696, + "recall_at_100": 0.78302, + "recall_at_1000": 0.96722, + "accuracy": 0.19109, + "precision_at_1": 0.44578, + "precision_at_3": 0.29116, + "precision_at_5": 0.24277, + "precision_at_10": 0.16988, + "precision_at_20": 0.11039, + "precision_at_100": 0.03262, + "precision_at_1000": 0.00448, + "mrr_at_1": 0.445783, + "mrr_at_3": 0.528112, + "mrr_at_5": 0.546787, + "mrr_at_10": 0.55798, + "mrr_at_20": 0.561121, + "mrr_at_100": 0.563546, + "mrr_at_1000": 0.56387, + "nauc_ndcg_at_1_max": 0.15442, + "nauc_ndcg_at_1_std": -0.006689, + "nauc_ndcg_at_1_diff1": 0.330821, + "nauc_ndcg_at_3_max": 0.191123, + "nauc_ndcg_at_3_std": 0.031296, + "nauc_ndcg_at_3_diff1": 0.28128, + "nauc_ndcg_at_5_max": 0.170493, + "nauc_ndcg_at_5_std": 0.012312, + "nauc_ndcg_at_5_diff1": 0.280491, + "nauc_ndcg_at_10_max": 0.167997, + "nauc_ndcg_at_10_std": -0.009496, + "nauc_ndcg_at_10_diff1": 0.294041, + "nauc_ndcg_at_20_max": 0.180915, + "nauc_ndcg_at_20_std": 0.016669, + "nauc_ndcg_at_20_diff1": 0.283404, + "nauc_ndcg_at_100_max": 0.192847, + "nauc_ndcg_at_100_std": 0.023509, + "nauc_ndcg_at_100_diff1": 0.30468, + "nauc_ndcg_at_1000_max": 0.198029, + "nauc_ndcg_at_1000_std": 0.031475, + "nauc_ndcg_at_1000_diff1": 0.31354, + "nauc_map_at_1_max": 0.145511, + "nauc_map_at_1_std": 0.049114, + "nauc_map_at_1_diff1": 0.283515, + "nauc_map_at_3_max": 0.16696, + "nauc_map_at_3_std": 0.036026, + "nauc_map_at_3_diff1": 0.268598, + "nauc_map_at_5_max": 0.154827, + "nauc_map_at_5_std": 0.018498, + "nauc_map_at_5_diff1": 0.26923, + "nauc_map_at_10_max": 0.160446, + "nauc_map_at_10_std": 0.007085, + "nauc_map_at_10_diff1": 0.285313, + "nauc_map_at_20_max": 0.170368, + "nauc_map_at_20_std": 0.022298, + "nauc_map_at_20_diff1": 0.281748, + "nauc_map_at_100_max": 0.176969, + "nauc_map_at_100_std": 0.024227, + "nauc_map_at_100_diff1": 0.287099, + "nauc_map_at_1000_max": 0.178221, + "nauc_map_at_1000_std": 0.025519, + "nauc_map_at_1000_diff1": 0.288944, + "nauc_recall_at_1_max": 0.145511, + "nauc_recall_at_1_std": 0.049114, + "nauc_recall_at_1_diff1": 0.283515, + "nauc_recall_at_3_max": 0.158887, + "nauc_recall_at_3_std": 0.019344, + "nauc_recall_at_3_diff1": 0.19415, + "nauc_recall_at_5_max": 0.143521, + "nauc_recall_at_5_std": -0.003119, + "nauc_recall_at_5_diff1": 0.199652, + "nauc_recall_at_10_max": 0.141872, + "nauc_recall_at_10_std": -0.048721, + "nauc_recall_at_10_diff1": 0.231499, + "nauc_recall_at_20_max": 0.153749, + "nauc_recall_at_20_std": 0.001592, + "nauc_recall_at_20_diff1": 0.191963, + "nauc_recall_at_100_max": 0.187372, + "nauc_recall_at_100_std": 0.035819, + "nauc_recall_at_100_diff1": 0.221846, + "nauc_recall_at_1000_max": 0.30351, + "nauc_recall_at_1000_std": 0.059766, + "nauc_recall_at_1000_diff1": 0.275221, + "nauc_precision_at_1_max": 0.160879, + "nauc_precision_at_1_std": -0.014124, + "nauc_precision_at_1_diff1": 0.337068, + "nauc_precision_at_3_max": 0.170795, + "nauc_precision_at_3_std": 0.013204, + "nauc_precision_at_3_diff1": 0.242543, + "nauc_precision_at_5_max": 0.107863, + "nauc_precision_at_5_std": -0.019716, + "nauc_precision_at_5_diff1": 0.189121, + "nauc_precision_at_10_max": 0.101271, + "nauc_precision_at_10_std": -0.028187, + "nauc_precision_at_10_diff1": 0.15006, + "nauc_precision_at_20_max": 0.104198, + "nauc_precision_at_20_std": 0.029704, + "nauc_precision_at_20_diff1": 0.091422, + "nauc_precision_at_100_max": 0.07244, + "nauc_precision_at_100_std": 0.035795, + "nauc_precision_at_100_diff1": 0.060534, + "nauc_precision_at_1000_max": 0.033833, + "nauc_precision_at_1000_std": 0.085394, + "nauc_precision_at_1000_diff1": 0.00737, + "nauc_mrr_at_1_max": 0.160879, + "nauc_mrr_at_1_std": -0.014124, + "nauc_mrr_at_1_diff1": 0.337068, + "nauc_mrr_at_3_max": 0.196842, + "nauc_mrr_at_3_std": 0.008587, + "nauc_mrr_at_3_diff1": 0.31241, + "nauc_mrr_at_5_max": 0.211487, + "nauc_mrr_at_5_std": 0.017814, + "nauc_mrr_at_5_diff1": 0.316803, + "nauc_mrr_at_10_max": 0.208003, + "nauc_mrr_at_10_std": 0.011775, + "nauc_mrr_at_10_diff1": 0.319615, + "nauc_mrr_at_20_max": 0.205217, + "nauc_mrr_at_20_std": 0.010365, + "nauc_mrr_at_20_diff1": 0.318471, + "nauc_mrr_at_100_max": 0.203138, + "nauc_mrr_at_100_std": 0.009111, + "nauc_mrr_at_100_diff1": 0.318529, + "nauc_mrr_at_1000_max": 0.202954, + "nauc_mrr_at_1000_std": 0.009047, + "nauc_mrr_at_1000_diff1": 0.318557, + "hit_rate_at_1": 0.44578, + "hit_rate_at_3": 0.63855, + "hit_rate_at_5": 0.71988, + "hit_rate_at_10": 0.8012, + "hit_rate_at_20": 0.84639, + "hit_rate_at_100": 0.92771, + "hit_rate_at_1000": 0.99398, + "main_score": 0.45127, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.10" + } + ] + }, + "evaluation_time": 1678.338508429937, + "kg_co2_emissions": null, + "date": 1792351746.866674, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 1.581292599439621e-05, + "end": 25.335277377977036, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 25.335309015004896, + "end": 25.335312369978055, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries", + "start": 25.360616117948666, + "end": 26.455633305944502, + "split": "test", + "subset": "french" + }, + { + "name": "Searching corpus", + "start": 26.459832552005537, + "end": 316.2418023230275, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 316.29971636494156, + "end": 316.5551025889581, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 316.607382053975, + "end": 317.0442271019565, + "split": "test", + "subset": "spanish" + }, + { + "name": "Searching corpus", + "start": 317.04812706797384, + "end": 585.3761591570219, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 585.4259173909668, + "end": 585.660196293029, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 585.6984193739481, + "end": 586.1814165259711, + "split": "test", + "subset": "english" + }, + { + "name": "Searching corpus", + "start": 586.1842892830027, + "end": 842.4073577949312, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 842.4612237119582, + "end": 842.6777010909282, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 842.7144376109354, + "end": 843.0967624819605, + "split": "test", + "subset": "german" + }, + { + "name": "Searching corpus", + "start": 843.0995984539622, + "end": 1130.2400779349264, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 1130.3089302589651, + "end": 1130.5730261870194, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 1130.6148396839853, + "end": 1130.9268869709922, + "split": "test", + "subset": "italian" + }, + { + "name": "Searching corpus", + "start": 1130.930622390937, + "end": 1397.7429930010112, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 1397.819102635025, + "end": 1398.0861550440313, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 1398.127093728981, + "end": 1398.557562412927, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Searching corpus", + "start": 1398.5611959110247, + "end": 1678.0585955990246, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 1678.1086312499829, + "end": 1678.3233970199944, + "split": "test", + "subset": "portuguese" + } + ] +} \ No newline at end of file diff --git a/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3TelecomRetrieval.json b/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3TelecomRetrieval.json new file mode 100644 index 0000000000..e5b5869a6b --- /dev/null +++ b/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3TelecomRetrieval.json @@ -0,0 +1,1090 @@ +{ + "dataset_revision": "a54635a274ef2835721b7cbe3eb27483b9ec964b", + "task_name": "Vidore3TelecomRetrieval", + "mteb_version": "2.26.10", + "scores": { + "test": [ + { + "ndcg_at_1": 0.60057, + "ndcg_at_3": 0.58055, + "ndcg_at_5": 0.58933, + "ndcg_at_10": 0.61839, + "ndcg_at_20": 0.64838, + "ndcg_at_100": 0.69591, + "ndcg_at_1000": 0.71278, + "map_at_1": 0.26845, + "map_at_3": 0.40952, + "map_at_5": 0.45292, + "map_at_10": 0.49841, + "map_at_20": 0.52395, + "map_at_100": 0.54292, + "map_at_1000": 0.54511, + "recall_at_1": 0.26845, + "recall_at_3": 0.47244, + "recall_at_5": 0.55144, + "recall_at_10": 0.66533, + "recall_at_20": 0.75816, + "recall_at_100": 0.90791, + "recall_at_1000": 0.98499, + "accuracy": 0.26845, + "precision_at_1": 0.62356, + "precision_at_3": 0.42816, + "precision_at_5": 0.32989, + "precision_at_10": 0.22672, + "precision_at_20": 0.14253, + "precision_at_100": 0.03865, + "precision_at_1000": 0.00439, + "mrr_at_1": 0.623563, + "mrr_at_3": 0.712165, + "mrr_at_5": 0.720929, + "mrr_at_10": 0.727283, + "mrr_at_20": 0.730068, + "mrr_at_100": 0.731318, + "mrr_at_1000": 0.731365, + "nauc_ndcg_at_1_max": -0.006, + "nauc_ndcg_at_1_std": -0.032256, + "nauc_ndcg_at_1_diff1": 0.469586, + "nauc_ndcg_at_3_max": 0.006548, + "nauc_ndcg_at_3_std": -0.033443, + "nauc_ndcg_at_3_diff1": 0.414186, + "nauc_ndcg_at_5_max": 0.013106, + "nauc_ndcg_at_5_std": -0.044445, + "nauc_ndcg_at_5_diff1": 0.393683, + "nauc_ndcg_at_10_max": 0.01941, + "nauc_ndcg_at_10_std": -0.035688, + "nauc_ndcg_at_10_diff1": 0.416117, + "nauc_ndcg_at_20_max": 0.030461, + "nauc_ndcg_at_20_std": -0.017123, + "nauc_ndcg_at_20_diff1": 0.415858, + "nauc_ndcg_at_100_max": 0.052029, + "nauc_ndcg_at_100_std": 0.025899, + "nauc_ndcg_at_100_diff1": 0.418209, + "nauc_ndcg_at_1000_max": 0.041407, + "nauc_ndcg_at_1000_std": 0.016947, + "nauc_ndcg_at_1000_diff1": 0.41879, + "nauc_map_at_1_max": -0.025856, + "nauc_map_at_1_std": -0.12427, + "nauc_map_at_1_diff1": 0.450814, + "nauc_map_at_3_max": 0.011773, + "nauc_map_at_3_std": -0.089286, + "nauc_map_at_3_diff1": 0.390924, + "nauc_map_at_5_max": 0.010864, + "nauc_map_at_5_std": -0.088764, + "nauc_map_at_5_diff1": 0.384405, + "nauc_map_at_10_max": 0.006591, + "nauc_map_at_10_std": -0.075252, + "nauc_map_at_10_diff1": 0.386343, + "nauc_map_at_20_max": 0.012077, + "nauc_map_at_20_std": -0.056618, + "nauc_map_at_20_diff1": 0.381345, + "nauc_map_at_100_max": 0.022092, + "nauc_map_at_100_std": -0.037889, + "nauc_map_at_100_diff1": 0.381304, + "nauc_map_at_1000_max": 0.021102, + "nauc_map_at_1000_std": -0.038141, + "nauc_map_at_1000_diff1": 0.380833, + "nauc_recall_at_1_max": -0.025856, + "nauc_recall_at_1_std": -0.12427, + "nauc_recall_at_1_diff1": 0.450814, + "nauc_recall_at_3_max": 0.015336, + "nauc_recall_at_3_std": -0.085566, + "nauc_recall_at_3_diff1": 0.345027, + "nauc_recall_at_5_max": 0.015163, + "nauc_recall_at_5_std": -0.095336, + "nauc_recall_at_5_diff1": 0.329474, + "nauc_recall_at_10_max": 0.032758, + "nauc_recall_at_10_std": -0.047143, + "nauc_recall_at_10_diff1": 0.361837, + "nauc_recall_at_20_max": 0.080288, + "nauc_recall_at_20_std": 0.023618, + "nauc_recall_at_20_diff1": 0.352722, + "nauc_recall_at_100_max": 0.226818, + "nauc_recall_at_100_std": 0.278104, + "nauc_recall_at_100_diff1": 0.354097, + "nauc_recall_at_1000_max": 0.330239, + "nauc_recall_at_1000_std": 0.61227, + "nauc_recall_at_1000_diff1": 0.717664, + "nauc_precision_at_1_max": -0.035855, + "nauc_precision_at_1_std": -0.038504, + "nauc_precision_at_1_diff1": 0.470624, + "nauc_precision_at_3_max": 0.013315, + "nauc_precision_at_3_std": 0.071395, + "nauc_precision_at_3_diff1": 0.134777, + "nauc_precision_at_5_max": 0.024048, + "nauc_precision_at_5_std": 0.090213, + "nauc_precision_at_5_diff1": 0.025186, + "nauc_precision_at_10_max": 0.01058, + "nauc_precision_at_10_std": 0.120549, + "nauc_precision_at_10_diff1": -0.038433, + "nauc_precision_at_20_max": 0.019375, + "nauc_precision_at_20_std": 0.153483, + "nauc_precision_at_20_diff1": -0.102408, + "nauc_precision_at_100_max": 0.052806, + "nauc_precision_at_100_std": 0.198336, + "nauc_precision_at_100_diff1": -0.163243, + "nauc_precision_at_1000_max": 0.012167, + "nauc_precision_at_1000_std": 0.155492, + "nauc_precision_at_1000_diff1": -0.203595, + "nauc_mrr_at_1_max": -0.035855, + "nauc_mrr_at_1_std": -0.038504, + "nauc_mrr_at_1_diff1": 0.470624, + "nauc_mrr_at_3_max": -0.030152, + "nauc_mrr_at_3_std": -0.000948, + "nauc_mrr_at_3_diff1": 0.461715, + "nauc_mrr_at_5_max": -0.021206, + "nauc_mrr_at_5_std": -0.001983, + "nauc_mrr_at_5_diff1": 0.464318, + "nauc_mrr_at_10_max": -0.015181, + "nauc_mrr_at_10_std": 0.004995, + "nauc_mrr_at_10_diff1": 0.470006, + "nauc_mrr_at_20_max": -0.015867, + "nauc_mrr_at_20_std": 0.002947, + "nauc_mrr_at_20_diff1": 0.470867, + "nauc_mrr_at_100_max": -0.018869, + "nauc_mrr_at_100_std": 0.000257, + "nauc_mrr_at_100_diff1": 0.469604, + "nauc_mrr_at_1000_max": -0.018778, + "nauc_mrr_at_1000_std": 0.000313, + "nauc_mrr_at_1000_diff1": 0.469541, + "hit_rate_at_1": 0.62356, + "hit_rate_at_3": 0.82759, + "hit_rate_at_5": 0.86494, + "hit_rate_at_10": 0.90805, + "hit_rate_at_20": 0.94828, + "hit_rate_at_100": 0.98563, + "hit_rate_at_1000": 0.99713, + "main_score": 0.61839, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.57902, + "ndcg_at_3": 0.57274, + "ndcg_at_5": 0.5803, + "ndcg_at_10": 0.60496, + "ndcg_at_20": 0.63983, + "ndcg_at_100": 0.68471, + "ndcg_at_1000": 0.70351, + "map_at_1": 0.26638, + "map_at_3": 0.40629, + "map_at_5": 0.44947, + "map_at_10": 0.49218, + "map_at_20": 0.51947, + "map_at_100": 0.53783, + "map_at_1000": 0.54019, + "recall_at_1": 0.26638, + "recall_at_3": 0.45937, + "recall_at_5": 0.54361, + "recall_at_10": 0.65142, + "recall_at_20": 0.75699, + "recall_at_100": 0.89579, + "recall_at_1000": 0.98244, + "accuracy": 0.26638, + "precision_at_1": 0.60632, + "precision_at_3": 0.42529, + "precision_at_5": 0.32931, + "precision_at_10": 0.2181, + "precision_at_20": 0.13994, + "precision_at_100": 0.03822, + "precision_at_1000": 0.00438, + "mrr_at_1": 0.606322, + "mrr_at_3": 0.695881, + "mrr_at_5": 0.706513, + "mrr_at_10": 0.713145, + "mrr_at_20": 0.715718, + "mrr_at_100": 0.716765, + "mrr_at_1000": 0.716889, + "nauc_ndcg_at_1_max": 0.052818, + "nauc_ndcg_at_1_std": 0.055088, + "nauc_ndcg_at_1_diff1": 0.462676, + "nauc_ndcg_at_3_max": 0.059307, + "nauc_ndcg_at_3_std": 0.005098, + "nauc_ndcg_at_3_diff1": 0.358313, + "nauc_ndcg_at_5_max": 0.042519, + "nauc_ndcg_at_5_std": -0.01398, + "nauc_ndcg_at_5_diff1": 0.36247, + "nauc_ndcg_at_10_max": 0.06473, + "nauc_ndcg_at_10_std": -0.004898, + "nauc_ndcg_at_10_diff1": 0.360442, + "nauc_ndcg_at_20_max": 0.078558, + "nauc_ndcg_at_20_std": 0.018087, + "nauc_ndcg_at_20_diff1": 0.367136, + "nauc_ndcg_at_100_max": 0.096887, + "nauc_ndcg_at_100_std": 0.062811, + "nauc_ndcg_at_100_diff1": 0.378359, + "nauc_ndcg_at_1000_max": 0.087054, + "nauc_ndcg_at_1000_std": 0.053373, + "nauc_ndcg_at_1000_diff1": 0.385457, + "nauc_map_at_1_max": -0.023028, + "nauc_map_at_1_std": -0.133152, + "nauc_map_at_1_diff1": 0.389679, + "nauc_map_at_3_max": 0.05301, + "nauc_map_at_3_std": -0.096631, + "nauc_map_at_3_diff1": 0.337367, + "nauc_map_at_5_max": 0.046881, + "nauc_map_at_5_std": -0.087977, + "nauc_map_at_5_diff1": 0.337322, + "nauc_map_at_10_max": 0.052852, + "nauc_map_at_10_std": -0.0665, + "nauc_map_at_10_diff1": 0.332575, + "nauc_map_at_20_max": 0.059379, + "nauc_map_at_20_std": -0.040984, + "nauc_map_at_20_diff1": 0.339356, + "nauc_map_at_100_max": 0.06478, + "nauc_map_at_100_std": -0.019396, + "nauc_map_at_100_diff1": 0.342656, + "nauc_map_at_1000_max": 0.064453, + "nauc_map_at_1000_std": -0.018991, + "nauc_map_at_1000_diff1": 0.343193, + "nauc_recall_at_1_max": -0.023028, + "nauc_recall_at_1_std": -0.133152, + "nauc_recall_at_1_diff1": 0.389679, + "nauc_recall_at_3_max": 0.044026, + "nauc_recall_at_3_std": -0.112191, + "nauc_recall_at_3_diff1": 0.28406, + "nauc_recall_at_5_max": 0.032507, + "nauc_recall_at_5_std": -0.105084, + "nauc_recall_at_5_diff1": 0.284619, + "nauc_recall_at_10_max": 0.082562, + "nauc_recall_at_10_std": -0.027457, + "nauc_recall_at_10_diff1": 0.260398, + "nauc_recall_at_20_max": 0.103676, + "nauc_recall_at_20_std": 0.048586, + "nauc_recall_at_20_diff1": 0.251302, + "nauc_recall_at_100_max": 0.260384, + "nauc_recall_at_100_std": 0.31965, + "nauc_recall_at_100_diff1": 0.262405, + "nauc_recall_at_1000_max": 0.369553, + "nauc_recall_at_1000_std": 0.592815, + "nauc_recall_at_1000_diff1": 0.31672, + "nauc_precision_at_1_max": 0.024969, + "nauc_precision_at_1_std": 0.063263, + "nauc_precision_at_1_diff1": 0.455746, + "nauc_precision_at_3_max": 0.094583, + "nauc_precision_at_3_std": 0.125219, + "nauc_precision_at_3_diff1": 0.110149, + "nauc_precision_at_5_max": 0.049199, + "nauc_precision_at_5_std": 0.147904, + "nauc_precision_at_5_diff1": 0.037337, + "nauc_precision_at_10_max": 0.04756, + "nauc_precision_at_10_std": 0.180137, + "nauc_precision_at_10_diff1": -0.001731, + "nauc_precision_at_20_max": 0.055829, + "nauc_precision_at_20_std": 0.237827, + "nauc_precision_at_20_diff1": -0.028834, + "nauc_precision_at_100_max": 0.034124, + "nauc_precision_at_100_std": 0.257113, + "nauc_precision_at_100_diff1": -0.09327, + "nauc_precision_at_1000_max": -0.002135, + "nauc_precision_at_1000_std": 0.207292, + "nauc_precision_at_1000_diff1": -0.124375, + "nauc_mrr_at_1_max": 0.024969, + "nauc_mrr_at_1_std": 0.063263, + "nauc_mrr_at_1_diff1": 0.455746, + "nauc_mrr_at_3_max": 0.033534, + "nauc_mrr_at_3_std": 0.07228, + "nauc_mrr_at_3_diff1": 0.44634, + "nauc_mrr_at_5_max": 0.023877, + "nauc_mrr_at_5_std": 0.069212, + "nauc_mrr_at_5_diff1": 0.445987, + "nauc_mrr_at_10_max": 0.036943, + "nauc_mrr_at_10_std": 0.081419, + "nauc_mrr_at_10_diff1": 0.449876, + "nauc_mrr_at_20_max": 0.038151, + "nauc_mrr_at_20_std": 0.083136, + "nauc_mrr_at_20_diff1": 0.452141, + "nauc_mrr_at_100_max": 0.038918, + "nauc_mrr_at_100_std": 0.082962, + "nauc_mrr_at_100_diff1": 0.451906, + "nauc_mrr_at_1000_max": 0.038513, + "nauc_mrr_at_1000_std": 0.082514, + "nauc_mrr_at_1000_diff1": 0.451856, + "hit_rate_at_1": 0.60632, + "hit_rate_at_3": 0.8046, + "hit_rate_at_5": 0.85057, + "hit_rate_at_10": 0.89943, + "hit_rate_at_20": 0.93391, + "hit_rate_at_100": 0.97414, + "hit_rate_at_1000": 0.99713, + "main_score": 0.60496, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.66379, + "ndcg_at_3": 0.63817, + "ndcg_at_5": 0.64762, + "ndcg_at_10": 0.67984, + "ndcg_at_20": 0.7081, + "ndcg_at_100": 0.74799, + "ndcg_at_1000": 0.76019, + "map_at_1": 0.30284, + "map_at_3": 0.45588, + "map_at_5": 0.50666, + "map_at_10": 0.56184, + "map_at_20": 0.58813, + "map_at_100": 0.60586, + "map_at_1000": 0.60759, + "recall_at_1": 0.30284, + "recall_at_3": 0.50371, + "recall_at_5": 0.59159, + "recall_at_10": 0.7225, + "recall_at_20": 0.81353, + "recall_at_100": 0.93407, + "recall_at_1000": 0.98807, + "accuracy": 0.30284, + "precision_at_1": 0.69253, + "precision_at_3": 0.47031, + "precision_at_5": 0.37069, + "precision_at_10": 0.25345, + "precision_at_20": 0.15503, + "precision_at_100": 0.04037, + "precision_at_1000": 0.00443, + "mrr_at_1": 0.692529, + "mrr_at_3": 0.771552, + "mrr_at_5": 0.782759, + "mrr_at_10": 0.787961, + "mrr_at_20": 0.789771, + "mrr_at_100": 0.790345, + "mrr_at_1000": 0.790359, + "nauc_ndcg_at_1_max": 0.216624, + "nauc_ndcg_at_1_std": 0.11475, + "nauc_ndcg_at_1_diff1": 0.49938, + "nauc_ndcg_at_3_max": 0.143342, + "nauc_ndcg_at_3_std": 0.016821, + "nauc_ndcg_at_3_diff1": 0.415758, + "nauc_ndcg_at_5_max": 0.157029, + "nauc_ndcg_at_5_std": 0.021301, + "nauc_ndcg_at_5_diff1": 0.420133, + "nauc_ndcg_at_10_max": 0.169929, + "nauc_ndcg_at_10_std": 0.034881, + "nauc_ndcg_at_10_diff1": 0.432402, + "nauc_ndcg_at_20_max": 0.196923, + "nauc_ndcg_at_20_std": 0.060506, + "nauc_ndcg_at_20_diff1": 0.442207, + "nauc_ndcg_at_100_max": 0.214476, + "nauc_ndcg_at_100_std": 0.102692, + "nauc_ndcg_at_100_diff1": 0.450145, + "nauc_ndcg_at_1000_max": 0.202214, + "nauc_ndcg_at_1000_std": 0.090437, + "nauc_ndcg_at_1000_diff1": 0.44411, + "nauc_map_at_1_max": 0.118113, + "nauc_map_at_1_std": -0.07658, + "nauc_map_at_1_diff1": 0.48252, + "nauc_map_at_3_max": 0.148477, + "nauc_map_at_3_std": -0.05206, + "nauc_map_at_3_diff1": 0.427178, + "nauc_map_at_5_max": 0.156164, + "nauc_map_at_5_std": -0.040188, + "nauc_map_at_5_diff1": 0.412418, + "nauc_map_at_10_max": 0.159932, + "nauc_map_at_10_std": -0.025097, + "nauc_map_at_10_diff1": 0.407235, + "nauc_map_at_20_max": 0.174156, + "nauc_map_at_20_std": -0.004057, + "nauc_map_at_20_diff1": 0.407747, + "nauc_map_at_100_max": 0.180372, + "nauc_map_at_100_std": 0.011687, + "nauc_map_at_100_diff1": 0.408108, + "nauc_map_at_1000_max": 0.179488, + "nauc_map_at_1000_std": 0.011734, + "nauc_map_at_1000_diff1": 0.407157, + "nauc_recall_at_1_max": 0.118113, + "nauc_recall_at_1_std": -0.07658, + "nauc_recall_at_1_diff1": 0.48252, + "nauc_recall_at_3_max": 0.130306, + "nauc_recall_at_3_std": -0.053844, + "nauc_recall_at_3_diff1": 0.393393, + "nauc_recall_at_5_max": 0.140732, + "nauc_recall_at_5_std": -0.032902, + "nauc_recall_at_5_diff1": 0.3695, + "nauc_recall_at_10_max": 0.162583, + "nauc_recall_at_10_std": 0.008353, + "nauc_recall_at_10_diff1": 0.352588, + "nauc_recall_at_20_max": 0.24029, + "nauc_recall_at_20_std": 0.090998, + "nauc_recall_at_20_diff1": 0.360468, + "nauc_recall_at_100_max": 0.467186, + "nauc_recall_at_100_std": 0.44329, + "nauc_recall_at_100_diff1": 0.439025, + "nauc_recall_at_1000_max": 0.896134, + "nauc_recall_at_1000_std": 0.858842, + "nauc_recall_at_1000_diff1": 0.571394, + "nauc_precision_at_1_max": 0.220252, + "nauc_precision_at_1_std": 0.127822, + "nauc_precision_at_1_diff1": 0.513537, + "nauc_precision_at_3_max": 0.065385, + "nauc_precision_at_3_std": 0.091253, + "nauc_precision_at_3_diff1": 0.039687, + "nauc_precision_at_5_max": 0.030818, + "nauc_precision_at_5_std": 0.105793, + "nauc_precision_at_5_diff1": -0.071975, + "nauc_precision_at_10_max": 0.001291, + "nauc_precision_at_10_std": 0.121148, + "nauc_precision_at_10_diff1": -0.145267, + "nauc_precision_at_20_max": 0.001939, + "nauc_precision_at_20_std": 0.153693, + "nauc_precision_at_20_diff1": -0.18196, + "nauc_precision_at_100_max": -0.026038, + "nauc_precision_at_100_std": 0.172891, + "nauc_precision_at_100_diff1": -0.217823, + "nauc_precision_at_1000_max": -0.064292, + "nauc_precision_at_1000_std": 0.142034, + "nauc_precision_at_1000_diff1": -0.252608, + "nauc_mrr_at_1_max": 0.220252, + "nauc_mrr_at_1_std": 0.127822, + "nauc_mrr_at_1_diff1": 0.513537, + "nauc_mrr_at_3_max": 0.188987, + "nauc_mrr_at_3_std": 0.128663, + "nauc_mrr_at_3_diff1": 0.498459, + "nauc_mrr_at_5_max": 0.197252, + "nauc_mrr_at_5_std": 0.13553, + "nauc_mrr_at_5_diff1": 0.505145, + "nauc_mrr_at_10_max": 0.194345, + "nauc_mrr_at_10_std": 0.134229, + "nauc_mrr_at_10_diff1": 0.503295, + "nauc_mrr_at_20_max": 0.196871, + "nauc_mrr_at_20_std": 0.133929, + "nauc_mrr_at_20_diff1": 0.503072, + "nauc_mrr_at_100_max": 0.197652, + "nauc_mrr_at_100_std": 0.133576, + "nauc_mrr_at_100_diff1": 0.502733, + "nauc_mrr_at_1000_max": 0.19764, + "nauc_mrr_at_1000_std": 0.133544, + "nauc_mrr_at_1000_diff1": 0.502716, + "hit_rate_at_1": 0.69253, + "hit_rate_at_3": 0.86207, + "hit_rate_at_5": 0.91092, + "hit_rate_at_10": 0.94828, + "hit_rate_at_20": 0.97126, + "hit_rate_at_100": 0.99138, + "hit_rate_at_1000": 0.99425, + "main_score": 0.67984, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.5704, + "ndcg_at_3": 0.56624, + "ndcg_at_5": 0.56815, + "ndcg_at_10": 0.59957, + "ndcg_at_20": 0.6317, + "ndcg_at_100": 0.6802, + "ndcg_at_1000": 0.69781, + "map_at_1": 0.26175, + "map_at_3": 0.39812, + "map_at_5": 0.43677, + "map_at_10": 0.48208, + "map_at_20": 0.50842, + "map_at_100": 0.52706, + "map_at_1000": 0.52932, + "recall_at_1": 0.26175, + "recall_at_3": 0.46133, + "recall_at_5": 0.53222, + "recall_at_10": 0.64762, + "recall_at_20": 0.74593, + "recall_at_100": 0.90051, + "recall_at_1000": 0.98069, + "accuracy": 0.26175, + "precision_at_1": 0.58908, + "precision_at_3": 0.41762, + "precision_at_5": 0.31724, + "precision_at_10": 0.21954, + "precision_at_20": 0.13879, + "precision_at_100": 0.03825, + "precision_at_1000": 0.00439, + "mrr_at_1": 0.58908, + "mrr_at_3": 0.693966, + "mrr_at_5": 0.701149, + "mrr_at_10": 0.70852, + "mrr_at_20": 0.710317, + "mrr_at_100": 0.711958, + "mrr_at_1000": 0.711989, + "nauc_ndcg_at_1_max": -0.023524, + "nauc_ndcg_at_1_std": 0.005279, + "nauc_ndcg_at_1_diff1": 0.451836, + "nauc_ndcg_at_3_max": 0.012819, + "nauc_ndcg_at_3_std": 0.011352, + "nauc_ndcg_at_3_diff1": 0.399906, + "nauc_ndcg_at_5_max": 0.020438, + "nauc_ndcg_at_5_std": 0.011164, + "nauc_ndcg_at_5_diff1": 0.408651, + "nauc_ndcg_at_10_max": 0.023879, + "nauc_ndcg_at_10_std": 0.01613, + "nauc_ndcg_at_10_diff1": 0.408344, + "nauc_ndcg_at_20_max": 0.039657, + "nauc_ndcg_at_20_std": 0.044522, + "nauc_ndcg_at_20_diff1": 0.422689, + "nauc_ndcg_at_100_max": 0.055454, + "nauc_ndcg_at_100_std": 0.075595, + "nauc_ndcg_at_100_diff1": 0.421406, + "nauc_ndcg_at_1000_max": 0.052329, + "nauc_ndcg_at_1000_std": 0.075663, + "nauc_ndcg_at_1000_diff1": 0.420444, + "nauc_map_at_1_max": -0.034913, + "nauc_map_at_1_std": -0.100481, + "nauc_map_at_1_diff1": 0.44078, + "nauc_map_at_3_max": 0.019109, + "nauc_map_at_3_std": -0.054958, + "nauc_map_at_3_diff1": 0.40941, + "nauc_map_at_5_max": 0.021632, + "nauc_map_at_5_std": -0.038898, + "nauc_map_at_5_diff1": 0.41341, + "nauc_map_at_10_max": 0.023904, + "nauc_map_at_10_std": -0.019843, + "nauc_map_at_10_diff1": 0.40728, + "nauc_map_at_20_max": 0.03317, + "nauc_map_at_20_std": 0.001035, + "nauc_map_at_20_diff1": 0.412702, + "nauc_map_at_100_max": 0.037953, + "nauc_map_at_100_std": 0.014066, + "nauc_map_at_100_diff1": 0.409345, + "nauc_map_at_1000_max": 0.03757, + "nauc_map_at_1000_std": 0.01525, + "nauc_map_at_1000_diff1": 0.40889, + "nauc_recall_at_1_max": -0.034913, + "nauc_recall_at_1_std": -0.100481, + "nauc_recall_at_1_diff1": 0.44078, + "nauc_recall_at_3_max": 0.020237, + "nauc_recall_at_3_std": -0.058158, + "nauc_recall_at_3_diff1": 0.344606, + "nauc_recall_at_5_max": 0.028517, + "nauc_recall_at_5_std": -0.048871, + "nauc_recall_at_5_diff1": 0.348583, + "nauc_recall_at_10_max": 0.025086, + "nauc_recall_at_10_std": -0.032647, + "nauc_recall_at_10_diff1": 0.325309, + "nauc_recall_at_20_max": 0.075767, + "nauc_recall_at_20_std": 0.057406, + "nauc_recall_at_20_diff1": 0.363411, + "nauc_recall_at_100_max": 0.20247, + "nauc_recall_at_100_std": 0.254976, + "nauc_recall_at_100_diff1": 0.36472, + "nauc_recall_at_1000_max": 0.541954, + "nauc_recall_at_1000_std": 0.683613, + "nauc_recall_at_1000_diff1": 0.312581, + "nauc_precision_at_1_max": -0.039324, + "nauc_precision_at_1_std": 0.007841, + "nauc_precision_at_1_diff1": 0.459105, + "nauc_precision_at_3_max": 0.06615, + "nauc_precision_at_3_std": 0.144931, + "nauc_precision_at_3_diff1": 0.162417, + "nauc_precision_at_5_max": 0.059917, + "nauc_precision_at_5_std": 0.171754, + "nauc_precision_at_5_diff1": 0.084351, + "nauc_precision_at_10_max": 0.054826, + "nauc_precision_at_10_std": 0.187102, + "nauc_precision_at_10_diff1": -0.020246, + "nauc_precision_at_20_max": 0.060814, + "nauc_precision_at_20_std": 0.203674, + "nauc_precision_at_20_diff1": -0.060168, + "nauc_precision_at_100_max": 0.055304, + "nauc_precision_at_100_std": 0.203583, + "nauc_precision_at_100_diff1": -0.154355, + "nauc_precision_at_1000_max": 0.032442, + "nauc_precision_at_1000_std": 0.173283, + "nauc_precision_at_1000_diff1": -0.187073, + "nauc_mrr_at_1_max": -0.039324, + "nauc_mrr_at_1_std": 0.007841, + "nauc_mrr_at_1_diff1": 0.459105, + "nauc_mrr_at_3_max": -0.012713, + "nauc_mrr_at_3_std": 0.06541, + "nauc_mrr_at_3_diff1": 0.439796, + "nauc_mrr_at_5_max": -0.012893, + "nauc_mrr_at_5_std": 0.061864, + "nauc_mrr_at_5_diff1": 0.435322, + "nauc_mrr_at_10_max": -0.012139, + "nauc_mrr_at_10_std": 0.057994, + "nauc_mrr_at_10_diff1": 0.441853, + "nauc_mrr_at_20_max": -0.010614, + "nauc_mrr_at_20_std": 0.060796, + "nauc_mrr_at_20_diff1": 0.442206, + "nauc_mrr_at_100_max": -0.010525, + "nauc_mrr_at_100_std": 0.059719, + "nauc_mrr_at_100_diff1": 0.442496, + "nauc_mrr_at_1000_max": -0.010562, + "nauc_mrr_at_1000_std": 0.059656, + "nauc_mrr_at_1000_diff1": 0.442492, + "hit_rate_at_1": 0.58908, + "hit_rate_at_3": 0.82184, + "hit_rate_at_5": 0.85345, + "hit_rate_at_10": 0.90805, + "hit_rate_at_20": 0.93391, + "hit_rate_at_100": 0.98563, + "hit_rate_at_1000": 0.99425, + "main_score": 0.59957, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.57615, + "ndcg_at_3": 0.55732, + "ndcg_at_5": 0.56775, + "ndcg_at_10": 0.59965, + "ndcg_at_20": 0.6336, + "ndcg_at_100": 0.6801, + "ndcg_at_1000": 0.69788, + "map_at_1": 0.2555, + "map_at_3": 0.39404, + "map_at_5": 0.438, + "map_at_10": 0.48104, + "map_at_20": 0.50818, + "map_at_100": 0.52648, + "map_at_1000": 0.52876, + "recall_at_1": 0.2555, + "recall_at_3": 0.45437, + "recall_at_5": 0.53936, + "recall_at_10": 0.65652, + "recall_at_20": 0.75854, + "recall_at_100": 0.90239, + "recall_at_1000": 0.98272, + "accuracy": 0.2555, + "precision_at_1": 0.60345, + "precision_at_3": 0.40709, + "precision_at_5": 0.31897, + "precision_at_10": 0.21897, + "precision_at_20": 0.13908, + "precision_at_100": 0.03822, + "precision_at_1000": 0.00438, + "mrr_at_1": 0.603448, + "mrr_at_3": 0.684387, + "mrr_at_5": 0.697749, + "mrr_at_10": 0.706682, + "mrr_at_20": 0.709401, + "mrr_at_100": 0.710465, + "mrr_at_1000": 0.710543, + "nauc_ndcg_at_1_max": 0.074046, + "nauc_ndcg_at_1_std": 0.054628, + "nauc_ndcg_at_1_diff1": 0.432492, + "nauc_ndcg_at_3_max": 0.037417, + "nauc_ndcg_at_3_std": 0.013035, + "nauc_ndcg_at_3_diff1": 0.340517, + "nauc_ndcg_at_5_max": 0.052585, + "nauc_ndcg_at_5_std": 0.002192, + "nauc_ndcg_at_5_diff1": 0.364047, + "nauc_ndcg_at_10_max": 0.040821, + "nauc_ndcg_at_10_std": -0.00964, + "nauc_ndcg_at_10_diff1": 0.358599, + "nauc_ndcg_at_20_max": 0.063238, + "nauc_ndcg_at_20_std": 0.018351, + "nauc_ndcg_at_20_diff1": 0.360868, + "nauc_ndcg_at_100_max": 0.087484, + "nauc_ndcg_at_100_std": 0.069419, + "nauc_ndcg_at_100_diff1": 0.370678, + "nauc_ndcg_at_1000_max": 0.079212, + "nauc_ndcg_at_1000_std": 0.061662, + "nauc_ndcg_at_1000_diff1": 0.372583, + "nauc_map_at_1_max": 0.00207, + "nauc_map_at_1_std": -0.110615, + "nauc_map_at_1_diff1": 0.396587, + "nauc_map_at_3_max": 0.040428, + "nauc_map_at_3_std": -0.071862, + "nauc_map_at_3_diff1": 0.319561, + "nauc_map_at_5_max": 0.055274, + "nauc_map_at_5_std": -0.059173, + "nauc_map_at_5_diff1": 0.326681, + "nauc_map_at_10_max": 0.04353, + "nauc_map_at_10_std": -0.05328, + "nauc_map_at_10_diff1": 0.315564, + "nauc_map_at_20_max": 0.054153, + "nauc_map_at_20_std": -0.029585, + "nauc_map_at_20_diff1": 0.324945, + "nauc_map_at_100_max": 0.061338, + "nauc_map_at_100_std": -0.011491, + "nauc_map_at_100_diff1": 0.330784, + "nauc_map_at_1000_max": 0.06092, + "nauc_map_at_1000_std": -0.010721, + "nauc_map_at_1000_diff1": 0.331086, + "nauc_recall_at_1_max": 0.00207, + "nauc_recall_at_1_std": -0.110615, + "nauc_recall_at_1_diff1": 0.396587, + "nauc_recall_at_3_max": 0.007129, + "nauc_recall_at_3_std": -0.096207, + "nauc_recall_at_3_diff1": 0.254995, + "nauc_recall_at_5_max": 0.044463, + "nauc_recall_at_5_std": -0.078594, + "nauc_recall_at_5_diff1": 0.289964, + "nauc_recall_at_10_max": 0.007092, + "nauc_recall_at_10_std": -0.083033, + "nauc_recall_at_10_diff1": 0.263492, + "nauc_recall_at_20_max": 0.082377, + "nauc_recall_at_20_std": 0.016355, + "nauc_recall_at_20_diff1": 0.273434, + "nauc_recall_at_100_max": 0.236601, + "nauc_recall_at_100_std": 0.329739, + "nauc_recall_at_100_diff1": 0.280707, + "nauc_recall_at_1000_max": 0.426212, + "nauc_recall_at_1000_std": 0.710859, + "nauc_recall_at_1000_diff1": 0.22979, + "nauc_precision_at_1_max": 0.056776, + "nauc_precision_at_1_std": 0.040786, + "nauc_precision_at_1_diff1": 0.43846, + "nauc_precision_at_3_max": 0.063482, + "nauc_precision_at_3_std": 0.124706, + "nauc_precision_at_3_diff1": 0.117431, + "nauc_precision_at_5_max": 0.064266, + "nauc_precision_at_5_std": 0.146496, + "nauc_precision_at_5_diff1": 0.069722, + "nauc_precision_at_10_max": 0.033598, + "nauc_precision_at_10_std": 0.161208, + "nauc_precision_at_10_diff1": 0.014247, + "nauc_precision_at_20_max": 0.056484, + "nauc_precision_at_20_std": 0.202678, + "nauc_precision_at_20_diff1": -0.003108, + "nauc_precision_at_100_max": 0.052596, + "nauc_precision_at_100_std": 0.223182, + "nauc_precision_at_100_diff1": -0.0589, + "nauc_precision_at_1000_max": 0.018669, + "nauc_precision_at_1000_std": 0.176083, + "nauc_precision_at_1000_diff1": -0.092554, + "nauc_mrr_at_1_max": 0.056776, + "nauc_mrr_at_1_std": 0.040786, + "nauc_mrr_at_1_diff1": 0.43846, + "nauc_mrr_at_3_max": 0.041374, + "nauc_mrr_at_3_std": 0.04852, + "nauc_mrr_at_3_diff1": 0.423756, + "nauc_mrr_at_5_max": 0.053964, + "nauc_mrr_at_5_std": 0.056628, + "nauc_mrr_at_5_diff1": 0.437904, + "nauc_mrr_at_10_max": 0.049653, + "nauc_mrr_at_10_std": 0.059459, + "nauc_mrr_at_10_diff1": 0.441541, + "nauc_mrr_at_20_max": 0.053988, + "nauc_mrr_at_20_std": 0.06188, + "nauc_mrr_at_20_diff1": 0.439941, + "nauc_mrr_at_100_max": 0.054309, + "nauc_mrr_at_100_std": 0.06122, + "nauc_mrr_at_100_diff1": 0.439812, + "nauc_mrr_at_1000_max": 0.054099, + "nauc_mrr_at_1000_std": 0.060967, + "nauc_mrr_at_1000_diff1": 0.43976, + "hit_rate_at_1": 0.60345, + "hit_rate_at_3": 0.78448, + "hit_rate_at_5": 0.84195, + "hit_rate_at_10": 0.90517, + "hit_rate_at_20": 0.9454, + "hit_rate_at_100": 0.97989, + "hit_rate_at_1000": 0.99713, + "main_score": 0.59965, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.5819, + "ndcg_at_3": 0.58002, + "ndcg_at_5": 0.58557, + "ndcg_at_10": 0.61999, + "ndcg_at_20": 0.64852, + "ndcg_at_100": 0.69293, + "ndcg_at_1000": 0.70911, + "map_at_1": 0.26339, + "map_at_3": 0.40832, + "map_at_5": 0.45071, + "map_at_10": 0.50248, + "map_at_20": 0.52676, + "map_at_100": 0.54546, + "map_at_1000": 0.54757, + "recall_at_1": 0.26339, + "recall_at_3": 0.46566, + "recall_at_5": 0.54909, + "recall_at_10": 0.68407, + "recall_at_20": 0.76928, + "recall_at_100": 0.90441, + "recall_at_1000": 0.97812, + "accuracy": 0.26339, + "precision_at_1": 0.6092, + "precision_at_3": 0.43774, + "precision_at_5": 0.33621, + "precision_at_10": 0.23161, + "precision_at_20": 0.14425, + "precision_at_100": 0.03876, + "precision_at_1000": 0.00437, + "mrr_at_1": 0.609195, + "mrr_at_3": 0.70067, + "mrr_at_5": 0.713602, + "mrr_at_10": 0.720995, + "mrr_at_20": 0.722599, + "mrr_at_100": 0.723443, + "mrr_at_1000": 0.72349, + "nauc_ndcg_at_1_max": 0.097292, + "nauc_ndcg_at_1_std": 0.069242, + "nauc_ndcg_at_1_diff1": 0.427727, + "nauc_ndcg_at_3_max": 0.080054, + "nauc_ndcg_at_3_std": 0.014173, + "nauc_ndcg_at_3_diff1": 0.365686, + "nauc_ndcg_at_5_max": 0.058478, + "nauc_ndcg_at_5_std": -0.024738, + "nauc_ndcg_at_5_diff1": 0.380666, + "nauc_ndcg_at_10_max": 0.071903, + "nauc_ndcg_at_10_std": -0.016173, + "nauc_ndcg_at_10_diff1": 0.370709, + "nauc_ndcg_at_20_max": 0.09415, + "nauc_ndcg_at_20_std": 0.008662, + "nauc_ndcg_at_20_diff1": 0.384578, + "nauc_ndcg_at_100_max": 0.122499, + "nauc_ndcg_at_100_std": 0.058939, + "nauc_ndcg_at_100_diff1": 0.384473, + "nauc_ndcg_at_1000_max": 0.116467, + "nauc_ndcg_at_1000_std": 0.059385, + "nauc_ndcg_at_1000_diff1": 0.385033, + "nauc_map_at_1_max": 0.023788, + "nauc_map_at_1_std": -0.103704, + "nauc_map_at_1_diff1": 0.427193, + "nauc_map_at_3_max": 0.057167, + "nauc_map_at_3_std": -0.088476, + "nauc_map_at_3_diff1": 0.365933, + "nauc_map_at_5_max": 0.050363, + "nauc_map_at_5_std": -0.087927, + "nauc_map_at_5_diff1": 0.36752, + "nauc_map_at_10_max": 0.052789, + "nauc_map_at_10_std": -0.066208, + "nauc_map_at_10_diff1": 0.353207, + "nauc_map_at_20_max": 0.067782, + "nauc_map_at_20_std": -0.040283, + "nauc_map_at_20_diff1": 0.358749, + "nauc_map_at_100_max": 0.077745, + "nauc_map_at_100_std": -0.020292, + "nauc_map_at_100_diff1": 0.356247, + "nauc_map_at_1000_max": 0.077609, + "nauc_map_at_1000_std": -0.01895, + "nauc_map_at_1000_diff1": 0.355659, + "nauc_recall_at_1_max": 0.023788, + "nauc_recall_at_1_std": -0.103704, + "nauc_recall_at_1_diff1": 0.427193, + "nauc_recall_at_3_max": 0.035469, + "nauc_recall_at_3_std": -0.106663, + "nauc_recall_at_3_diff1": 0.324914, + "nauc_recall_at_5_max": 0.008135, + "nauc_recall_at_5_std": -0.133086, + "nauc_recall_at_5_diff1": 0.319839, + "nauc_recall_at_10_max": 0.033938, + "nauc_recall_at_10_std": -0.082552, + "nauc_recall_at_10_diff1": 0.29212, + "nauc_recall_at_20_max": 0.100009, + "nauc_recall_at_20_std": 0.008662, + "nauc_recall_at_20_diff1": 0.314495, + "nauc_recall_at_100_max": 0.289236, + "nauc_recall_at_100_std": 0.295547, + "nauc_recall_at_100_diff1": 0.340363, + "nauc_recall_at_1000_max": 0.519118, + "nauc_recall_at_1000_std": 0.738843, + "nauc_recall_at_1000_diff1": 0.555566, + "nauc_precision_at_1_max": 0.084585, + "nauc_precision_at_1_std": 0.084742, + "nauc_precision_at_1_diff1": 0.429689, + "nauc_precision_at_3_max": 0.081294, + "nauc_precision_at_3_std": 0.117976, + "nauc_precision_at_3_diff1": 0.089659, + "nauc_precision_at_5_max": 0.040195, + "nauc_precision_at_5_std": 0.111461, + "nauc_precision_at_5_diff1": 0.026214, + "nauc_precision_at_10_max": 0.047423, + "nauc_precision_at_10_std": 0.175005, + "nauc_precision_at_10_diff1": -0.055379, + "nauc_precision_at_20_max": 0.074168, + "nauc_precision_at_20_std": 0.226648, + "nauc_precision_at_20_diff1": -0.084589, + "nauc_precision_at_100_max": 0.073409, + "nauc_precision_at_100_std": 0.257027, + "nauc_precision_at_100_diff1": -0.150089, + "nauc_precision_at_1000_max": 0.031839, + "nauc_precision_at_1000_std": 0.226594, + "nauc_precision_at_1000_diff1": -0.184922, + "nauc_mrr_at_1_max": 0.084585, + "nauc_mrr_at_1_std": 0.084742, + "nauc_mrr_at_1_diff1": 0.429689, + "nauc_mrr_at_3_max": 0.080043, + "nauc_mrr_at_3_std": 0.070464, + "nauc_mrr_at_3_diff1": 0.423431, + "nauc_mrr_at_5_max": 0.073788, + "nauc_mrr_at_5_std": 0.06373, + "nauc_mrr_at_5_diff1": 0.421365, + "nauc_mrr_at_10_max": 0.08175, + "nauc_mrr_at_10_std": 0.072614, + "nauc_mrr_at_10_diff1": 0.426413, + "nauc_mrr_at_20_max": 0.08255, + "nauc_mrr_at_20_std": 0.07588, + "nauc_mrr_at_20_diff1": 0.425716, + "nauc_mrr_at_100_max": 0.082987, + "nauc_mrr_at_100_std": 0.075733, + "nauc_mrr_at_100_diff1": 0.425472, + "nauc_mrr_at_1000_max": 0.082878, + "nauc_mrr_at_1000_std": 0.075575, + "nauc_mrr_at_1000_diff1": 0.425484, + "hit_rate_at_1": 0.6092, + "hit_rate_at_3": 0.81897, + "hit_rate_at_5": 0.87356, + "hit_rate_at_10": 0.92816, + "hit_rate_at_20": 0.95402, + "hit_rate_at_100": 0.98276, + "hit_rate_at_1000": 0.99138, + "main_score": 0.61999, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.10" + } + ] + }, + "evaluation_time": 2755.9326619240455, + "kg_co2_emissions": null, + "date": 1792352824.403651, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 2.163299359381199e-05, + "end": 25.601980820065364, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 25.60208025900647, + "end": 25.602088110987097, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries", + "start": 25.63299456506502, + "end": 26.718799588037655, + "split": "test", + "subset": "french" + }, + { + "name": "Searching corpus", + "start": 26.722043363028206, + "end": 472.1460343230283, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 472.2160293980269, + "end": 472.5149405649863, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 472.5517754780594, + "end": 473.21272720000707, + "split": "test", + "subset": "spanish" + }, + { + "name": "Searching corpus", + "start": 473.2157477190485, + "end": 962.6373774280073, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 962.7017086710548, + "end": 962.9631057210499, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 963.0125461370917, + "end": 963.8453953490825, + "split": "test", + "subset": "english" + }, + { + "name": "Searching corpus", + "start": 963.8483718420612, + "end": 1421.8833465980133, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 1421.9497248220723, + "end": 1422.230438729981, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 1422.2690025680931, + "end": 1422.771108763991, + "split": "test", + "subset": "german" + }, + { + "name": "Searching corpus", + "start": 1422.7740154520143, + "end": 1873.849547113059, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 1873.9262149680872, + "end": 1874.2244836220052, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 1874.2746347890934, + "end": 1874.6711738620652, + "split": "test", + "subset": "italian" + }, + { + "name": "Searching corpus", + "start": 1874.6741593980696, + "end": 2306.5633734490257, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 2306.6270090760663, + "end": 2306.902044377988, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 2306.942134876037, + "end": 2307.622891237028, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Searching corpus", + "start": 2307.625905568013, + "end": 2755.546362298075, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 2755.6120010610903, + "end": 2755.90490829607, + "split": "test", + "subset": "portuguese" + } + ] +} \ No newline at end of file diff --git a/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3TelecomRetrieval.v2.json b/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3TelecomRetrieval.v2.json new file mode 100644 index 0000000000..9cc51c29d1 --- /dev/null +++ b/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/Vidore3TelecomRetrieval.v2.json @@ -0,0 +1,1090 @@ +{ + "dataset_revision": "057fe83342b446ede7928cb51bc6c7c3223da3d0", + "task_name": "Vidore3TelecomRetrieval.v2", + "mteb_version": "2.26.10", + "scores": { + "test": [ + { + "ndcg_at_1": 0.63362, + "ndcg_at_3": 0.59938, + "ndcg_at_5": 0.60964, + "ndcg_at_10": 0.64191, + "ndcg_at_20": 0.67215, + "ndcg_at_100": 0.71575, + "ndcg_at_1000": 0.73138, + "map_at_1": 0.287, + "map_at_3": 0.42694, + "map_at_5": 0.47332, + "map_at_10": 0.5239, + "map_at_20": 0.54973, + "map_at_100": 0.56839, + "map_at_1000": 0.57043, + "recall_at_1": 0.287, + "recall_at_3": 0.4803, + "recall_at_5": 0.56219, + "recall_at_10": 0.69329, + "recall_at_20": 0.78716, + "recall_at_100": 0.92075, + "recall_at_1000": 0.99275, + "accuracy": 0.287, + "precision_at_1": 0.66379, + "precision_at_3": 0.4387, + "precision_at_5": 0.34368, + "precision_at_10": 0.23621, + "precision_at_20": 0.14784, + "precision_at_100": 0.03948, + "precision_at_1000": 0.00443, + "mrr_at_1": 0.663793, + "mrr_at_3": 0.738985, + "mrr_at_5": 0.748611, + "mrr_at_10": 0.757169, + "mrr_at_20": 0.758598, + "mrr_at_100": 0.759453, + "mrr_at_1000": 0.759504, + "nauc_ndcg_at_1_max": 0.02397, + "nauc_ndcg_at_1_std": 0.006972, + "nauc_ndcg_at_1_diff1": 0.457124, + "nauc_ndcg_at_3_max": -0.007257, + "nauc_ndcg_at_3_std": -0.06814, + "nauc_ndcg_at_3_diff1": 0.361294, + "nauc_ndcg_at_5_max": 0.007073, + "nauc_ndcg_at_5_std": -0.061963, + "nauc_ndcg_at_5_diff1": 0.375334, + "nauc_ndcg_at_10_max": 0.047954, + "nauc_ndcg_at_10_std": -0.043959, + "nauc_ndcg_at_10_diff1": 0.402511, + "nauc_ndcg_at_20_max": 0.04765, + "nauc_ndcg_at_20_std": -0.025305, + "nauc_ndcg_at_20_diff1": 0.410328, + "nauc_ndcg_at_100_max": 0.065433, + "nauc_ndcg_at_100_std": 0.018302, + "nauc_ndcg_at_100_diff1": 0.411997, + "nauc_ndcg_at_1000_max": 0.057127, + "nauc_ndcg_at_1000_std": 0.010947, + "nauc_ndcg_at_1000_diff1": 0.411216, + "nauc_map_at_1_max": 0.011159, + "nauc_map_at_1_std": -0.091595, + "nauc_map_at_1_diff1": 0.429776, + "nauc_map_at_3_max": 0.020175, + "nauc_map_at_3_std": -0.094462, + "nauc_map_at_3_diff1": 0.348279, + "nauc_map_at_5_max": 0.018327, + "nauc_map_at_5_std": -0.088501, + "nauc_map_at_5_diff1": 0.35728, + "nauc_map_at_10_max": 0.033162, + "nauc_map_at_10_std": -0.070945, + "nauc_map_at_10_diff1": 0.373751, + "nauc_map_at_20_max": 0.034472, + "nauc_map_at_20_std": -0.052252, + "nauc_map_at_20_diff1": 0.369849, + "nauc_map_at_100_max": 0.04148, + "nauc_map_at_100_std": -0.033443, + "nauc_map_at_100_diff1": 0.367084, + "nauc_map_at_1000_max": 0.040831, + "nauc_map_at_1000_std": -0.033278, + "nauc_map_at_1000_diff1": 0.366949, + "nauc_recall_at_1_max": 0.011159, + "nauc_recall_at_1_std": -0.091595, + "nauc_recall_at_1_diff1": 0.429776, + "nauc_recall_at_3_max": -0.008323, + "nauc_recall_at_3_std": -0.136552, + "nauc_recall_at_3_diff1": 0.287537, + "nauc_recall_at_5_max": 0.009602, + "nauc_recall_at_5_std": -0.124876, + "nauc_recall_at_5_diff1": 0.319876, + "nauc_recall_at_10_max": 0.070929, + "nauc_recall_at_10_std": -0.074844, + "nauc_recall_at_10_diff1": 0.338107, + "nauc_recall_at_20_max": 0.066561, + "nauc_recall_at_20_std": -0.015087, + "nauc_recall_at_20_diff1": 0.33788, + "nauc_recall_at_100_max": 0.21653, + "nauc_recall_at_100_std": 0.246568, + "nauc_recall_at_100_diff1": 0.37137, + "nauc_recall_at_1000_max": 0.50352, + "nauc_recall_at_1000_std": 0.746555, + "nauc_recall_at_1000_diff1": 0.38262, + "nauc_precision_at_1_max": -0.010004, + "nauc_precision_at_1_std": 0.021685, + "nauc_precision_at_1_diff1": 0.452974, + "nauc_precision_at_3_max": -0.008036, + "nauc_precision_at_3_std": 0.021093, + "nauc_precision_at_3_diff1": 0.090756, + "nauc_precision_at_5_max": 0.000528, + "nauc_precision_at_5_std": 0.06881, + "nauc_precision_at_5_diff1": 0.03379, + "nauc_precision_at_10_max": 0.050872, + "nauc_precision_at_10_std": 0.141256, + "nauc_precision_at_10_diff1": -0.017731, + "nauc_precision_at_20_max": 0.039353, + "nauc_precision_at_20_std": 0.181787, + "nauc_precision_at_20_diff1": -0.088409, + "nauc_precision_at_100_max": 0.035917, + "nauc_precision_at_100_std": 0.20392, + "nauc_precision_at_100_diff1": -0.1526, + "nauc_precision_at_1000_max": 0.004714, + "nauc_precision_at_1000_std": 0.17393, + "nauc_precision_at_1000_diff1": -0.182227, + "nauc_mrr_at_1_max": -0.010004, + "nauc_mrr_at_1_std": 0.021685, + "nauc_mrr_at_1_diff1": 0.452974, + "nauc_mrr_at_3_max": -0.029964, + "nauc_mrr_at_3_std": -0.01217, + "nauc_mrr_at_3_diff1": 0.440831, + "nauc_mrr_at_5_max": -0.017026, + "nauc_mrr_at_5_std": 0.002268, + "nauc_mrr_at_5_diff1": 0.453522, + "nauc_mrr_at_10_max": -0.012103, + "nauc_mrr_at_10_std": 0.00892, + "nauc_mrr_at_10_diff1": 0.452447, + "nauc_mrr_at_20_max": -0.012634, + "nauc_mrr_at_20_std": 0.009578, + "nauc_mrr_at_20_diff1": 0.452807, + "nauc_mrr_at_100_max": -0.013574, + "nauc_mrr_at_100_std": 0.008878, + "nauc_mrr_at_100_diff1": 0.452393, + "nauc_mrr_at_1000_max": -0.013528, + "nauc_mrr_at_1000_std": 0.008873, + "nauc_mrr_at_1000_diff1": 0.452347, + "hit_rate_at_1": 0.66379, + "hit_rate_at_3": 0.83046, + "hit_rate_at_5": 0.87356, + "hit_rate_at_10": 0.93966, + "hit_rate_at_20": 0.95977, + "hit_rate_at_100": 0.98563, + "hit_rate_at_1000": 1.0, + "main_score": 0.64191, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.60201, + "ndcg_at_3": 0.58836, + "ndcg_at_5": 0.59961, + "ndcg_at_10": 0.62532, + "ndcg_at_20": 0.6592, + "ndcg_at_100": 0.70267, + "ndcg_at_1000": 0.71956, + "map_at_1": 0.27801, + "map_at_3": 0.41968, + "map_at_5": 0.4663, + "map_at_10": 0.51296, + "map_at_20": 0.53931, + "map_at_100": 0.55726, + "map_at_1000": 0.55958, + "recall_at_1": 0.27801, + "recall_at_3": 0.47036, + "recall_at_5": 0.56471, + "recall_at_10": 0.67295, + "recall_at_20": 0.77338, + "recall_at_100": 0.90997, + "recall_at_1000": 0.98624, + "accuracy": 0.27801, + "precision_at_1": 0.63218, + "precision_at_3": 0.43582, + "precision_at_5": 0.33793, + "precision_at_10": 0.22902, + "precision_at_20": 0.1454, + "precision_at_100": 0.03885, + "precision_at_1000": 0.0044, + "mrr_at_1": 0.632184, + "mrr_at_3": 0.711207, + "mrr_at_5": 0.727011, + "mrr_at_10": 0.731785, + "mrr_at_20": 0.734283, + "mrr_at_100": 0.735421, + "mrr_at_1000": 0.735481, + "nauc_ndcg_at_1_max": 0.071643, + "nauc_ndcg_at_1_std": 0.024357, + "nauc_ndcg_at_1_diff1": 0.50986, + "nauc_ndcg_at_3_max": 0.087651, + "nauc_ndcg_at_3_std": 0.015941, + "nauc_ndcg_at_3_diff1": 0.386788, + "nauc_ndcg_at_5_max": 0.067542, + "nauc_ndcg_at_5_std": -0.027402, + "nauc_ndcg_at_5_diff1": 0.398409, + "nauc_ndcg_at_10_max": 0.096868, + "nauc_ndcg_at_10_std": -0.00927, + "nauc_ndcg_at_10_diff1": 0.405442, + "nauc_ndcg_at_20_max": 0.111197, + "nauc_ndcg_at_20_std": 0.016404, + "nauc_ndcg_at_20_diff1": 0.401179, + "nauc_ndcg_at_100_max": 0.118263, + "nauc_ndcg_at_100_std": 0.043002, + "nauc_ndcg_at_100_diff1": 0.406401, + "nauc_ndcg_at_1000_max": 0.117302, + "nauc_ndcg_at_1000_std": 0.044377, + "nauc_ndcg_at_1000_diff1": 0.408706, + "nauc_map_at_1_max": 0.019001, + "nauc_map_at_1_std": -0.109818, + "nauc_map_at_1_diff1": 0.437862, + "nauc_map_at_3_max": 0.091514, + "nauc_map_at_3_std": -0.06097, + "nauc_map_at_3_diff1": 0.367778, + "nauc_map_at_5_max": 0.082999, + "nauc_map_at_5_std": -0.070451, + "nauc_map_at_5_diff1": 0.384728, + "nauc_map_at_10_max": 0.092388, + "nauc_map_at_10_std": -0.041955, + "nauc_map_at_10_diff1": 0.384092, + "nauc_map_at_20_max": 0.102992, + "nauc_map_at_20_std": -0.01944, + "nauc_map_at_20_diff1": 0.379952, + "nauc_map_at_100_max": 0.107648, + "nauc_map_at_100_std": -0.004368, + "nauc_map_at_100_diff1": 0.379045, + "nauc_map_at_1000_max": 0.107447, + "nauc_map_at_1000_std": -0.003245, + "nauc_map_at_1000_diff1": 0.378724, + "nauc_recall_at_1_max": 0.019001, + "nauc_recall_at_1_std": -0.109818, + "nauc_recall_at_1_diff1": 0.437862, + "nauc_recall_at_3_max": 0.074185, + "nauc_recall_at_3_std": -0.084963, + "nauc_recall_at_3_diff1": 0.313446, + "nauc_recall_at_5_max": 0.048362, + "nauc_recall_at_5_std": -0.115953, + "nauc_recall_at_5_diff1": 0.313963, + "nauc_recall_at_10_max": 0.08619, + "nauc_recall_at_10_std": -0.052972, + "nauc_recall_at_10_diff1": 0.307752, + "nauc_recall_at_20_max": 0.139532, + "nauc_recall_at_20_std": 0.034261, + "nauc_recall_at_20_diff1": 0.290737, + "nauc_recall_at_100_max": 0.207255, + "nauc_recall_at_100_std": 0.20027, + "nauc_recall_at_100_diff1": 0.315751, + "nauc_recall_at_1000_max": 0.56114, + "nauc_recall_at_1000_std": 0.625258, + "nauc_recall_at_1000_diff1": 0.372138, + "nauc_precision_at_1_max": 0.043416, + "nauc_precision_at_1_std": 0.028603, + "nauc_precision_at_1_diff1": 0.494834, + "nauc_precision_at_3_max": 0.098916, + "nauc_precision_at_3_std": 0.130701, + "nauc_precision_at_3_diff1": 0.115176, + "nauc_precision_at_5_max": 0.059233, + "nauc_precision_at_5_std": 0.110071, + "nauc_precision_at_5_diff1": 0.063462, + "nauc_precision_at_10_max": 0.073033, + "nauc_precision_at_10_std": 0.17443, + "nauc_precision_at_10_diff1": -0.008657, + "nauc_precision_at_20_max": 0.065372, + "nauc_precision_at_20_std": 0.205931, + "nauc_precision_at_20_diff1": -0.095371, + "nauc_precision_at_100_max": 0.038269, + "nauc_precision_at_100_std": 0.206833, + "nauc_precision_at_100_diff1": -0.159324, + "nauc_precision_at_1000_max": 0.008904, + "nauc_precision_at_1000_std": 0.179879, + "nauc_precision_at_1000_diff1": -0.194236, + "nauc_mrr_at_1_max": 0.043416, + "nauc_mrr_at_1_std": 0.028603, + "nauc_mrr_at_1_diff1": 0.494834, + "nauc_mrr_at_3_max": 0.04929, + "nauc_mrr_at_3_std": 0.042864, + "nauc_mrr_at_3_diff1": 0.474021, + "nauc_mrr_at_5_max": 0.046214, + "nauc_mrr_at_5_std": 0.043171, + "nauc_mrr_at_5_diff1": 0.467505, + "nauc_mrr_at_10_max": 0.058321, + "nauc_mrr_at_10_std": 0.04677, + "nauc_mrr_at_10_diff1": 0.473504, + "nauc_mrr_at_20_max": 0.0577, + "nauc_mrr_at_20_std": 0.048191, + "nauc_mrr_at_20_diff1": 0.471954, + "nauc_mrr_at_100_max": 0.057835, + "nauc_mrr_at_100_std": 0.048544, + "nauc_mrr_at_100_diff1": 0.472691, + "nauc_mrr_at_1000_max": 0.057795, + "nauc_mrr_at_1000_std": 0.04845, + "nauc_mrr_at_1000_diff1": 0.47271, + "hit_rate_at_1": 0.63218, + "hit_rate_at_3": 0.8046, + "hit_rate_at_5": 0.87069, + "hit_rate_at_10": 0.90517, + "hit_rate_at_20": 0.93966, + "hit_rate_at_100": 0.98276, + "hit_rate_at_1000": 0.99713, + "main_score": 0.62532, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.68822, + "ndcg_at_3": 0.6474, + "ndcg_at_5": 0.66403, + "ndcg_at_10": 0.69438, + "ndcg_at_20": 0.72323, + "ndcg_at_100": 0.76135, + "ndcg_at_1000": 0.77309, + "map_at_1": 0.31816, + "map_at_3": 0.46775, + "map_at_5": 0.52501, + "map_at_10": 0.57975, + "map_at_20": 0.60681, + "map_at_100": 0.62426, + "map_at_1000": 0.62605, + "recall_at_1": 0.31816, + "recall_at_3": 0.50633, + "recall_at_5": 0.6112, + "recall_at_10": 0.73851, + "recall_at_20": 0.82784, + "recall_at_100": 0.94002, + "recall_at_1000": 0.99126, + "accuracy": 0.31816, + "precision_at_1": 0.72126, + "precision_at_3": 0.47222, + "precision_at_5": 0.37931, + "precision_at_10": 0.2569, + "precision_at_20": 0.15776, + "precision_at_100": 0.0406, + "precision_at_1000": 0.00444, + "mrr_at_1": 0.721264, + "mrr_at_3": 0.780172, + "mrr_at_5": 0.794971, + "mrr_at_10": 0.801367, + "mrr_at_20": 0.802428, + "mrr_at_100": 0.802795, + "mrr_at_1000": 0.802829, + "nauc_ndcg_at_1_max": 0.176079, + "nauc_ndcg_at_1_std": 0.001845, + "nauc_ndcg_at_1_diff1": 0.412525, + "nauc_ndcg_at_3_max": 0.172978, + "nauc_ndcg_at_3_std": -0.01263, + "nauc_ndcg_at_3_diff1": 0.328604, + "nauc_ndcg_at_5_max": 0.187237, + "nauc_ndcg_at_5_std": 0.005787, + "nauc_ndcg_at_5_diff1": 0.356736, + "nauc_ndcg_at_10_max": 0.211312, + "nauc_ndcg_at_10_std": 0.005068, + "nauc_ndcg_at_10_diff1": 0.367209, + "nauc_ndcg_at_20_max": 0.210831, + "nauc_ndcg_at_20_std": 0.004037, + "nauc_ndcg_at_20_diff1": 0.362265, + "nauc_ndcg_at_100_max": 0.221743, + "nauc_ndcg_at_100_std": 0.046688, + "nauc_ndcg_at_100_diff1": 0.379204, + "nauc_ndcg_at_1000_max": 0.2129, + "nauc_ndcg_at_1000_std": 0.042155, + "nauc_ndcg_at_1000_diff1": 0.369337, + "nauc_map_at_1_max": 0.124823, + "nauc_map_at_1_std": -0.099133, + "nauc_map_at_1_diff1": 0.411499, + "nauc_map_at_3_max": 0.189107, + "nauc_map_at_3_std": -0.051751, + "nauc_map_at_3_diff1": 0.36132, + "nauc_map_at_5_max": 0.184902, + "nauc_map_at_5_std": -0.037781, + "nauc_map_at_5_diff1": 0.364071, + "nauc_map_at_10_max": 0.193564, + "nauc_map_at_10_std": -0.032119, + "nauc_map_at_10_diff1": 0.355434, + "nauc_map_at_20_max": 0.196293, + "nauc_map_at_20_std": -0.024126, + "nauc_map_at_20_diff1": 0.347449, + "nauc_map_at_100_max": 0.201004, + "nauc_map_at_100_std": -0.006821, + "nauc_map_at_100_diff1": 0.348474, + "nauc_map_at_1000_max": 0.200521, + "nauc_map_at_1000_std": -0.00573, + "nauc_map_at_1000_diff1": 0.34794, + "nauc_recall_at_1_max": 0.124823, + "nauc_recall_at_1_std": -0.099133, + "nauc_recall_at_1_diff1": 0.411499, + "nauc_recall_at_3_max": 0.185128, + "nauc_recall_at_3_std": -0.055566, + "nauc_recall_at_3_diff1": 0.334434, + "nauc_recall_at_5_max": 0.187391, + "nauc_recall_at_5_std": -0.016972, + "nauc_recall_at_5_diff1": 0.336789, + "nauc_recall_at_10_max": 0.218696, + "nauc_recall_at_10_std": -0.009314, + "nauc_recall_at_10_diff1": 0.305455, + "nauc_recall_at_20_max": 0.235872, + "nauc_recall_at_20_std": 0.003862, + "nauc_recall_at_20_diff1": 0.269619, + "nauc_recall_at_100_max": 0.437018, + "nauc_recall_at_100_std": 0.339361, + "nauc_recall_at_100_diff1": 0.406885, + "nauc_recall_at_1000_max": 0.904892, + "nauc_recall_at_1000_std": 0.906546, + "nauc_recall_at_1000_diff1": 0.294698, + "nauc_precision_at_1_max": 0.172946, + "nauc_precision_at_1_std": 0.030001, + "nauc_precision_at_1_diff1": 0.409446, + "nauc_precision_at_3_max": 0.100184, + "nauc_precision_at_3_std": 0.080552, + "nauc_precision_at_3_diff1": 0.017116, + "nauc_precision_at_5_max": 0.05381, + "nauc_precision_at_5_std": 0.115171, + "nauc_precision_at_5_diff1": -0.050246, + "nauc_precision_at_10_max": 0.014619, + "nauc_precision_at_10_std": 0.112752, + "nauc_precision_at_10_diff1": -0.136383, + "nauc_precision_at_20_max": -0.024144, + "nauc_precision_at_20_std": 0.128699, + "nauc_precision_at_20_diff1": -0.189452, + "nauc_precision_at_100_max": -0.056691, + "nauc_precision_at_100_std": 0.170695, + "nauc_precision_at_100_diff1": -0.209146, + "nauc_precision_at_1000_max": -0.086512, + "nauc_precision_at_1000_std": 0.156785, + "nauc_precision_at_1000_diff1": -0.237953, + "nauc_mrr_at_1_max": 0.172946, + "nauc_mrr_at_1_std": 0.030001, + "nauc_mrr_at_1_diff1": 0.409446, + "nauc_mrr_at_3_max": 0.159778, + "nauc_mrr_at_3_std": 0.03908, + "nauc_mrr_at_3_diff1": 0.403292, + "nauc_mrr_at_5_max": 0.189894, + "nauc_mrr_at_5_std": 0.064556, + "nauc_mrr_at_5_diff1": 0.41232, + "nauc_mrr_at_10_max": 0.184275, + "nauc_mrr_at_10_std": 0.054317, + "nauc_mrr_at_10_diff1": 0.40707, + "nauc_mrr_at_20_max": 0.182122, + "nauc_mrr_at_20_std": 0.054892, + "nauc_mrr_at_20_diff1": 0.404695, + "nauc_mrr_at_100_max": 0.182111, + "nauc_mrr_at_100_std": 0.053808, + "nauc_mrr_at_100_diff1": 0.406006, + "nauc_mrr_at_1000_max": 0.18209, + "nauc_mrr_at_1000_std": 0.053815, + "nauc_mrr_at_1000_diff1": 0.405958, + "hit_rate_at_1": 0.72126, + "hit_rate_at_3": 0.85345, + "hit_rate_at_5": 0.91667, + "hit_rate_at_10": 0.96264, + "hit_rate_at_20": 0.97701, + "hit_rate_at_100": 0.98851, + "hit_rate_at_1000": 0.99713, + "main_score": 0.69438, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.62787, + "ndcg_at_3": 0.59412, + "ndcg_at_5": 0.60586, + "ndcg_at_10": 0.62819, + "ndcg_at_20": 0.66541, + "ndcg_at_100": 0.71116, + "ndcg_at_1000": 0.72662, + "map_at_1": 0.29613, + "map_at_3": 0.42493, + "map_at_5": 0.47148, + "map_at_10": 0.51561, + "map_at_20": 0.54453, + "map_at_100": 0.56231, + "map_at_1000": 0.56446, + "recall_at_1": 0.29613, + "recall_at_3": 0.46639, + "recall_at_5": 0.5602, + "recall_at_10": 0.66203, + "recall_at_20": 0.77148, + "recall_at_100": 0.91854, + "recall_at_1000": 0.98824, + "accuracy": 0.29613, + "precision_at_1": 0.65805, + "precision_at_3": 0.43103, + "precision_at_5": 0.33736, + "precision_at_10": 0.22557, + "precision_at_20": 0.14583, + "precision_at_100": 0.03908, + "precision_at_1000": 0.00442, + "mrr_at_1": 0.658046, + "mrr_at_3": 0.73228, + "mrr_at_5": 0.743343, + "mrr_at_10": 0.749412, + "mrr_at_20": 0.752398, + "mrr_at_100": 0.753636, + "mrr_at_1000": 0.753649, + "nauc_ndcg_at_1_max": -0.04203, + "nauc_ndcg_at_1_std": -0.014245, + "nauc_ndcg_at_1_diff1": 0.402895, + "nauc_ndcg_at_3_max": -0.020342, + "nauc_ndcg_at_3_std": -0.00483, + "nauc_ndcg_at_3_diff1": 0.351627, + "nauc_ndcg_at_5_max": 0.006633, + "nauc_ndcg_at_5_std": -0.010683, + "nauc_ndcg_at_5_diff1": 0.34681, + "nauc_ndcg_at_10_max": 0.016852, + "nauc_ndcg_at_10_std": -0.002631, + "nauc_ndcg_at_10_diff1": 0.363957, + "nauc_ndcg_at_20_max": 0.03843, + "nauc_ndcg_at_20_std": 0.016433, + "nauc_ndcg_at_20_diff1": 0.370971, + "nauc_ndcg_at_100_max": 0.047134, + "nauc_ndcg_at_100_std": 0.04327, + "nauc_ndcg_at_100_diff1": 0.388908, + "nauc_ndcg_at_1000_max": 0.041362, + "nauc_ndcg_at_1000_std": 0.043245, + "nauc_ndcg_at_1000_diff1": 0.380884, + "nauc_map_at_1_max": -0.014526, + "nauc_map_at_1_std": -0.1134, + "nauc_map_at_1_diff1": 0.419022, + "nauc_map_at_3_max": 0.002071, + "nauc_map_at_3_std": -0.073955, + "nauc_map_at_3_diff1": 0.375075, + "nauc_map_at_5_max": 0.022555, + "nauc_map_at_5_std": -0.061965, + "nauc_map_at_5_diff1": 0.370576, + "nauc_map_at_10_max": 0.023269, + "nauc_map_at_10_std": -0.03924, + "nauc_map_at_10_diff1": 0.376401, + "nauc_map_at_20_max": 0.030869, + "nauc_map_at_20_std": -0.022838, + "nauc_map_at_20_diff1": 0.374711, + "nauc_map_at_100_max": 0.035911, + "nauc_map_at_100_std": -0.009544, + "nauc_map_at_100_diff1": 0.37701, + "nauc_map_at_1000_max": 0.035594, + "nauc_map_at_1000_std": -0.0084, + "nauc_map_at_1000_diff1": 0.376343, + "nauc_recall_at_1_max": -0.014526, + "nauc_recall_at_1_std": -0.1134, + "nauc_recall_at_1_diff1": 0.419022, + "nauc_recall_at_3_max": -0.004779, + "nauc_recall_at_3_std": -0.071544, + "nauc_recall_at_3_diff1": 0.326879, + "nauc_recall_at_5_max": 0.03895, + "nauc_recall_at_5_std": -0.066596, + "nauc_recall_at_5_diff1": 0.297784, + "nauc_recall_at_10_max": 0.026327, + "nauc_recall_at_10_std": -0.039725, + "nauc_recall_at_10_diff1": 0.30606, + "nauc_recall_at_20_max": 0.098936, + "nauc_recall_at_20_std": 0.025959, + "nauc_recall_at_20_diff1": 0.294082, + "nauc_recall_at_100_max": 0.194644, + "nauc_recall_at_100_std": 0.198553, + "nauc_recall_at_100_diff1": 0.394227, + "nauc_recall_at_1000_max": 0.467861, + "nauc_recall_at_1000_std": 0.568446, + "nauc_recall_at_1000_diff1": 0.12179, + "nauc_precision_at_1_max": -0.07349, + "nauc_precision_at_1_std": 0.006881, + "nauc_precision_at_1_diff1": 0.399716, + "nauc_precision_at_3_max": 0.014499, + "nauc_precision_at_3_std": 0.134591, + "nauc_precision_at_3_diff1": 0.083865, + "nauc_precision_at_5_max": 0.040905, + "nauc_precision_at_5_std": 0.161274, + "nauc_precision_at_5_diff1": 0.0037, + "nauc_precision_at_10_max": 0.053077, + "nauc_precision_at_10_std": 0.200225, + "nauc_precision_at_10_diff1": -0.031408, + "nauc_precision_at_20_max": 0.065104, + "nauc_precision_at_20_std": 0.203169, + "nauc_precision_at_20_diff1": -0.082061, + "nauc_precision_at_100_max": 0.057838, + "nauc_precision_at_100_std": 0.214336, + "nauc_precision_at_100_diff1": -0.128793, + "nauc_precision_at_1000_max": 0.021864, + "nauc_precision_at_1000_std": 0.179529, + "nauc_precision_at_1000_diff1": -0.17201, + "nauc_mrr_at_1_max": -0.07349, + "nauc_mrr_at_1_std": 0.006881, + "nauc_mrr_at_1_diff1": 0.399716, + "nauc_mrr_at_3_max": -0.067634, + "nauc_mrr_at_3_std": 0.040788, + "nauc_mrr_at_3_diff1": 0.355298, + "nauc_mrr_at_5_max": -0.060793, + "nauc_mrr_at_5_std": 0.040116, + "nauc_mrr_at_5_diff1": 0.357389, + "nauc_mrr_at_10_max": -0.063415, + "nauc_mrr_at_10_std": 0.039838, + "nauc_mrr_at_10_diff1": 0.367691, + "nauc_mrr_at_20_max": -0.059689, + "nauc_mrr_at_20_std": 0.039741, + "nauc_mrr_at_20_diff1": 0.367518, + "nauc_mrr_at_100_max": -0.060864, + "nauc_mrr_at_100_std": 0.038854, + "nauc_mrr_at_100_diff1": 0.367504, + "nauc_mrr_at_1000_max": -0.060852, + "nauc_mrr_at_1000_std": 0.038852, + "nauc_mrr_at_1000_diff1": 0.367464, + "hit_rate_at_1": 0.65805, + "hit_rate_at_3": 0.81897, + "hit_rate_at_5": 0.86782, + "hit_rate_at_10": 0.90805, + "hit_rate_at_20": 0.94828, + "hit_rate_at_100": 0.99138, + "hit_rate_at_1000": 0.99713, + "main_score": 0.62819, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.60632, + "ndcg_at_3": 0.58969, + "ndcg_at_5": 0.60069, + "ndcg_at_10": 0.62413, + "ndcg_at_20": 0.66108, + "ndcg_at_100": 0.70432, + "ndcg_at_1000": 0.72113, + "map_at_1": 0.27538, + "map_at_3": 0.42249, + "map_at_5": 0.46905, + "map_at_10": 0.51081, + "map_at_20": 0.53956, + "map_at_100": 0.55727, + "map_at_1000": 0.55953, + "recall_at_1": 0.27538, + "recall_at_3": 0.47552, + "recall_at_5": 0.56333, + "recall_at_10": 0.66832, + "recall_at_20": 0.77657, + "recall_at_100": 0.91281, + "recall_at_1000": 0.98623, + "accuracy": 0.27538, + "precision_at_1": 0.63793, + "precision_at_3": 0.43295, + "precision_at_5": 0.33793, + "precision_at_10": 0.22328, + "precision_at_20": 0.14483, + "precision_at_100": 0.03882, + "precision_at_1000": 0.00441, + "mrr_at_1": 0.637931, + "mrr_at_3": 0.713602, + "mrr_at_5": 0.726676, + "mrr_at_10": 0.73387, + "mrr_at_20": 0.737526, + "mrr_at_100": 0.738255, + "mrr_at_1000": 0.73832, + "nauc_ndcg_at_1_max": 0.046308, + "nauc_ndcg_at_1_std": 0.034499, + "nauc_ndcg_at_1_diff1": 0.475836, + "nauc_ndcg_at_3_max": 0.027972, + "nauc_ndcg_at_3_std": -0.001774, + "nauc_ndcg_at_3_diff1": 0.363216, + "nauc_ndcg_at_5_max": 0.030522, + "nauc_ndcg_at_5_std": -0.006401, + "nauc_ndcg_at_5_diff1": 0.360814, + "nauc_ndcg_at_10_max": 0.04693, + "nauc_ndcg_at_10_std": -0.008219, + "nauc_ndcg_at_10_diff1": 0.357593, + "nauc_ndcg_at_20_max": 0.063878, + "nauc_ndcg_at_20_std": 0.004532, + "nauc_ndcg_at_20_diff1": 0.365838, + "nauc_ndcg_at_100_max": 0.08106, + "nauc_ndcg_at_100_std": 0.051712, + "nauc_ndcg_at_100_diff1": 0.384166, + "nauc_ndcg_at_1000_max": 0.073562, + "nauc_ndcg_at_1000_std": 0.047457, + "nauc_ndcg_at_1000_diff1": 0.374293, + "nauc_map_at_1_max": 0.024873, + "nauc_map_at_1_std": -0.071939, + "nauc_map_at_1_diff1": 0.411993, + "nauc_map_at_3_max": 0.052142, + "nauc_map_at_3_std": -0.048786, + "nauc_map_at_3_diff1": 0.332253, + "nauc_map_at_5_max": 0.054218, + "nauc_map_at_5_std": -0.036269, + "nauc_map_at_5_diff1": 0.330878, + "nauc_map_at_10_max": 0.053744, + "nauc_map_at_10_std": -0.030121, + "nauc_map_at_10_diff1": 0.328742, + "nauc_map_at_20_max": 0.064402, + "nauc_map_at_20_std": -0.015027, + "nauc_map_at_20_diff1": 0.334859, + "nauc_map_at_100_max": 0.071873, + "nauc_map_at_100_std": 0.002824, + "nauc_map_at_100_diff1": 0.340037, + "nauc_map_at_1000_max": 0.071568, + "nauc_map_at_1000_std": 0.003629, + "nauc_map_at_1000_diff1": 0.338673, + "nauc_recall_at_1_max": 0.024873, + "nauc_recall_at_1_std": -0.071939, + "nauc_recall_at_1_diff1": 0.411993, + "nauc_recall_at_3_max": 0.032023, + "nauc_recall_at_3_std": -0.074017, + "nauc_recall_at_3_diff1": 0.287628, + "nauc_recall_at_5_max": 0.02675, + "nauc_recall_at_5_std": -0.071551, + "nauc_recall_at_5_diff1": 0.27849, + "nauc_recall_at_10_max": 0.062349, + "nauc_recall_at_10_std": -0.046222, + "nauc_recall_at_10_diff1": 0.260954, + "nauc_recall_at_20_max": 0.108817, + "nauc_recall_at_20_std": -0.009418, + "nauc_recall_at_20_diff1": 0.265645, + "nauc_recall_at_100_max": 0.251606, + "nauc_recall_at_100_std": 0.279917, + "nauc_recall_at_100_diff1": 0.401492, + "nauc_recall_at_1000_max": 0.548942, + "nauc_recall_at_1000_std": 0.776035, + "nauc_recall_at_1000_diff1": 0.241201, + "nauc_precision_at_1_max": 0.027133, + "nauc_precision_at_1_std": 0.04523, + "nauc_precision_at_1_diff1": 0.482355, + "nauc_precision_at_3_max": 0.053562, + "nauc_precision_at_3_std": 0.08995, + "nauc_precision_at_3_diff1": 0.136104, + "nauc_precision_at_5_max": 0.040884, + "nauc_precision_at_5_std": 0.116938, + "nauc_precision_at_5_diff1": 0.052179, + "nauc_precision_at_10_max": 0.035427, + "nauc_precision_at_10_std": 0.122685, + "nauc_precision_at_10_diff1": -0.018204, + "nauc_precision_at_20_max": 0.054116, + "nauc_precision_at_20_std": 0.151722, + "nauc_precision_at_20_diff1": -0.062628, + "nauc_precision_at_100_max": 0.033037, + "nauc_precision_at_100_std": 0.166804, + "nauc_precision_at_100_diff1": -0.114848, + "nauc_precision_at_1000_max": 0.001742, + "nauc_precision_at_1000_std": 0.137024, + "nauc_precision_at_1000_diff1": -0.170914, + "nauc_mrr_at_1_max": 0.027133, + "nauc_mrr_at_1_std": 0.04523, + "nauc_mrr_at_1_diff1": 0.482355, + "nauc_mrr_at_3_max": 0.001716, + "nauc_mrr_at_3_std": 0.045198, + "nauc_mrr_at_3_diff1": 0.466773, + "nauc_mrr_at_5_max": 0.000953, + "nauc_mrr_at_5_std": 0.043241, + "nauc_mrr_at_5_diff1": 0.466862, + "nauc_mrr_at_10_max": 0.004464, + "nauc_mrr_at_10_std": 0.046187, + "nauc_mrr_at_10_diff1": 0.461563, + "nauc_mrr_at_20_max": 0.006508, + "nauc_mrr_at_20_std": 0.047433, + "nauc_mrr_at_20_diff1": 0.463183, + "nauc_mrr_at_100_max": 0.007692, + "nauc_mrr_at_100_std": 0.048925, + "nauc_mrr_at_100_diff1": 0.463623, + "nauc_mrr_at_1000_max": 0.00749, + "nauc_mrr_at_1000_std": 0.048685, + "nauc_mrr_at_1000_diff1": 0.46355, + "hit_rate_at_1": 0.63793, + "hit_rate_at_3": 0.79885, + "hit_rate_at_5": 0.85345, + "hit_rate_at_10": 0.90517, + "hit_rate_at_20": 0.9569, + "hit_rate_at_100": 0.97989, + "hit_rate_at_1000": 0.99713, + "main_score": 0.62413, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.63937, + "ndcg_at_3": 0.61363, + "ndcg_at_5": 0.62045, + "ndcg_at_10": 0.64932, + "ndcg_at_20": 0.68269, + "ndcg_at_100": 0.72403, + "ndcg_at_1000": 0.73907, + "map_at_1": 0.29879, + "map_at_3": 0.44096, + "map_at_5": 0.48781, + "map_at_10": 0.53698, + "map_at_20": 0.56437, + "map_at_100": 0.58226, + "map_at_1000": 0.58428, + "recall_at_1": 0.29879, + "recall_at_3": 0.48508, + "recall_at_5": 0.56932, + "recall_at_10": 0.69103, + "recall_at_20": 0.7919, + "recall_at_100": 0.91747, + "recall_at_1000": 0.98525, + "accuracy": 0.29879, + "precision_at_1": 0.66954, + "precision_at_3": 0.44828, + "precision_at_5": 0.34713, + "precision_at_10": 0.23678, + "precision_at_20": 0.14899, + "precision_at_100": 0.0394, + "precision_at_1000": 0.0044, + "mrr_at_1": 0.66954, + "mrr_at_3": 0.741858, + "mrr_at_5": 0.753927, + "mrr_at_10": 0.761355, + "mrr_at_20": 0.763825, + "mrr_at_100": 0.764393, + "mrr_at_1000": 0.764456, + "nauc_ndcg_at_1_max": 0.06138, + "nauc_ndcg_at_1_std": 0.049497, + "nauc_ndcg_at_1_diff1": 0.432921, + "nauc_ndcg_at_3_max": 0.051862, + "nauc_ndcg_at_3_std": -0.001456, + "nauc_ndcg_at_3_diff1": 0.356293, + "nauc_ndcg_at_5_max": 0.030354, + "nauc_ndcg_at_5_std": -0.038746, + "nauc_ndcg_at_5_diff1": 0.356037, + "nauc_ndcg_at_10_max": 0.05299, + "nauc_ndcg_at_10_std": -0.041325, + "nauc_ndcg_at_10_diff1": 0.380869, + "nauc_ndcg_at_20_max": 0.06103, + "nauc_ndcg_at_20_std": -0.020995, + "nauc_ndcg_at_20_diff1": 0.388545, + "nauc_ndcg_at_100_max": 0.081036, + "nauc_ndcg_at_100_std": 0.008679, + "nauc_ndcg_at_100_diff1": 0.400146, + "nauc_ndcg_at_1000_max": 0.07954, + "nauc_ndcg_at_1000_std": 0.019766, + "nauc_ndcg_at_1000_diff1": 0.396488, + "nauc_map_at_1_max": -0.011796, + "nauc_map_at_1_std": -0.10748, + "nauc_map_at_1_diff1": 0.381565, + "nauc_map_at_3_max": 0.043349, + "nauc_map_at_3_std": -0.086139, + "nauc_map_at_3_diff1": 0.342621, + "nauc_map_at_5_max": 0.033747, + "nauc_map_at_5_std": -0.087585, + "nauc_map_at_5_diff1": 0.332053, + "nauc_map_at_10_max": 0.040342, + "nauc_map_at_10_std": -0.078519, + "nauc_map_at_10_diff1": 0.347021, + "nauc_map_at_20_max": 0.047797, + "nauc_map_at_20_std": -0.054569, + "nauc_map_at_20_diff1": 0.346655, + "nauc_map_at_100_max": 0.057972, + "nauc_map_at_100_std": -0.039585, + "nauc_map_at_100_diff1": 0.349459, + "nauc_map_at_1000_max": 0.058595, + "nauc_map_at_1000_std": -0.037282, + "nauc_map_at_1000_diff1": 0.349352, + "nauc_recall_at_1_max": -0.011796, + "nauc_recall_at_1_std": -0.10748, + "nauc_recall_at_1_diff1": 0.381565, + "nauc_recall_at_3_max": 0.027629, + "nauc_recall_at_3_std": -0.113457, + "nauc_recall_at_3_diff1": 0.311988, + "nauc_recall_at_5_max": 0.003091, + "nauc_recall_at_5_std": -0.142095, + "nauc_recall_at_5_diff1": 0.29179, + "nauc_recall_at_10_max": 0.054363, + "nauc_recall_at_10_std": -0.093856, + "nauc_recall_at_10_diff1": 0.325507, + "nauc_recall_at_20_max": 0.097398, + "nauc_recall_at_20_std": -0.013513, + "nauc_recall_at_20_diff1": 0.342253, + "nauc_recall_at_100_max": 0.219688, + "nauc_recall_at_100_std": 0.105841, + "nauc_recall_at_100_diff1": 0.444333, + "nauc_recall_at_1000_max": 0.588235, + "nauc_recall_at_1000_std": 0.691321, + "nauc_recall_at_1000_diff1": 0.639104, + "nauc_precision_at_1_max": 0.022965, + "nauc_precision_at_1_std": 0.044196, + "nauc_precision_at_1_diff1": 0.430205, + "nauc_precision_at_3_max": 0.06531, + "nauc_precision_at_3_std": 0.096658, + "nauc_precision_at_3_diff1": 0.09021, + "nauc_precision_at_5_max": 0.036794, + "nauc_precision_at_5_std": 0.099167, + "nauc_precision_at_5_diff1": 0.024285, + "nauc_precision_at_10_max": 0.056278, + "nauc_precision_at_10_std": 0.139454, + "nauc_precision_at_10_diff1": -0.016199, + "nauc_precision_at_20_max": 0.063965, + "nauc_precision_at_20_std": 0.194943, + "nauc_precision_at_20_diff1": -0.079503, + "nauc_precision_at_100_max": 0.063711, + "nauc_precision_at_100_std": 0.211284, + "nauc_precision_at_100_diff1": -0.120342, + "nauc_precision_at_1000_max": 0.036296, + "nauc_precision_at_1000_std": 0.202765, + "nauc_precision_at_1000_diff1": -0.160727, + "nauc_mrr_at_1_max": 0.022965, + "nauc_mrr_at_1_std": 0.044196, + "nauc_mrr_at_1_diff1": 0.430205, + "nauc_mrr_at_3_max": -0.015143, + "nauc_mrr_at_3_std": 0.005809, + "nauc_mrr_at_3_diff1": 0.419599, + "nauc_mrr_at_5_max": -0.008686, + "nauc_mrr_at_5_std": 0.003723, + "nauc_mrr_at_5_diff1": 0.435105, + "nauc_mrr_at_10_max": 0.003046, + "nauc_mrr_at_10_std": 0.014793, + "nauc_mrr_at_10_diff1": 0.431833, + "nauc_mrr_at_20_max": 0.005389, + "nauc_mrr_at_20_std": 0.018425, + "nauc_mrr_at_20_diff1": 0.431823, + "nauc_mrr_at_100_max": 0.00378, + "nauc_mrr_at_100_std": 0.017776, + "nauc_mrr_at_100_diff1": 0.431349, + "nauc_mrr_at_1000_max": 0.003897, + "nauc_mrr_at_1000_std": 0.01783, + "nauc_mrr_at_1000_diff1": 0.43144, + "hit_rate_at_1": 0.66954, + "hit_rate_at_3": 0.82184, + "hit_rate_at_5": 0.87356, + "hit_rate_at_10": 0.92816, + "hit_rate_at_20": 0.96264, + "hit_rate_at_100": 0.98276, + "hit_rate_at_1000": 0.99425, + "main_score": 0.64932, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.10" + } + ] + }, + "evaluation_time": 2857.0802892658394, + "kg_co2_emissions": null, + "date": 1792352925.591118, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 1.2928969226777554e-05, + "end": 24.78183574799914, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 24.78188941802364, + "end": 24.78189173201099, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries", + "start": 24.807307317038067, + "end": 25.759752779034898, + "split": "test", + "subset": "french" + }, + { + "name": "Searching corpus", + "start": 25.762974540004507, + "end": 510.13994995795656, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 510.21532206900883, + "end": 510.4577172040008, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 510.4947150370572, + "end": 511.2679652980296, + "split": "test", + "subset": "spanish" + }, + { + "name": "Searching corpus", + "start": 511.2709352209931, + "end": 1049.3982696390012, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 1049.4570481410483, + "end": 1049.6911341450177, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 1049.729298420949, + "end": 1051.4276310879504, + "split": "test", + "subset": "english" + }, + { + "name": "Searching corpus", + "start": 1051.4305586270057, + "end": 1567.5650219490053, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 1567.622977563995, + "end": 1567.8695221790113, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 1567.9116447620327, + "end": 1568.3396037500352, + "split": "test", + "subset": "german" + }, + { + "name": "Searching corpus", + "start": 1568.343681444996, + "end": 2023.799873262993, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 2023.8495609130478, + "end": 2024.0674553350545, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 2024.1065310239792, + "end": 2024.4055881790118, + "split": "test", + "subset": "italian" + }, + { + "name": "Searching corpus", + "start": 2024.4084371699719, + "end": 2455.414563547005, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 2455.469862855971, + "end": 2455.6925263589947, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 2455.734484763001, + "end": 2456.0280610859627, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Searching corpus", + "start": 2456.03106606903, + "end": 2856.766813580063, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 2856.818098496995, + "end": 2857.058862337959, + "split": "test", + "subset": "portuguese" + } + ] +} \ No newline at end of file diff --git a/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/run_settings.jsonl b/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/run_settings.jsonl index 8ae33e8575..371958a91a 100644 --- a/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/run_settings.jsonl +++ b/results/athrael-soju__ColUltraVBERT/8f85c126eb97dcabe8faf69b2208d8d471eb1214/run_settings.jsonl @@ -20,3 +20,7 @@ {"task": "Vidore3ComputerScienceRetrieval", "splits": ["test"], "version": {"mteb": "2.26.3", "torch": "2.14.0+cu130", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3PhysicsRetrieval", "splits": ["test"], "version": {"mteb": "2.26.3", "torch": "2.14.0+cu130", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3HrRetrieval", "splits": ["test"], "version": {"mteb": "2.26.3", "torch": "2.14.0+cu130", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3TelecomRetrieval", "splits": ["test"], "version": {"mteb": "2.26.10", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3NuclearRetrieval", "splits": ["test"], "version": {"mteb": "2.26.10", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3TelecomRetrieval.v2", "splits": ["test"], "version": {"mteb": "2.26.10", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3NuclearRetrieval.v2", "splits": ["test"], "version": {"mteb": "2.26.10", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]}