diff --git a/results/BAAI__bge-reranker-v2-m3/953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e/Vidore3NuclearNemotron35Embed8BReranking.v2.json b/results/BAAI__bge-reranker-v2-m3/953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e/Vidore3NuclearNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..f091b41aa2 --- /dev/null +++ b/results/BAAI__bge-reranker-v2-m3/953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e/Vidore3NuclearNemotron35Embed8BReranking.v2.json @@ -0,0 +1,1048 @@ +{ + "dataset_revision": "97ff9e10e265e64abece589ff40c552215754f26", + "task_name": "Vidore3NuclearNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.39458, + "ndcg_at_3": 0.42928, + "ndcg_at_5": 0.4416, + "ndcg_at_10": 0.47043, + "ndcg_at_20": 0.51399, + "ndcg_at_100": 0.56077, + "ndcg_at_1000": 0.56077, + "map_at_1": 0.16983, + "map_at_3": 0.28171, + "map_at_5": 0.3157, + "map_at_10": 0.35231, + "map_at_20": 0.37974, + "map_at_100": 0.3979, + "map_at_1000": 0.3979, + "recall_at_1": 0.16983, + "recall_at_3": 0.35573, + "recall_at_5": 0.43488, + "recall_at_10": 0.54316, + "recall_at_20": 0.6652, + "recall_at_100": 0.81132, + "recall_at_1000": 0.81132, + "accuracy": 0.16983, + "precision_at_1": 0.41265, + "precision_at_3": 0.3243, + "precision_at_5": 0.26506, + "precision_at_10": 0.18946, + "precision_at_20": 0.12681, + "precision_at_100": 0.0338, + "precision_at_1000": 0.00338, + "mrr_at_1": 0.412651, + "mrr_at_3": 0.533133, + "mrr_at_5": 0.552711, + "mrr_at_10": 0.5626, + "mrr_at_20": 0.566486, + "mrr_at_100": 0.568493, + "mrr_at_1000": 0.568493, + "nauc_ndcg_at_1_max": 0.172754, + "nauc_ndcg_at_1_std": 0.106427, + "nauc_ndcg_at_1_diff1": 0.321367, + "nauc_ndcg_at_3_max": 0.152756, + "nauc_ndcg_at_3_std": 0.040717, + "nauc_ndcg_at_3_diff1": 0.251243, + "nauc_ndcg_at_5_max": 0.158666, + "nauc_ndcg_at_5_std": 0.046623, + "nauc_ndcg_at_5_diff1": 0.277763, + "nauc_ndcg_at_10_max": 0.165633, + "nauc_ndcg_at_10_std": 0.07064, + "nauc_ndcg_at_10_diff1": 0.286023, + "nauc_ndcg_at_20_max": 0.161733, + "nauc_ndcg_at_20_std": 0.099125, + "nauc_ndcg_at_20_diff1": 0.275853, + "nauc_ndcg_at_100_max": 0.152356, + "nauc_ndcg_at_100_std": 0.10354, + "nauc_ndcg_at_100_diff1": 0.2679, + "nauc_ndcg_at_1000_max": 0.152356, + "nauc_ndcg_at_1000_std": 0.10354, + "nauc_ndcg_at_1000_diff1": 0.2679, + "nauc_map_at_1_max": 0.136311, + "nauc_map_at_1_std": -0.051933, + "nauc_map_at_1_diff1": 0.435088, + "nauc_map_at_3_max": 0.13892, + "nauc_map_at_3_std": -0.042385, + "nauc_map_at_3_diff1": 0.297434, + "nauc_map_at_5_max": 0.153425, + "nauc_map_at_5_std": -0.020835, + "nauc_map_at_5_diff1": 0.295788, + "nauc_map_at_10_max": 0.15934, + "nauc_map_at_10_std": 0.016039, + "nauc_map_at_10_diff1": 0.292919, + "nauc_map_at_20_max": 0.155297, + "nauc_map_at_20_std": 0.037867, + "nauc_map_at_20_diff1": 0.286696, + "nauc_map_at_100_max": 0.150473, + "nauc_map_at_100_std": 0.045, + "nauc_map_at_100_diff1": 0.283788, + "nauc_map_at_1000_max": 0.150473, + "nauc_map_at_1000_std": 0.045, + "nauc_map_at_1000_diff1": 0.283788, + "nauc_recall_at_1_max": 0.136311, + "nauc_recall_at_1_std": -0.051933, + "nauc_recall_at_1_diff1": 0.435088, + "nauc_recall_at_3_max": 0.088024, + "nauc_recall_at_3_std": -0.076482, + "nauc_recall_at_3_diff1": 0.202671, + "nauc_recall_at_5_max": 0.109637, + "nauc_recall_at_5_std": -0.024968, + "nauc_recall_at_5_diff1": 0.19891, + "nauc_recall_at_10_max": 0.094213, + "nauc_recall_at_10_std": 0.028811, + "nauc_recall_at_10_diff1": 0.221101, + "nauc_recall_at_20_max": 0.078583, + "nauc_recall_at_20_std": 0.085227, + "nauc_recall_at_20_diff1": 0.179954, + "nauc_recall_at_100_max": 0.040396, + "nauc_recall_at_100_std": 0.079722, + "nauc_recall_at_100_diff1": 0.140805, + "nauc_recall_at_1000_max": 0.040396, + "nauc_recall_at_1000_std": 0.079722, + "nauc_recall_at_1000_diff1": 0.140805, + "nauc_precision_at_1_max": 0.168234, + "nauc_precision_at_1_std": 0.104277, + "nauc_precision_at_1_diff1": 0.342566, + "nauc_precision_at_3_max": 0.127777, + "nauc_precision_at_3_std": 0.119726, + "nauc_precision_at_3_diff1": 0.068983, + "nauc_precision_at_5_max": 0.093067, + "nauc_precision_at_5_std": 0.159968, + "nauc_precision_at_5_diff1": 0.074292, + "nauc_precision_at_10_max": 0.07203, + "nauc_precision_at_10_std": 0.241944, + "nauc_precision_at_10_diff1": 0.021197, + "nauc_precision_at_20_max": 0.029545, + "nauc_precision_at_20_std": 0.255093, + "nauc_precision_at_20_diff1": -0.02357, + "nauc_precision_at_100_max": -0.025694, + "nauc_precision_at_100_std": 0.215387, + "nauc_precision_at_100_diff1": -0.071595, + "nauc_precision_at_1000_max": -0.025694, + "nauc_precision_at_1000_std": 0.215387, + "nauc_precision_at_1000_diff1": -0.071595, + "nauc_mrr_at_1_max": 0.168234, + "nauc_mrr_at_1_std": 0.104277, + "nauc_mrr_at_1_diff1": 0.342566, + "nauc_mrr_at_3_max": 0.164724, + "nauc_mrr_at_3_std": 0.105187, + "nauc_mrr_at_3_diff1": 0.30745, + "nauc_mrr_at_5_max": 0.167483, + "nauc_mrr_at_5_std": 0.120163, + "nauc_mrr_at_5_diff1": 0.318824, + "nauc_mrr_at_10_max": 0.165873, + "nauc_mrr_at_10_std": 0.126015, + "nauc_mrr_at_10_diff1": 0.325942, + "nauc_mrr_at_20_max": 0.163435, + "nauc_mrr_at_20_std": 0.123694, + "nauc_mrr_at_20_diff1": 0.325154, + "nauc_mrr_at_100_max": 0.164259, + "nauc_mrr_at_100_std": 0.122702, + "nauc_mrr_at_100_diff1": 0.323586, + "nauc_mrr_at_1000_max": 0.164259, + "nauc_mrr_at_1000_std": 0.122702, + "nauc_mrr_at_1000_diff1": 0.323586, + "hit_rate_at_1": 0.41265, + "hit_rate_at_3": 0.68072, + "hit_rate_at_5": 0.76506, + "hit_rate_at_10": 0.83735, + "hit_rate_at_20": 0.88855, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.47043, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.35542, + "ndcg_at_3": 0.38447, + "ndcg_at_5": 0.39632, + "ndcg_at_10": 0.42296, + "ndcg_at_20": 0.46721, + "ndcg_at_100": 0.52548, + "ndcg_at_1000": 0.52548, + "map_at_1": 0.17161, + "map_at_3": 0.25944, + "map_at_5": 0.28728, + "map_at_10": 0.31959, + "map_at_20": 0.34356, + "map_at_100": 0.36399, + "map_at_1000": 0.36399, + "recall_at_1": 0.17161, + "recall_at_3": 0.32012, + "recall_at_5": 0.38373, + "recall_at_10": 0.48845, + "recall_at_20": 0.61251, + "recall_at_100": 0.79964, + "recall_at_1000": 0.79964, + "accuracy": 0.17161, + "precision_at_1": 0.37651, + "precision_at_3": 0.27912, + "precision_at_5": 0.22952, + "precision_at_10": 0.16235, + "precision_at_20": 0.1131, + "precision_at_100": 0.0328, + "precision_at_1000": 0.00328, + "mrr_at_1": 0.376506, + "mrr_at_3": 0.471386, + "mrr_at_5": 0.493825, + "mrr_at_10": 0.506772, + "mrr_at_20": 0.512989, + "mrr_at_100": 0.51566, + "mrr_at_1000": 0.51566, + "nauc_ndcg_at_1_max": 0.231332, + "nauc_ndcg_at_1_std": 0.127999, + "nauc_ndcg_at_1_diff1": 0.30346, + "nauc_ndcg_at_3_max": 0.249408, + "nauc_ndcg_at_3_std": 0.152084, + "nauc_ndcg_at_3_diff1": 0.306369, + "nauc_ndcg_at_5_max": 0.234057, + "nauc_ndcg_at_5_std": 0.139933, + "nauc_ndcg_at_5_diff1": 0.3003, + "nauc_ndcg_at_10_max": 0.222455, + "nauc_ndcg_at_10_std": 0.135972, + "nauc_ndcg_at_10_diff1": 0.2932, + "nauc_ndcg_at_20_max": 0.217625, + "nauc_ndcg_at_20_std": 0.136908, + "nauc_ndcg_at_20_diff1": 0.28255, + "nauc_ndcg_at_100_max": 0.185746, + "nauc_ndcg_at_100_std": 0.110266, + "nauc_ndcg_at_100_diff1": 0.283851, + "nauc_ndcg_at_1000_max": 0.185746, + "nauc_ndcg_at_1000_std": 0.110266, + "nauc_ndcg_at_1000_diff1": 0.283851, + "nauc_map_at_1_max": 0.246159, + "nauc_map_at_1_std": 0.120334, + "nauc_map_at_1_diff1": 0.333331, + "nauc_map_at_3_max": 0.271811, + "nauc_map_at_3_std": 0.139846, + "nauc_map_at_3_diff1": 0.317708, + "nauc_map_at_5_max": 0.260388, + "nauc_map_at_5_std": 0.139896, + "nauc_map_at_5_diff1": 0.315154, + "nauc_map_at_10_max": 0.249354, + "nauc_map_at_10_std": 0.142289, + "nauc_map_at_10_diff1": 0.296957, + "nauc_map_at_20_max": 0.245523, + "nauc_map_at_20_std": 0.147143, + "nauc_map_at_20_diff1": 0.287096, + "nauc_map_at_100_max": 0.232057, + "nauc_map_at_100_std": 0.137731, + "nauc_map_at_100_diff1": 0.285632, + "nauc_map_at_1000_max": 0.232057, + "nauc_map_at_1000_std": 0.137731, + "nauc_map_at_1000_diff1": 0.285632, + "nauc_recall_at_1_max": 0.246159, + "nauc_recall_at_1_std": 0.120334, + "nauc_recall_at_1_diff1": 0.333331, + "nauc_recall_at_3_max": 0.246537, + "nauc_recall_at_3_std": 0.121307, + "nauc_recall_at_3_diff1": 0.293838, + "nauc_recall_at_5_max": 0.209835, + "nauc_recall_at_5_std": 0.105911, + "nauc_recall_at_5_diff1": 0.281798, + "nauc_recall_at_10_max": 0.167322, + "nauc_recall_at_10_std": 0.106247, + "nauc_recall_at_10_diff1": 0.241023, + "nauc_recall_at_20_max": 0.150532, + "nauc_recall_at_20_std": 0.097828, + "nauc_recall_at_20_diff1": 0.208615, + "nauc_recall_at_100_max": 0.008212, + "nauc_recall_at_100_std": -0.025122, + "nauc_recall_at_100_diff1": 0.21843, + "nauc_recall_at_1000_max": 0.008212, + "nauc_recall_at_1000_std": -0.025122, + "nauc_recall_at_1000_diff1": 0.21843, + "nauc_precision_at_1_max": 0.237685, + "nauc_precision_at_1_std": 0.142284, + "nauc_precision_at_1_diff1": 0.297061, + "nauc_precision_at_3_max": 0.186909, + "nauc_precision_at_3_std": 0.143543, + "nauc_precision_at_3_diff1": 0.178971, + "nauc_precision_at_5_max": 0.103334, + "nauc_precision_at_5_std": 0.115919, + "nauc_precision_at_5_diff1": 0.094385, + "nauc_precision_at_10_max": 0.027858, + "nauc_precision_at_10_std": 0.090033, + "nauc_precision_at_10_diff1": 0.011122, + "nauc_precision_at_20_max": -0.029644, + "nauc_precision_at_20_std": 0.070616, + "nauc_precision_at_20_diff1": -0.051487, + "nauc_precision_at_100_max": -0.13766, + "nauc_precision_at_100_std": -0.030587, + "nauc_precision_at_100_diff1": -0.093432, + "nauc_precision_at_1000_max": -0.13766, + "nauc_precision_at_1000_std": -0.030587, + "nauc_precision_at_1000_diff1": -0.093432, + "nauc_mrr_at_1_max": 0.237685, + "nauc_mrr_at_1_std": 0.142284, + "nauc_mrr_at_1_diff1": 0.297061, + "nauc_mrr_at_3_max": 0.232528, + "nauc_mrr_at_3_std": 0.152834, + "nauc_mrr_at_3_diff1": 0.314426, + "nauc_mrr_at_5_max": 0.223502, + "nauc_mrr_at_5_std": 0.148948, + "nauc_mrr_at_5_diff1": 0.309779, + "nauc_mrr_at_10_max": 0.218379, + "nauc_mrr_at_10_std": 0.150493, + "nauc_mrr_at_10_diff1": 0.311643, + "nauc_mrr_at_20_max": 0.215696, + "nauc_mrr_at_20_std": 0.147275, + "nauc_mrr_at_20_diff1": 0.31017, + "nauc_mrr_at_100_max": 0.216672, + "nauc_mrr_at_100_std": 0.147488, + "nauc_mrr_at_100_diff1": 0.309371, + "nauc_mrr_at_1000_max": 0.216672, + "nauc_mrr_at_1000_std": 0.147488, + "nauc_mrr_at_1000_diff1": 0.309371, + "hit_rate_at_1": 0.37651, + "hit_rate_at_3": 0.59036, + "hit_rate_at_5": 0.68675, + "hit_rate_at_10": 0.77711, + "hit_rate_at_20": 0.86145, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.42296, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.38102, + "ndcg_at_3": 0.38532, + "ndcg_at_5": 0.39399, + "ndcg_at_10": 0.41635, + "ndcg_at_20": 0.46479, + "ndcg_at_100": 0.52624, + "ndcg_at_1000": 0.52624, + "map_at_1": 0.18016, + "map_at_3": 0.25487, + "map_at_5": 0.28346, + "map_at_10": 0.31509, + "map_at_20": 0.34135, + "map_at_100": 0.36215, + "map_at_1000": 0.36215, + "recall_at_1": 0.18016, + "recall_at_3": 0.3076, + "recall_at_5": 0.37666, + "recall_at_10": 0.46728, + "recall_at_20": 0.60526, + "recall_at_100": 0.80754, + "recall_at_1000": 0.80754, + "accuracy": 0.18016, + "precision_at_1": 0.40964, + "precision_at_3": 0.28313, + "precision_at_5": 0.22651, + "precision_at_10": 0.16024, + "precision_at_20": 0.11355, + "precision_at_100": 0.03304, + "precision_at_1000": 0.0033, + "mrr_at_1": 0.409639, + "mrr_at_3": 0.506526, + "mrr_at_5": 0.523544, + "mrr_at_10": 0.533905, + "mrr_at_20": 0.538995, + "mrr_at_100": 0.541889, + "mrr_at_1000": 0.541889, + "nauc_ndcg_at_1_max": 0.227969, + "nauc_ndcg_at_1_std": 0.174536, + "nauc_ndcg_at_1_diff1": 0.244679, + "nauc_ndcg_at_3_max": 0.182091, + "nauc_ndcg_at_3_std": 0.120116, + "nauc_ndcg_at_3_diff1": 0.217188, + "nauc_ndcg_at_5_max": 0.207201, + "nauc_ndcg_at_5_std": 0.150118, + "nauc_ndcg_at_5_diff1": 0.229973, + "nauc_ndcg_at_10_max": 0.208739, + "nauc_ndcg_at_10_std": 0.14646, + "nauc_ndcg_at_10_diff1": 0.221619, + "nauc_ndcg_at_20_max": 0.210697, + "nauc_ndcg_at_20_std": 0.158918, + "nauc_ndcg_at_20_diff1": 0.217402, + "nauc_ndcg_at_100_max": 0.184036, + "nauc_ndcg_at_100_std": 0.137339, + "nauc_ndcg_at_100_diff1": 0.215556, + "nauc_ndcg_at_1000_max": 0.184036, + "nauc_ndcg_at_1000_std": 0.137339, + "nauc_ndcg_at_1000_diff1": 0.215556, + "nauc_map_at_1_max": 0.219665, + "nauc_map_at_1_std": 0.104288, + "nauc_map_at_1_diff1": 0.253768, + "nauc_map_at_3_max": 0.217046, + "nauc_map_at_3_std": 0.103121, + "nauc_map_at_3_diff1": 0.221746, + "nauc_map_at_5_max": 0.228302, + "nauc_map_at_5_std": 0.125414, + "nauc_map_at_5_diff1": 0.221283, + "nauc_map_at_10_max": 0.232636, + "nauc_map_at_10_std": 0.139353, + "nauc_map_at_10_diff1": 0.213806, + "nauc_map_at_20_max": 0.232384, + "nauc_map_at_20_std": 0.15005, + "nauc_map_at_20_diff1": 0.209027, + "nauc_map_at_100_max": 0.221856, + "nauc_map_at_100_std": 0.142825, + "nauc_map_at_100_diff1": 0.206462, + "nauc_map_at_1000_max": 0.221856, + "nauc_map_at_1000_std": 0.142825, + "nauc_map_at_1000_diff1": 0.206462, + "nauc_recall_at_1_max": 0.219665, + "nauc_recall_at_1_std": 0.104288, + "nauc_recall_at_1_diff1": 0.253768, + "nauc_recall_at_3_max": 0.159878, + "nauc_recall_at_3_std": 0.061216, + "nauc_recall_at_3_diff1": 0.203496, + "nauc_recall_at_5_max": 0.178513, + "nauc_recall_at_5_std": 0.106762, + "nauc_recall_at_5_diff1": 0.187312, + "nauc_recall_at_10_max": 0.160984, + "nauc_recall_at_10_std": 0.101642, + "nauc_recall_at_10_diff1": 0.169192, + "nauc_recall_at_20_max": 0.146423, + "nauc_recall_at_20_std": 0.107729, + "nauc_recall_at_20_diff1": 0.146967, + "nauc_recall_at_100_max": 0.085291, + "nauc_recall_at_100_std": 0.038643, + "nauc_recall_at_100_diff1": 0.157543, + "nauc_recall_at_1000_max": 0.085291, + "nauc_recall_at_1000_std": 0.038643, + "nauc_recall_at_1000_diff1": 0.157543, + "nauc_precision_at_1_max": 0.232196, + "nauc_precision_at_1_std": 0.188518, + "nauc_precision_at_1_diff1": 0.240326, + "nauc_precision_at_3_max": 0.107172, + "nauc_precision_at_3_std": 0.119359, + "nauc_precision_at_3_diff1": 0.117192, + "nauc_precision_at_5_max": 0.115249, + "nauc_precision_at_5_std": 0.168924, + "nauc_precision_at_5_diff1": 0.106504, + "nauc_precision_at_10_max": 0.054472, + "nauc_precision_at_10_std": 0.130153, + "nauc_precision_at_10_diff1": 0.036248, + "nauc_precision_at_20_max": 0.009361, + "nauc_precision_at_20_std": 0.12562, + "nauc_precision_at_20_diff1": -0.013849, + "nauc_precision_at_100_max": -0.114935, + "nauc_precision_at_100_std": 0.003044, + "nauc_precision_at_100_diff1": -0.06757, + "nauc_precision_at_1000_max": -0.114935, + "nauc_precision_at_1000_std": 0.003044, + "nauc_precision_at_1000_diff1": -0.06757, + "nauc_mrr_at_1_max": 0.232196, + "nauc_mrr_at_1_std": 0.188518, + "nauc_mrr_at_1_diff1": 0.240326, + "nauc_mrr_at_3_max": 0.176936, + "nauc_mrr_at_3_std": 0.155686, + "nauc_mrr_at_3_diff1": 0.244629, + "nauc_mrr_at_5_max": 0.182752, + "nauc_mrr_at_5_std": 0.170894, + "nauc_mrr_at_5_diff1": 0.232636, + "nauc_mrr_at_10_max": 0.172384, + "nauc_mrr_at_10_std": 0.161014, + "nauc_mrr_at_10_diff1": 0.226387, + "nauc_mrr_at_20_max": 0.17465, + "nauc_mrr_at_20_std": 0.162981, + "nauc_mrr_at_20_diff1": 0.227352, + "nauc_mrr_at_100_max": 0.176688, + "nauc_mrr_at_100_std": 0.163868, + "nauc_mrr_at_100_diff1": 0.227879, + "nauc_mrr_at_1000_max": 0.176688, + "nauc_mrr_at_1000_std": 0.163868, + "nauc_mrr_at_1000_diff1": 0.227879, + "hit_rate_at_1": 0.40964, + "hit_rate_at_3": 0.62952, + "hit_rate_at_5": 0.70482, + "hit_rate_at_10": 0.78012, + "hit_rate_at_20": 0.85843, + "hit_rate_at_100": 0.95181, + "hit_rate_at_1000": 0.95181, + "main_score": 0.41635, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.37349, + "ndcg_at_3": 0.38512, + "ndcg_at_5": 0.39891, + "ndcg_at_10": 0.42644, + "ndcg_at_20": 0.46962, + "ndcg_at_100": 0.53123, + "ndcg_at_1000": 0.53123, + "map_at_1": 0.16591, + "map_at_3": 0.25315, + "map_at_5": 0.28507, + "map_at_10": 0.31845, + "map_at_20": 0.34221, + "map_at_100": 0.36426, + "map_at_1000": 0.36426, + "recall_at_1": 0.16591, + "recall_at_3": 0.30947, + "recall_at_5": 0.38317, + "recall_at_10": 0.49219, + "recall_at_20": 0.6119, + "recall_at_100": 0.80792, + "recall_at_1000": 0.80792, + "accuracy": 0.16591, + "precision_at_1": 0.39157, + "precision_at_3": 0.28414, + "precision_at_5": 0.23193, + "precision_at_10": 0.16536, + "precision_at_20": 0.1134, + "precision_at_100": 0.03334, + "precision_at_1000": 0.00333, + "mrr_at_1": 0.391566, + "mrr_at_3": 0.491968, + "mrr_at_5": 0.507932, + "mrr_at_10": 0.522031, + "mrr_at_20": 0.528775, + "mrr_at_100": 0.531167, + "mrr_at_1000": 0.531167, + "nauc_ndcg_at_1_max": 0.238072, + "nauc_ndcg_at_1_std": 0.179361, + "nauc_ndcg_at_1_diff1": 0.339443, + "nauc_ndcg_at_3_max": 0.235505, + "nauc_ndcg_at_3_std": 0.159733, + "nauc_ndcg_at_3_diff1": 0.319062, + "nauc_ndcg_at_5_max": 0.236666, + "nauc_ndcg_at_5_std": 0.164365, + "nauc_ndcg_at_5_diff1": 0.317516, + "nauc_ndcg_at_10_max": 0.206524, + "nauc_ndcg_at_10_std": 0.134333, + "nauc_ndcg_at_10_diff1": 0.333137, + "nauc_ndcg_at_20_max": 0.199496, + "nauc_ndcg_at_20_std": 0.13102, + "nauc_ndcg_at_20_diff1": 0.324624, + "nauc_ndcg_at_100_max": 0.169366, + "nauc_ndcg_at_100_std": 0.101941, + "nauc_ndcg_at_100_diff1": 0.325354, + "nauc_ndcg_at_1000_max": 0.169366, + "nauc_ndcg_at_1000_std": 0.101941, + "nauc_ndcg_at_1000_diff1": 0.325354, + "nauc_map_at_1_max": 0.235677, + "nauc_map_at_1_std": 0.140569, + "nauc_map_at_1_diff1": 0.325917, + "nauc_map_at_3_max": 0.239788, + "nauc_map_at_3_std": 0.144816, + "nauc_map_at_3_diff1": 0.297845, + "nauc_map_at_5_max": 0.24862, + "nauc_map_at_5_std": 0.161903, + "nauc_map_at_5_diff1": 0.289758, + "nauc_map_at_10_max": 0.2332, + "nauc_map_at_10_std": 0.150547, + "nauc_map_at_10_diff1": 0.305686, + "nauc_map_at_20_max": 0.226608, + "nauc_map_at_20_std": 0.149561, + "nauc_map_at_20_diff1": 0.305195, + "nauc_map_at_100_max": 0.212902, + "nauc_map_at_100_std": 0.138128, + "nauc_map_at_100_diff1": 0.305367, + "nauc_map_at_1000_max": 0.212902, + "nauc_map_at_1000_std": 0.138128, + "nauc_map_at_1000_diff1": 0.305367, + "nauc_recall_at_1_max": 0.235677, + "nauc_recall_at_1_std": 0.140569, + "nauc_recall_at_1_diff1": 0.325917, + "nauc_recall_at_3_max": 0.1997, + "nauc_recall_at_3_std": 0.113821, + "nauc_recall_at_3_diff1": 0.270121, + "nauc_recall_at_5_max": 0.197865, + "nauc_recall_at_5_std": 0.12916, + "nauc_recall_at_5_diff1": 0.239608, + "nauc_recall_at_10_max": 0.127716, + "nauc_recall_at_10_std": 0.070972, + "nauc_recall_at_10_diff1": 0.269634, + "nauc_recall_at_20_max": 0.10632, + "nauc_recall_at_20_std": 0.042591, + "nauc_recall_at_20_diff1": 0.232606, + "nauc_recall_at_100_max": -0.033555, + "nauc_recall_at_100_std": -0.111155, + "nauc_recall_at_100_diff1": 0.233161, + "nauc_recall_at_1000_max": -0.033555, + "nauc_recall_at_1000_std": -0.111155, + "nauc_recall_at_1000_diff1": 0.233161, + "nauc_precision_at_1_max": 0.246276, + "nauc_precision_at_1_std": 0.188624, + "nauc_precision_at_1_diff1": 0.341746, + "nauc_precision_at_3_max": 0.180305, + "nauc_precision_at_3_std": 0.145701, + "nauc_precision_at_3_diff1": 0.248674, + "nauc_precision_at_5_max": 0.151044, + "nauc_precision_at_5_std": 0.149647, + "nauc_precision_at_5_diff1": 0.181672, + "nauc_precision_at_10_max": 0.040486, + "nauc_precision_at_10_std": 0.060504, + "nauc_precision_at_10_diff1": 0.168779, + "nauc_precision_at_20_max": -0.009499, + "nauc_precision_at_20_std": 0.042813, + "nauc_precision_at_20_diff1": 0.095672, + "nauc_precision_at_100_max": -0.11641, + "nauc_precision_at_100_std": -0.055634, + "nauc_precision_at_100_diff1": 0.0008, + "nauc_precision_at_1000_max": -0.11641, + "nauc_precision_at_1000_std": -0.055634, + "nauc_precision_at_1000_diff1": 0.0008, + "nauc_mrr_at_1_max": 0.246276, + "nauc_mrr_at_1_std": 0.188624, + "nauc_mrr_at_1_diff1": 0.341746, + "nauc_mrr_at_3_max": 0.230605, + "nauc_mrr_at_3_std": 0.178398, + "nauc_mrr_at_3_diff1": 0.365164, + "nauc_mrr_at_5_max": 0.219922, + "nauc_mrr_at_5_std": 0.167981, + "nauc_mrr_at_5_diff1": 0.366364, + "nauc_mrr_at_10_max": 0.204424, + "nauc_mrr_at_10_std": 0.155426, + "nauc_mrr_at_10_diff1": 0.364451, + "nauc_mrr_at_20_max": 0.208105, + "nauc_mrr_at_20_std": 0.159883, + "nauc_mrr_at_20_diff1": 0.36175, + "nauc_mrr_at_100_max": 0.209772, + "nauc_mrr_at_100_std": 0.161537, + "nauc_mrr_at_100_diff1": 0.362227, + "nauc_mrr_at_1000_max": 0.209772, + "nauc_mrr_at_1000_std": 0.161537, + "nauc_mrr_at_1000_diff1": 0.362227, + "hit_rate_at_1": 0.39157, + "hit_rate_at_3": 0.61446, + "hit_rate_at_5": 0.68373, + "hit_rate_at_10": 0.78313, + "hit_rate_at_20": 0.87349, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.42644, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.40211, + "ndcg_at_3": 0.4058, + "ndcg_at_5": 0.4206, + "ndcg_at_10": 0.44449, + "ndcg_at_20": 0.48502, + "ndcg_at_100": 0.54451, + "ndcg_at_1000": 0.54451, + "map_at_1": 0.1843, + "map_at_3": 0.27039, + "map_at_5": 0.30364, + "map_at_10": 0.33696, + "map_at_20": 0.35934, + "map_at_100": 0.38166, + "map_at_1000": 0.38166, + "recall_at_1": 0.1843, + "recall_at_3": 0.32405, + "recall_at_5": 0.39793, + "recall_at_10": 0.50805, + "recall_at_20": 0.6194, + "recall_at_100": 0.80575, + "recall_at_1000": 0.80575, + "accuracy": 0.1843, + "precision_at_1": 0.41867, + "precision_at_3": 0.29217, + "precision_at_5": 0.24398, + "precision_at_10": 0.16898, + "precision_at_20": 0.11386, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.418675, + "mrr_at_3": 0.52008, + "mrr_at_5": 0.535743, + "mrr_at_10": 0.548262, + "mrr_at_20": 0.55411, + "mrr_at_100": 0.556081, + "mrr_at_1000": 0.556081, + "nauc_ndcg_at_1_max": 0.239872, + "nauc_ndcg_at_1_std": 0.111453, + "nauc_ndcg_at_1_diff1": 0.303115, + "nauc_ndcg_at_3_max": 0.262959, + "nauc_ndcg_at_3_std": 0.126128, + "nauc_ndcg_at_3_diff1": 0.270932, + "nauc_ndcg_at_5_max": 0.25378, + "nauc_ndcg_at_5_std": 0.131326, + "nauc_ndcg_at_5_diff1": 0.280544, + "nauc_ndcg_at_10_max": 0.25, + "nauc_ndcg_at_10_std": 0.136463, + "nauc_ndcg_at_10_diff1": 0.295557, + "nauc_ndcg_at_20_max": 0.227085, + "nauc_ndcg_at_20_std": 0.124569, + "nauc_ndcg_at_20_diff1": 0.297059, + "nauc_ndcg_at_100_max": 0.207036, + "nauc_ndcg_at_100_std": 0.113536, + "nauc_ndcg_at_100_diff1": 0.282921, + "nauc_ndcg_at_1000_max": 0.207036, + "nauc_ndcg_at_1000_std": 0.113536, + "nauc_ndcg_at_1000_diff1": 0.282921, + "nauc_map_at_1_max": 0.191083, + "nauc_map_at_1_std": 0.046303, + "nauc_map_at_1_diff1": 0.310127, + "nauc_map_at_3_max": 0.246055, + "nauc_map_at_3_std": 0.098416, + "nauc_map_at_3_diff1": 0.262684, + "nauc_map_at_5_max": 0.249578, + "nauc_map_at_5_std": 0.114408, + "nauc_map_at_5_diff1": 0.260804, + "nauc_map_at_10_max": 0.245238, + "nauc_map_at_10_std": 0.12315, + "nauc_map_at_10_diff1": 0.269154, + "nauc_map_at_20_max": 0.238038, + "nauc_map_at_20_std": 0.122663, + "nauc_map_at_20_diff1": 0.268582, + "nauc_map_at_100_max": 0.228143, + "nauc_map_at_100_std": 0.118804, + "nauc_map_at_100_diff1": 0.263755, + "nauc_map_at_1000_max": 0.228143, + "nauc_map_at_1000_std": 0.118804, + "nauc_map_at_1000_diff1": 0.263755, + "nauc_recall_at_1_max": 0.191083, + "nauc_recall_at_1_std": 0.046303, + "nauc_recall_at_1_diff1": 0.310127, + "nauc_recall_at_3_max": 0.233064, + "nauc_recall_at_3_std": 0.092024, + "nauc_recall_at_3_diff1": 0.216328, + "nauc_recall_at_5_max": 0.21859, + "nauc_recall_at_5_std": 0.107994, + "nauc_recall_at_5_diff1": 0.204481, + "nauc_recall_at_10_max": 0.19798, + "nauc_recall_at_10_std": 0.117997, + "nauc_recall_at_10_diff1": 0.213029, + "nauc_recall_at_20_max": 0.132799, + "nauc_recall_at_20_std": 0.086567, + "nauc_recall_at_20_diff1": 0.21824, + "nauc_recall_at_100_max": 0.028132, + "nauc_recall_at_100_std": 0.014808, + "nauc_recall_at_100_diff1": 0.162652, + "nauc_recall_at_1000_max": 0.028132, + "nauc_recall_at_1000_std": 0.014808, + "nauc_recall_at_1000_diff1": 0.162652, + "nauc_precision_at_1_max": 0.247012, + "nauc_precision_at_1_std": 0.118095, + "nauc_precision_at_1_diff1": 0.310254, + "nauc_precision_at_3_max": 0.229223, + "nauc_precision_at_3_std": 0.152674, + "nauc_precision_at_3_diff1": 0.17973, + "nauc_precision_at_5_max": 0.161379, + "nauc_precision_at_5_std": 0.152126, + "nauc_precision_at_5_diff1": 0.132985, + "nauc_precision_at_10_max": 0.075443, + "nauc_precision_at_10_std": 0.131216, + "nauc_precision_at_10_diff1": 0.080183, + "nauc_precision_at_20_max": -0.014936, + "nauc_precision_at_20_std": 0.07867, + "nauc_precision_at_20_diff1": 0.025658, + "nauc_precision_at_100_max": -0.095839, + "nauc_precision_at_100_std": 0.014017, + "nauc_precision_at_100_diff1": -0.062849, + "nauc_precision_at_1000_max": -0.095839, + "nauc_precision_at_1000_std": 0.014017, + "nauc_precision_at_1000_diff1": -0.062849, + "nauc_mrr_at_1_max": 0.247012, + "nauc_mrr_at_1_std": 0.118095, + "nauc_mrr_at_1_diff1": 0.310254, + "nauc_mrr_at_3_max": 0.260508, + "nauc_mrr_at_3_std": 0.132425, + "nauc_mrr_at_3_diff1": 0.32988, + "nauc_mrr_at_5_max": 0.252556, + "nauc_mrr_at_5_std": 0.134764, + "nauc_mrr_at_5_diff1": 0.333704, + "nauc_mrr_at_10_max": 0.251804, + "nauc_mrr_at_10_std": 0.138629, + "nauc_mrr_at_10_diff1": 0.333505, + "nauc_mrr_at_20_max": 0.247653, + "nauc_mrr_at_20_std": 0.137261, + "nauc_mrr_at_20_diff1": 0.333427, + "nauc_mrr_at_100_max": 0.247967, + "nauc_mrr_at_100_std": 0.136876, + "nauc_mrr_at_100_diff1": 0.331877, + "nauc_mrr_at_1000_max": 0.247967, + "nauc_mrr_at_1000_std": 0.136876, + "nauc_mrr_at_1000_diff1": 0.331877, + "hit_rate_at_1": 0.41867, + "hit_rate_at_3": 0.64458, + "hit_rate_at_5": 0.71386, + "hit_rate_at_10": 0.80422, + "hit_rate_at_20": 0.88253, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.44449, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.39759, + "ndcg_at_3": 0.39591, + "ndcg_at_5": 0.41873, + "ndcg_at_10": 0.4434, + "ndcg_at_20": 0.48759, + "ndcg_at_100": 0.54317, + "ndcg_at_1000": 0.54317, + "map_at_1": 0.18582, + "map_at_3": 0.26441, + "map_at_5": 0.3023, + "map_at_10": 0.3357, + "map_at_20": 0.36058, + "map_at_100": 0.38133, + "map_at_1000": 0.38133, + "recall_at_1": 0.18582, + "recall_at_3": 0.31034, + "recall_at_5": 0.40344, + "recall_at_10": 0.5125, + "recall_at_20": 0.63103, + "recall_at_100": 0.80599, + "recall_at_1000": 0.80599, + "accuracy": 0.18582, + "precision_at_1": 0.41566, + "precision_at_3": 0.28715, + "precision_at_5": 0.24578, + "precision_at_10": 0.16958, + "precision_at_20": 0.11702, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.415663, + "mrr_at_3": 0.508534, + "mrr_at_5": 0.531124, + "mrr_at_10": 0.543201, + "mrr_at_20": 0.548484, + "mrr_at_100": 0.550304, + "mrr_at_1000": 0.550304, + "nauc_ndcg_at_1_max": 0.257991, + "nauc_ndcg_at_1_std": 0.164996, + "nauc_ndcg_at_1_diff1": 0.278172, + "nauc_ndcg_at_3_max": 0.264239, + "nauc_ndcg_at_3_std": 0.142871, + "nauc_ndcg_at_3_diff1": 0.290662, + "nauc_ndcg_at_5_max": 0.250002, + "nauc_ndcg_at_5_std": 0.141089, + "nauc_ndcg_at_5_diff1": 0.296572, + "nauc_ndcg_at_10_max": 0.238369, + "nauc_ndcg_at_10_std": 0.123319, + "nauc_ndcg_at_10_diff1": 0.33322, + "nauc_ndcg_at_20_max": 0.228337, + "nauc_ndcg_at_20_std": 0.120448, + "nauc_ndcg_at_20_diff1": 0.333401, + "nauc_ndcg_at_100_max": 0.210042, + "nauc_ndcg_at_100_std": 0.116138, + "nauc_ndcg_at_100_diff1": 0.314168, + "nauc_ndcg_at_1000_max": 0.210042, + "nauc_ndcg_at_1000_std": 0.116138, + "nauc_ndcg_at_1000_diff1": 0.314168, + "nauc_map_at_1_max": 0.256395, + "nauc_map_at_1_std": 0.093183, + "nauc_map_at_1_diff1": 0.319225, + "nauc_map_at_3_max": 0.273092, + "nauc_map_at_3_std": 0.112827, + "nauc_map_at_3_diff1": 0.312858, + "nauc_map_at_5_max": 0.260754, + "nauc_map_at_5_std": 0.123238, + "nauc_map_at_5_diff1": 0.29735, + "nauc_map_at_10_max": 0.252962, + "nauc_map_at_10_std": 0.121403, + "nauc_map_at_10_diff1": 0.309999, + "nauc_map_at_20_max": 0.24724, + "nauc_map_at_20_std": 0.123339, + "nauc_map_at_20_diff1": 0.309679, + "nauc_map_at_100_max": 0.238395, + "nauc_map_at_100_std": 0.121722, + "nauc_map_at_100_diff1": 0.303775, + "nauc_map_at_1000_max": 0.238395, + "nauc_map_at_1000_std": 0.121722, + "nauc_map_at_1000_diff1": 0.303775, + "nauc_recall_at_1_max": 0.256395, + "nauc_recall_at_1_std": 0.093183, + "nauc_recall_at_1_diff1": 0.319225, + "nauc_recall_at_3_max": 0.24588, + "nauc_recall_at_3_std": 0.097807, + "nauc_recall_at_3_diff1": 0.290923, + "nauc_recall_at_5_max": 0.203476, + "nauc_recall_at_5_std": 0.097806, + "nauc_recall_at_5_diff1": 0.26359, + "nauc_recall_at_10_max": 0.156925, + "nauc_recall_at_10_std": 0.070696, + "nauc_recall_at_10_diff1": 0.290402, + "nauc_recall_at_20_max": 0.131272, + "nauc_recall_at_20_std": 0.053969, + "nauc_recall_at_20_diff1": 0.29342, + "nauc_recall_at_100_max": 0.048887, + "nauc_recall_at_100_std": 0.019395, + "nauc_recall_at_100_diff1": 0.255483, + "nauc_recall_at_1000_max": 0.048887, + "nauc_recall_at_1000_std": 0.019395, + "nauc_recall_at_1000_diff1": 0.255483, + "nauc_precision_at_1_max": 0.271094, + "nauc_precision_at_1_std": 0.189556, + "nauc_precision_at_1_diff1": 0.272694, + "nauc_precision_at_3_max": 0.190008, + "nauc_precision_at_3_std": 0.133555, + "nauc_precision_at_3_diff1": 0.195305, + "nauc_precision_at_5_max": 0.11327, + "nauc_precision_at_5_std": 0.133436, + "nauc_precision_at_5_diff1": 0.105744, + "nauc_precision_at_10_max": 0.03443, + "nauc_precision_at_10_std": 0.086846, + "nauc_precision_at_10_diff1": 0.101878, + "nauc_precision_at_20_max": -0.038409, + "nauc_precision_at_20_std": 0.049499, + "nauc_precision_at_20_diff1": 0.030194, + "nauc_precision_at_100_max": -0.115198, + "nauc_precision_at_100_std": -0.000179, + "nauc_precision_at_100_diff1": -0.068913, + "nauc_precision_at_1000_max": -0.115198, + "nauc_precision_at_1000_std": -0.000179, + "nauc_precision_at_1000_diff1": -0.068913, + "nauc_mrr_at_1_max": 0.271094, + "nauc_mrr_at_1_std": 0.189556, + "nauc_mrr_at_1_diff1": 0.272694, + "nauc_mrr_at_3_max": 0.262332, + "nauc_mrr_at_3_std": 0.169903, + "nauc_mrr_at_3_diff1": 0.30918, + "nauc_mrr_at_5_max": 0.252555, + "nauc_mrr_at_5_std": 0.168398, + "nauc_mrr_at_5_diff1": 0.310859, + "nauc_mrr_at_10_max": 0.24763, + "nauc_mrr_at_10_std": 0.167254, + "nauc_mrr_at_10_diff1": 0.310829, + "nauc_mrr_at_20_max": 0.245495, + "nauc_mrr_at_20_std": 0.165491, + "nauc_mrr_at_20_diff1": 0.308277, + "nauc_mrr_at_100_max": 0.247124, + "nauc_mrr_at_100_std": 0.167803, + "nauc_mrr_at_100_diff1": 0.306431, + "nauc_mrr_at_1000_max": 0.247124, + "nauc_mrr_at_1000_std": 0.167803, + "nauc_mrr_at_1000_diff1": 0.306431, + "hit_rate_at_1": 0.41566, + "hit_rate_at_3": 0.62048, + "hit_rate_at_5": 0.72289, + "hit_rate_at_10": 0.81024, + "hit_rate_at_20": 0.88554, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.4434, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 2222.05667703005, + "kg_co2_emissions": null, + "date": 1792441121.770636, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 1.4121062122285366e-05, + "end": 35.79794501198921, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 35.79801035998389, + "end": 35.79802681400906, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries and documents", + "start": 35.80450381105766, + "end": 407.4867357100593, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 407.48685531900264, + "end": 407.5411490100669, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries and documents", + "start": 407.55656098702457, + "end": 776.3134779779939, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 776.3135173970368, + "end": 776.3600465140771, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries and documents", + "start": 776.3727159100818, + "end": 1136.0192632160615, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 1136.01930808404, + "end": 1136.0662133850856, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries and documents", + "start": 1136.0831878080498, + "end": 1502.5334864030592, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 1502.5335151660256, + "end": 1502.5796970390948, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries and documents", + "start": 1502.593337057042, + "end": 1862.3136563760927, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 1862.3136972280918, + "end": 1862.3604561809916, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries and documents", + "start": 1862.3739934120094, + "end": 2221.999913225067, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 2221.9999382130336, + "end": 2222.046255232999, + "split": "test", + "subset": "portuguese" + } + ] +} diff --git a/results/BAAI__bge-reranker-v2-m3/953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e/Vidore3TelecomNemotron35Embed8BReranking.v2.json b/results/BAAI__bge-reranker-v2-m3/953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e/Vidore3TelecomNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..99d1afa9ca --- /dev/null +++ b/results/BAAI__bge-reranker-v2-m3/953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e/Vidore3TelecomNemotron35Embed8BReranking.v2.json @@ -0,0 +1,1048 @@ +{ + "dataset_revision": "47c9b703527792f3fbf0a082f4589ff40c1aa8cc", + "task_name": "Vidore3TelecomNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.63937, + "ndcg_at_3": 0.62526, + "ndcg_at_5": 0.62823, + "ndcg_at_10": 0.65785, + "ndcg_at_20": 0.69093, + "ndcg_at_100": 0.73231, + "ndcg_at_1000": 0.73231, + "map_at_1": 0.28232, + "map_at_3": 0.44235, + "map_at_5": 0.4892, + "map_at_10": 0.54084, + "map_at_20": 0.56897, + "map_at_100": 0.58854, + "map_at_1000": 0.58854, + "recall_at_1": 0.28232, + "recall_at_3": 0.49914, + "recall_at_5": 0.58346, + "recall_at_10": 0.70398, + "recall_at_20": 0.80339, + "recall_at_100": 0.92961, + "recall_at_1000": 0.92961, + "accuracy": 0.28232, + "precision_at_1": 0.66092, + "precision_at_3": 0.46648, + "precision_at_5": 0.36322, + "precision_at_10": 0.2454, + "precision_at_20": 0.1546, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.66092, + "mrr_at_3": 0.748563, + "mrr_at_5": 0.76092, + "mrr_at_10": 0.766598, + "mrr_at_20": 0.768534, + "mrr_at_100": 0.769288, + "mrr_at_1000": 0.769288, + "nauc_ndcg_at_1_max": 0.464803, + "nauc_ndcg_at_1_std": 0.106133, + "nauc_ndcg_at_1_diff1": 0.307973, + "nauc_ndcg_at_3_max": 0.466582, + "nauc_ndcg_at_3_std": 0.152794, + "nauc_ndcg_at_3_diff1": 0.183806, + "nauc_ndcg_at_5_max": 0.456838, + "nauc_ndcg_at_5_std": 0.150312, + "nauc_ndcg_at_5_diff1": 0.191586, + "nauc_ndcg_at_10_max": 0.471711, + "nauc_ndcg_at_10_std": 0.172063, + "nauc_ndcg_at_10_diff1": 0.191804, + "nauc_ndcg_at_20_max": 0.478232, + "nauc_ndcg_at_20_std": 0.186757, + "nauc_ndcg_at_20_diff1": 0.190455, + "nauc_ndcg_at_100_max": 0.488524, + "nauc_ndcg_at_100_std": 0.192717, + "nauc_ndcg_at_100_diff1": 0.196817, + "nauc_ndcg_at_1000_max": 0.488524, + "nauc_ndcg_at_1000_std": 0.192717, + "nauc_ndcg_at_1000_diff1": 0.196817, + "nauc_map_at_1_max": 0.324024, + "nauc_map_at_1_std": -0.024447, + "nauc_map_at_1_diff1": 0.337845, + "nauc_map_at_3_max": 0.404239, + "nauc_map_at_3_std": 0.063653, + "nauc_map_at_3_diff1": 0.202759, + "nauc_map_at_5_max": 0.412146, + "nauc_map_at_5_std": 0.092941, + "nauc_map_at_5_diff1": 0.185149, + "nauc_map_at_10_max": 0.425386, + "nauc_map_at_10_std": 0.128645, + "nauc_map_at_10_diff1": 0.178497, + "nauc_map_at_20_max": 0.435432, + "nauc_map_at_20_std": 0.146005, + "nauc_map_at_20_diff1": 0.176759, + "nauc_map_at_100_max": 0.441026, + "nauc_map_at_100_std": 0.151119, + "nauc_map_at_100_diff1": 0.176362, + "nauc_map_at_1000_max": 0.441026, + "nauc_map_at_1000_std": 0.151119, + "nauc_map_at_1000_diff1": 0.176362, + "nauc_recall_at_1_max": 0.324024, + "nauc_recall_at_1_std": -0.024447, + "nauc_recall_at_1_diff1": 0.337845, + "nauc_recall_at_3_max": 0.368969, + "nauc_recall_at_3_std": 0.075192, + "nauc_recall_at_3_diff1": 0.139815, + "nauc_recall_at_5_max": 0.357965, + "nauc_recall_at_5_std": 0.099229, + "nauc_recall_at_5_diff1": 0.103855, + "nauc_recall_at_10_max": 0.36384, + "nauc_recall_at_10_std": 0.168709, + "nauc_recall_at_10_diff1": 0.08779, + "nauc_recall_at_20_max": 0.372244, + "nauc_recall_at_20_std": 0.222021, + "nauc_recall_at_20_diff1": 0.0787, + "nauc_recall_at_100_max": 0.473658, + "nauc_recall_at_100_std": 0.422319, + "nauc_recall_at_100_diff1": -0.006464, + "nauc_recall_at_1000_max": 0.473658, + "nauc_recall_at_1000_std": 0.422319, + "nauc_recall_at_1000_diff1": -0.006464, + "nauc_precision_at_1_max": 0.507489, + "nauc_precision_at_1_std": 0.143756, + "nauc_precision_at_1_diff1": 0.305739, + "nauc_precision_at_3_max": 0.29288, + "nauc_precision_at_3_std": 0.234923, + "nauc_precision_at_3_diff1": -0.090765, + "nauc_precision_at_5_max": 0.182588, + "nauc_precision_at_5_std": 0.229249, + "nauc_precision_at_5_diff1": -0.116223, + "nauc_precision_at_10_max": 0.087754, + "nauc_precision_at_10_std": 0.214422, + "nauc_precision_at_10_diff1": -0.116725, + "nauc_precision_at_20_max": 0.00414, + "nauc_precision_at_20_std": 0.158336, + "nauc_precision_at_20_diff1": -0.111365, + "nauc_precision_at_100_max": -0.07828, + "nauc_precision_at_100_std": 0.089779, + "nauc_precision_at_100_diff1": -0.110178, + "nauc_precision_at_1000_max": -0.07828, + "nauc_precision_at_1000_std": 0.089779, + "nauc_precision_at_1000_diff1": -0.110178, + "nauc_mrr_at_1_max": 0.507489, + "nauc_mrr_at_1_std": 0.143756, + "nauc_mrr_at_1_diff1": 0.305739, + "nauc_mrr_at_3_max": 0.569693, + "nauc_mrr_at_3_std": 0.227138, + "nauc_mrr_at_3_diff1": 0.226784, + "nauc_mrr_at_5_max": 0.554312, + "nauc_mrr_at_5_std": 0.206918, + "nauc_mrr_at_5_diff1": 0.240085, + "nauc_mrr_at_10_max": 0.55514, + "nauc_mrr_at_10_std": 0.207673, + "nauc_mrr_at_10_diff1": 0.245835, + "nauc_mrr_at_20_max": 0.553781, + "nauc_mrr_at_20_std": 0.206619, + "nauc_mrr_at_20_diff1": 0.245449, + "nauc_mrr_at_100_max": 0.553131, + "nauc_mrr_at_100_std": 0.20589, + "nauc_mrr_at_100_diff1": 0.247139, + "nauc_mrr_at_1000_max": 0.553131, + "nauc_mrr_at_1000_std": 0.20589, + "nauc_mrr_at_1000_diff1": 0.247139, + "hit_rate_at_1": 0.66092, + "hit_rate_at_3": 0.85345, + "hit_rate_at_5": 0.90517, + "hit_rate_at_10": 0.9454, + "hit_rate_at_20": 0.97126, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.65785, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.61207, + "ndcg_at_3": 0.57635, + "ndcg_at_5": 0.5898, + "ndcg_at_10": 0.61831, + "ndcg_at_20": 0.65628, + "ndcg_at_100": 0.70384, + "ndcg_at_1000": 0.70384, + "map_at_1": 0.26605, + "map_at_3": 0.40954, + "map_at_5": 0.45567, + "map_at_10": 0.49753, + "map_at_20": 0.52601, + "map_at_100": 0.54852, + "map_at_1000": 0.54852, + "recall_at_1": 0.26605, + "recall_at_3": 0.4623, + "recall_at_5": 0.54942, + "recall_at_10": 0.66385, + "recall_at_20": 0.77601, + "recall_at_100": 0.92187, + "recall_at_1000": 0.92187, + "accuracy": 0.26605, + "precision_at_1": 0.62931, + "precision_at_3": 0.42529, + "precision_at_5": 0.33678, + "precision_at_10": 0.22644, + "precision_at_20": 0.1477, + "precision_at_100": 0.03882, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.62931, + "mrr_at_3": 0.706418, + "mrr_at_5": 0.722653, + "mrr_at_10": 0.730083, + "mrr_at_20": 0.732825, + "mrr_at_100": 0.733605, + "mrr_at_1000": 0.733605, + "nauc_ndcg_at_1_max": 0.38757, + "nauc_ndcg_at_1_std": 0.158459, + "nauc_ndcg_at_1_diff1": 0.477096, + "nauc_ndcg_at_3_max": 0.4135, + "nauc_ndcg_at_3_std": 0.223052, + "nauc_ndcg_at_3_diff1": 0.379267, + "nauc_ndcg_at_5_max": 0.415143, + "nauc_ndcg_at_5_std": 0.236147, + "nauc_ndcg_at_5_diff1": 0.396268, + "nauc_ndcg_at_10_max": 0.421706, + "nauc_ndcg_at_10_std": 0.256392, + "nauc_ndcg_at_10_diff1": 0.405713, + "nauc_ndcg_at_20_max": 0.432836, + "nauc_ndcg_at_20_std": 0.267369, + "nauc_ndcg_at_20_diff1": 0.402534, + "nauc_ndcg_at_100_max": 0.429467, + "nauc_ndcg_at_100_std": 0.259925, + "nauc_ndcg_at_100_diff1": 0.397719, + "nauc_ndcg_at_1000_max": 0.429467, + "nauc_ndcg_at_1000_std": 0.259925, + "nauc_ndcg_at_1000_diff1": 0.397719, + "nauc_map_at_1_max": 0.245901, + "nauc_map_at_1_std": 0.044954, + "nauc_map_at_1_diff1": 0.422153, + "nauc_map_at_3_max": 0.344729, + "nauc_map_at_3_std": 0.163222, + "nauc_map_at_3_diff1": 0.343962, + "nauc_map_at_5_max": 0.367958, + "nauc_map_at_5_std": 0.191308, + "nauc_map_at_5_diff1": 0.3531, + "nauc_map_at_10_max": 0.388595, + "nauc_map_at_10_std": 0.223073, + "nauc_map_at_10_diff1": 0.364224, + "nauc_map_at_20_max": 0.403171, + "nauc_map_at_20_std": 0.241004, + "nauc_map_at_20_diff1": 0.370118, + "nauc_map_at_100_max": 0.400239, + "nauc_map_at_100_std": 0.238054, + "nauc_map_at_100_diff1": 0.368166, + "nauc_map_at_1000_max": 0.400239, + "nauc_map_at_1000_std": 0.238054, + "nauc_map_at_1000_diff1": 0.368166, + "nauc_recall_at_1_max": 0.245901, + "nauc_recall_at_1_std": 0.044954, + "nauc_recall_at_1_diff1": 0.422153, + "nauc_recall_at_3_max": 0.323969, + "nauc_recall_at_3_std": 0.175195, + "nauc_recall_at_3_diff1": 0.282412, + "nauc_recall_at_5_max": 0.343911, + "nauc_recall_at_5_std": 0.204109, + "nauc_recall_at_5_diff1": 0.297774, + "nauc_recall_at_10_max": 0.348297, + "nauc_recall_at_10_std": 0.248294, + "nauc_recall_at_10_diff1": 0.309489, + "nauc_recall_at_20_max": 0.357806, + "nauc_recall_at_20_std": 0.273335, + "nauc_recall_at_20_diff1": 0.296076, + "nauc_recall_at_100_max": 0.376786, + "nauc_recall_at_100_std": 0.326914, + "nauc_recall_at_100_diff1": 0.201674, + "nauc_recall_at_1000_max": 0.376786, + "nauc_recall_at_1000_std": 0.326914, + "nauc_recall_at_1000_diff1": 0.201674, + "nauc_precision_at_1_max": 0.428392, + "nauc_precision_at_1_std": 0.187198, + "nauc_precision_at_1_diff1": 0.508487, + "nauc_precision_at_3_max": 0.357181, + "nauc_precision_at_3_std": 0.286161, + "nauc_precision_at_3_diff1": 0.166884, + "nauc_precision_at_5_max": 0.288855, + "nauc_precision_at_5_std": 0.275002, + "nauc_precision_at_5_diff1": 0.11566, + "nauc_precision_at_10_max": 0.182309, + "nauc_precision_at_10_std": 0.232536, + "nauc_precision_at_10_diff1": 0.052595, + "nauc_precision_at_20_max": 0.072562, + "nauc_precision_at_20_std": 0.145117, + "nauc_precision_at_20_diff1": -0.020069, + "nauc_precision_at_100_max": -0.055615, + "nauc_precision_at_100_std": 0.027524, + "nauc_precision_at_100_diff1": -0.103462, + "nauc_precision_at_1000_max": -0.055615, + "nauc_precision_at_1000_std": 0.027524, + "nauc_precision_at_1000_diff1": -0.103462, + "nauc_mrr_at_1_max": 0.428392, + "nauc_mrr_at_1_std": 0.187198, + "nauc_mrr_at_1_diff1": 0.508487, + "nauc_mrr_at_3_max": 0.474094, + "nauc_mrr_at_3_std": 0.248502, + "nauc_mrr_at_3_diff1": 0.497666, + "nauc_mrr_at_5_max": 0.475418, + "nauc_mrr_at_5_std": 0.251937, + "nauc_mrr_at_5_diff1": 0.509582, + "nauc_mrr_at_10_max": 0.473537, + "nauc_mrr_at_10_std": 0.246755, + "nauc_mrr_at_10_diff1": 0.507162, + "nauc_mrr_at_20_max": 0.472157, + "nauc_mrr_at_20_std": 0.24477, + "nauc_mrr_at_20_diff1": 0.50578, + "nauc_mrr_at_100_max": 0.471511, + "nauc_mrr_at_100_std": 0.24387, + "nauc_mrr_at_100_diff1": 0.506115, + "nauc_mrr_at_1000_max": 0.471511, + "nauc_mrr_at_1000_std": 0.24387, + "nauc_mrr_at_1000_diff1": 0.506115, + "hit_rate_at_1": 0.62931, + "hit_rate_at_3": 0.80747, + "hit_rate_at_5": 0.87931, + "hit_rate_at_10": 0.93103, + "hit_rate_at_20": 0.96839, + "hit_rate_at_100": 0.99138, + "hit_rate_at_1000": 0.99138, + "main_score": 0.61831, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.57471, + "ndcg_at_3": 0.56702, + "ndcg_at_5": 0.5793, + "ndcg_at_10": 0.60839, + "ndcg_at_20": 0.64609, + "ndcg_at_100": 0.69695, + "ndcg_at_1000": 0.69695, + "map_at_1": 0.25908, + "map_at_3": 0.4031, + "map_at_5": 0.44828, + "map_at_10": 0.49259, + "map_at_20": 0.52115, + "map_at_100": 0.54468, + "map_at_1000": 0.54468, + "recall_at_1": 0.25908, + "recall_at_3": 0.45659, + "recall_at_5": 0.5454, + "recall_at_10": 0.6623, + "recall_at_20": 0.77263, + "recall_at_100": 0.93008, + "recall_at_1000": 0.93008, + "accuracy": 0.25908, + "precision_at_1": 0.60057, + "precision_at_3": 0.4272, + "precision_at_5": 0.33621, + "precision_at_10": 0.22816, + "precision_at_20": 0.14655, + "precision_at_100": 0.03905, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.600575, + "mrr_at_3": 0.690134, + "mrr_at_5": 0.702634, + "mrr_at_10": 0.712401, + "mrr_at_20": 0.715043, + "mrr_at_100": 0.715982, + "mrr_at_1000": 0.715982, + "nauc_ndcg_at_1_max": 0.416685, + "nauc_ndcg_at_1_std": 0.207491, + "nauc_ndcg_at_1_diff1": 0.511967, + "nauc_ndcg_at_3_max": 0.395813, + "nauc_ndcg_at_3_std": 0.241669, + "nauc_ndcg_at_3_diff1": 0.375183, + "nauc_ndcg_at_5_max": 0.422118, + "nauc_ndcg_at_5_std": 0.261742, + "nauc_ndcg_at_5_diff1": 0.390365, + "nauc_ndcg_at_10_max": 0.436337, + "nauc_ndcg_at_10_std": 0.281682, + "nauc_ndcg_at_10_diff1": 0.396326, + "nauc_ndcg_at_20_max": 0.462908, + "nauc_ndcg_at_20_std": 0.312149, + "nauc_ndcg_at_20_diff1": 0.394484, + "nauc_ndcg_at_100_max": 0.448247, + "nauc_ndcg_at_100_std": 0.289471, + "nauc_ndcg_at_100_diff1": 0.403286, + "nauc_ndcg_at_1000_max": 0.448247, + "nauc_ndcg_at_1000_std": 0.289471, + "nauc_ndcg_at_1000_diff1": 0.403286, + "nauc_map_at_1_max": 0.259199, + "nauc_map_at_1_std": 0.029134, + "nauc_map_at_1_diff1": 0.467328, + "nauc_map_at_3_max": 0.347723, + "nauc_map_at_3_std": 0.165838, + "nauc_map_at_3_diff1": 0.370708, + "nauc_map_at_5_max": 0.369177, + "nauc_map_at_5_std": 0.192032, + "nauc_map_at_5_diff1": 0.370398, + "nauc_map_at_10_max": 0.388608, + "nauc_map_at_10_std": 0.227852, + "nauc_map_at_10_diff1": 0.362883, + "nauc_map_at_20_max": 0.411143, + "nauc_map_at_20_std": 0.260015, + "nauc_map_at_20_diff1": 0.358355, + "nauc_map_at_100_max": 0.407135, + "nauc_map_at_100_std": 0.256132, + "nauc_map_at_100_diff1": 0.360726, + "nauc_map_at_1000_max": 0.407135, + "nauc_map_at_1000_std": 0.256132, + "nauc_map_at_1000_diff1": 0.360726, + "nauc_recall_at_1_max": 0.259199, + "nauc_recall_at_1_std": 0.029134, + "nauc_recall_at_1_diff1": 0.467328, + "nauc_recall_at_3_max": 0.326903, + "nauc_recall_at_3_std": 0.181218, + "nauc_recall_at_3_diff1": 0.314942, + "nauc_recall_at_5_max": 0.350947, + "nauc_recall_at_5_std": 0.209675, + "nauc_recall_at_5_diff1": 0.306034, + "nauc_recall_at_10_max": 0.378614, + "nauc_recall_at_10_std": 0.281845, + "nauc_recall_at_10_diff1": 0.278969, + "nauc_recall_at_20_max": 0.452499, + "nauc_recall_at_20_std": 0.389908, + "nauc_recall_at_20_diff1": 0.244545, + "nauc_recall_at_100_max": 0.484852, + "nauc_recall_at_100_std": 0.432722, + "nauc_recall_at_100_diff1": 0.223584, + "nauc_recall_at_1000_max": 0.484852, + "nauc_recall_at_1000_std": 0.432722, + "nauc_recall_at_1000_diff1": 0.223584, + "nauc_precision_at_1_max": 0.44641, + "nauc_precision_at_1_std": 0.227614, + "nauc_precision_at_1_diff1": 0.517585, + "nauc_precision_at_3_max": 0.307283, + "nauc_precision_at_3_std": 0.295937, + "nauc_precision_at_3_diff1": 0.102082, + "nauc_precision_at_5_max": 0.250805, + "nauc_precision_at_5_std": 0.293584, + "nauc_precision_at_5_diff1": 0.016505, + "nauc_precision_at_10_max": 0.158871, + "nauc_precision_at_10_std": 0.25564, + "nauc_precision_at_10_diff1": -0.063693, + "nauc_precision_at_20_max": 0.093044, + "nauc_precision_at_20_std": 0.214262, + "nauc_precision_at_20_diff1": -0.117148, + "nauc_precision_at_100_max": -0.057025, + "nauc_precision_at_100_std": 0.061364, + "nauc_precision_at_100_diff1": -0.164076, + "nauc_precision_at_1000_max": -0.057025, + "nauc_precision_at_1000_std": 0.061364, + "nauc_precision_at_1000_diff1": -0.164076, + "nauc_mrr_at_1_max": 0.44641, + "nauc_mrr_at_1_std": 0.227614, + "nauc_mrr_at_1_diff1": 0.517585, + "nauc_mrr_at_3_max": 0.495426, + "nauc_mrr_at_3_std": 0.302878, + "nauc_mrr_at_3_diff1": 0.486932, + "nauc_mrr_at_5_max": 0.505104, + "nauc_mrr_at_5_std": 0.312779, + "nauc_mrr_at_5_diff1": 0.488223, + "nauc_mrr_at_10_max": 0.497634, + "nauc_mrr_at_10_std": 0.302052, + "nauc_mrr_at_10_diff1": 0.49486, + "nauc_mrr_at_20_max": 0.496556, + "nauc_mrr_at_20_std": 0.299677, + "nauc_mrr_at_20_diff1": 0.49469, + "nauc_mrr_at_100_max": 0.495557, + "nauc_mrr_at_100_std": 0.298169, + "nauc_mrr_at_100_diff1": 0.494783, + "nauc_mrr_at_1000_max": 0.495557, + "nauc_mrr_at_1000_std": 0.298169, + "nauc_mrr_at_1000_diff1": 0.494783, + "hit_rate_at_1": 0.60057, + "hit_rate_at_3": 0.8046, + "hit_rate_at_5": 0.8592, + "hit_rate_at_10": 0.93103, + "hit_rate_at_20": 0.96552, + "hit_rate_at_100": 0.99713, + "hit_rate_at_1000": 0.99713, + "main_score": 0.60839, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.59339, + "ndcg_at_3": 0.58407, + "ndcg_at_5": 0.59447, + "ndcg_at_10": 0.62254, + "ndcg_at_20": 0.66231, + "ndcg_at_100": 0.70889, + "ndcg_at_1000": 0.70889, + "map_at_1": 0.26541, + "map_at_3": 0.4123, + "map_at_5": 0.45718, + "map_at_10": 0.50352, + "map_at_20": 0.53407, + "map_at_100": 0.55552, + "map_at_1000": 0.55552, + "recall_at_1": 0.26541, + "recall_at_3": 0.4633, + "recall_at_5": 0.55399, + "recall_at_10": 0.67113, + "recall_at_20": 0.79188, + "recall_at_100": 0.93189, + "recall_at_1000": 0.93189, + "accuracy": 0.26541, + "precision_at_1": 0.61207, + "precision_at_3": 0.43487, + "precision_at_5": 0.34195, + "precision_at_10": 0.23075, + "precision_at_20": 0.15043, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.612069, + "mrr_at_3": 0.703544, + "mrr_at_5": 0.720929, + "mrr_at_10": 0.728394, + "mrr_at_20": 0.730497, + "mrr_at_100": 0.731166, + "mrr_at_1000": 0.731166, + "nauc_ndcg_at_1_max": 0.424379, + "nauc_ndcg_at_1_std": 0.26877, + "nauc_ndcg_at_1_diff1": 0.429852, + "nauc_ndcg_at_3_max": 0.383262, + "nauc_ndcg_at_3_std": 0.25219, + "nauc_ndcg_at_3_diff1": 0.343634, + "nauc_ndcg_at_5_max": 0.415954, + "nauc_ndcg_at_5_std": 0.291123, + "nauc_ndcg_at_5_diff1": 0.357004, + "nauc_ndcg_at_10_max": 0.404123, + "nauc_ndcg_at_10_std": 0.27861, + "nauc_ndcg_at_10_diff1": 0.373726, + "nauc_ndcg_at_20_max": 0.431164, + "nauc_ndcg_at_20_std": 0.3078, + "nauc_ndcg_at_20_diff1": 0.382929, + "nauc_ndcg_at_100_max": 0.41452, + "nauc_ndcg_at_100_std": 0.286086, + "nauc_ndcg_at_100_diff1": 0.387997, + "nauc_ndcg_at_1000_max": 0.41452, + "nauc_ndcg_at_1000_std": 0.286086, + "nauc_ndcg_at_1000_diff1": 0.387997, + "nauc_map_at_1_max": 0.231263, + "nauc_map_at_1_std": 0.090926, + "nauc_map_at_1_diff1": 0.370568, + "nauc_map_at_3_max": 0.321291, + "nauc_map_at_3_std": 0.183625, + "nauc_map_at_3_diff1": 0.34796, + "nauc_map_at_5_max": 0.348146, + "nauc_map_at_5_std": 0.222984, + "nauc_map_at_5_diff1": 0.344087, + "nauc_map_at_10_max": 0.362974, + "nauc_map_at_10_std": 0.245223, + "nauc_map_at_10_diff1": 0.346073, + "nauc_map_at_20_max": 0.379687, + "nauc_map_at_20_std": 0.265734, + "nauc_map_at_20_diff1": 0.347863, + "nauc_map_at_100_max": 0.370775, + "nauc_map_at_100_std": 0.257077, + "nauc_map_at_100_diff1": 0.348336, + "nauc_map_at_1000_max": 0.370775, + "nauc_map_at_1000_std": 0.257077, + "nauc_map_at_1000_diff1": 0.348336, + "nauc_recall_at_1_max": 0.231263, + "nauc_recall_at_1_std": 0.090926, + "nauc_recall_at_1_diff1": 0.370568, + "nauc_recall_at_3_max": 0.324286, + "nauc_recall_at_3_std": 0.199107, + "nauc_recall_at_3_diff1": 0.320816, + "nauc_recall_at_5_max": 0.341822, + "nauc_recall_at_5_std": 0.246518, + "nauc_recall_at_5_diff1": 0.283774, + "nauc_recall_at_10_max": 0.330541, + "nauc_recall_at_10_std": 0.248348, + "nauc_recall_at_10_diff1": 0.283615, + "nauc_recall_at_20_max": 0.405889, + "nauc_recall_at_20_std": 0.354944, + "nauc_recall_at_20_diff1": 0.284464, + "nauc_recall_at_100_max": 0.427835, + "nauc_recall_at_100_std": 0.386326, + "nauc_recall_at_100_diff1": 0.349257, + "nauc_recall_at_1000_max": 0.427835, + "nauc_recall_at_1000_std": 0.386326, + "nauc_recall_at_1000_diff1": 0.349257, + "nauc_precision_at_1_max": 0.429504, + "nauc_precision_at_1_std": 0.269198, + "nauc_precision_at_1_diff1": 0.451277, + "nauc_precision_at_3_max": 0.295793, + "nauc_precision_at_3_std": 0.266933, + "nauc_precision_at_3_diff1": 0.10586, + "nauc_precision_at_5_max": 0.266998, + "nauc_precision_at_5_std": 0.29096, + "nauc_precision_at_5_diff1": 0.038692, + "nauc_precision_at_10_max": 0.134727, + "nauc_precision_at_10_std": 0.180983, + "nauc_precision_at_10_diff1": -0.020861, + "nauc_precision_at_20_max": 0.055829, + "nauc_precision_at_20_std": 0.119006, + "nauc_precision_at_20_diff1": -0.070297, + "nauc_precision_at_100_max": -0.091161, + "nauc_precision_at_100_std": -0.020856, + "nauc_precision_at_100_diff1": -0.126441, + "nauc_precision_at_1000_max": -0.091161, + "nauc_precision_at_1000_std": -0.020856, + "nauc_precision_at_1000_diff1": -0.126441, + "nauc_mrr_at_1_max": 0.429504, + "nauc_mrr_at_1_std": 0.269198, + "nauc_mrr_at_1_diff1": 0.451277, + "nauc_mrr_at_3_max": 0.496193, + "nauc_mrr_at_3_std": 0.331705, + "nauc_mrr_at_3_diff1": 0.453036, + "nauc_mrr_at_5_max": 0.483002, + "nauc_mrr_at_5_std": 0.319689, + "nauc_mrr_at_5_diff1": 0.450398, + "nauc_mrr_at_10_max": 0.477281, + "nauc_mrr_at_10_std": 0.309644, + "nauc_mrr_at_10_diff1": 0.452368, + "nauc_mrr_at_20_max": 0.478782, + "nauc_mrr_at_20_std": 0.312753, + "nauc_mrr_at_20_diff1": 0.453271, + "nauc_mrr_at_100_max": 0.478103, + "nauc_mrr_at_100_std": 0.312192, + "nauc_mrr_at_100_diff1": 0.453197, + "nauc_mrr_at_1000_max": 0.478103, + "nauc_mrr_at_1000_std": 0.312192, + "nauc_mrr_at_1000_diff1": 0.453197, + "hit_rate_at_1": 0.61207, + "hit_rate_at_3": 0.81322, + "hit_rate_at_5": 0.88793, + "hit_rate_at_10": 0.94253, + "hit_rate_at_20": 0.97126, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.62254, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.58908, + "ndcg_at_3": 0.57148, + "ndcg_at_5": 0.58162, + "ndcg_at_10": 0.61012, + "ndcg_at_20": 0.6508, + "ndcg_at_100": 0.69879, + "ndcg_at_1000": 0.69879, + "map_at_1": 0.25554, + "map_at_3": 0.39887, + "map_at_5": 0.44564, + "map_at_10": 0.49106, + "map_at_20": 0.52129, + "map_at_100": 0.54318, + "map_at_1000": 0.54318, + "recall_at_1": 0.25554, + "recall_at_3": 0.45891, + "recall_at_5": 0.54847, + "recall_at_10": 0.66247, + "recall_at_20": 0.78017, + "recall_at_100": 0.93007, + "recall_at_1000": 0.93007, + "accuracy": 0.25554, + "precision_at_1": 0.61494, + "precision_at_3": 0.42529, + "precision_at_5": 0.33621, + "precision_at_10": 0.22845, + "precision_at_20": 0.14914, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.614943, + "mrr_at_3": 0.704502, + "mrr_at_5": 0.718151, + "mrr_at_10": 0.725299, + "mrr_at_20": 0.727951, + "mrr_at_100": 0.72906, + "mrr_at_1000": 0.72906, + "nauc_ndcg_at_1_max": 0.404157, + "nauc_ndcg_at_1_std": 0.23125, + "nauc_ndcg_at_1_diff1": 0.463737, + "nauc_ndcg_at_3_max": 0.385394, + "nauc_ndcg_at_3_std": 0.243694, + "nauc_ndcg_at_3_diff1": 0.339376, + "nauc_ndcg_at_5_max": 0.419307, + "nauc_ndcg_at_5_std": 0.277571, + "nauc_ndcg_at_5_diff1": 0.384418, + "nauc_ndcg_at_10_max": 0.432826, + "nauc_ndcg_at_10_std": 0.299968, + "nauc_ndcg_at_10_diff1": 0.405322, + "nauc_ndcg_at_20_max": 0.430466, + "nauc_ndcg_at_20_std": 0.297861, + "nauc_ndcg_at_20_diff1": 0.407338, + "nauc_ndcg_at_100_max": 0.424807, + "nauc_ndcg_at_100_std": 0.291117, + "nauc_ndcg_at_100_diff1": 0.408886, + "nauc_ndcg_at_1000_max": 0.424807, + "nauc_ndcg_at_1000_std": 0.291117, + "nauc_ndcg_at_1000_diff1": 0.408886, + "nauc_map_at_1_max": 0.234748, + "nauc_map_at_1_std": 0.055277, + "nauc_map_at_1_diff1": 0.450503, + "nauc_map_at_3_max": 0.323571, + "nauc_map_at_3_std": 0.168971, + "nauc_map_at_3_diff1": 0.356256, + "nauc_map_at_5_max": 0.356641, + "nauc_map_at_5_std": 0.208527, + "nauc_map_at_5_diff1": 0.368045, + "nauc_map_at_10_max": 0.377455, + "nauc_map_at_10_std": 0.241891, + "nauc_map_at_10_diff1": 0.3758, + "nauc_map_at_20_max": 0.382121, + "nauc_map_at_20_std": 0.255164, + "nauc_map_at_20_diff1": 0.378415, + "nauc_map_at_100_max": 0.379957, + "nauc_map_at_100_std": 0.255546, + "nauc_map_at_100_diff1": 0.379987, + "nauc_map_at_1000_max": 0.379957, + "nauc_map_at_1000_std": 0.255546, + "nauc_map_at_1000_diff1": 0.379987, + "nauc_recall_at_1_max": 0.234748, + "nauc_recall_at_1_std": 0.055277, + "nauc_recall_at_1_diff1": 0.450503, + "nauc_recall_at_3_max": 0.305607, + "nauc_recall_at_3_std": 0.179088, + "nauc_recall_at_3_diff1": 0.278196, + "nauc_recall_at_5_max": 0.341944, + "nauc_recall_at_5_std": 0.230061, + "nauc_recall_at_5_diff1": 0.291987, + "nauc_recall_at_10_max": 0.366866, + "nauc_recall_at_10_std": 0.273364, + "nauc_recall_at_10_diff1": 0.325172, + "nauc_recall_at_20_max": 0.362355, + "nauc_recall_at_20_std": 0.297272, + "nauc_recall_at_20_diff1": 0.330846, + "nauc_recall_at_100_max": 0.392576, + "nauc_recall_at_100_std": 0.33163, + "nauc_recall_at_100_diff1": 0.344522, + "nauc_recall_at_1000_max": 0.392576, + "nauc_recall_at_1000_std": 0.33163, + "nauc_recall_at_1000_diff1": 0.344522, + "nauc_precision_at_1_max": 0.408093, + "nauc_precision_at_1_std": 0.23437, + "nauc_precision_at_1_diff1": 0.494846, + "nauc_precision_at_3_max": 0.308467, + "nauc_precision_at_3_std": 0.29626, + "nauc_precision_at_3_diff1": 0.072878, + "nauc_precision_at_5_max": 0.26491, + "nauc_precision_at_5_std": 0.298869, + "nauc_precision_at_5_diff1": 0.066628, + "nauc_precision_at_10_max": 0.177388, + "nauc_precision_at_10_std": 0.265005, + "nauc_precision_at_10_diff1": -0.001583, + "nauc_precision_at_20_max": 0.057597, + "nauc_precision_at_20_std": 0.174307, + "nauc_precision_at_20_diff1": -0.076433, + "nauc_precision_at_100_max": -0.066397, + "nauc_precision_at_100_std": 0.057695, + "nauc_precision_at_100_diff1": -0.146073, + "nauc_precision_at_1000_max": -0.066397, + "nauc_precision_at_1000_std": 0.057695, + "nauc_precision_at_1000_diff1": -0.146073, + "nauc_mrr_at_1_max": 0.408093, + "nauc_mrr_at_1_std": 0.23437, + "nauc_mrr_at_1_diff1": 0.494846, + "nauc_mrr_at_3_max": 0.459821, + "nauc_mrr_at_3_std": 0.294887, + "nauc_mrr_at_3_diff1": 0.456988, + "nauc_mrr_at_5_max": 0.460418, + "nauc_mrr_at_5_std": 0.30133, + "nauc_mrr_at_5_diff1": 0.467786, + "nauc_mrr_at_10_max": 0.456658, + "nauc_mrr_at_10_std": 0.293388, + "nauc_mrr_at_10_diff1": 0.468171, + "nauc_mrr_at_20_max": 0.45435, + "nauc_mrr_at_20_std": 0.290655, + "nauc_mrr_at_20_diff1": 0.468598, + "nauc_mrr_at_100_max": 0.45318, + "nauc_mrr_at_100_std": 0.289071, + "nauc_mrr_at_100_diff1": 0.468187, + "nauc_mrr_at_1000_max": 0.45318, + "nauc_mrr_at_1000_std": 0.289071, + "nauc_mrr_at_1000_diff1": 0.468187, + "hit_rate_at_1": 0.61494, + "hit_rate_at_3": 0.81034, + "hit_rate_at_5": 0.86782, + "hit_rate_at_10": 0.92241, + "hit_rate_at_20": 0.95977, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.61012, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.57615, + "ndcg_at_3": 0.55996, + "ndcg_at_5": 0.56707, + "ndcg_at_10": 0.60432, + "ndcg_at_20": 0.64258, + "ndcg_at_100": 0.69118, + "ndcg_at_1000": 0.69118, + "map_at_1": 0.24472, + "map_at_3": 0.38574, + "map_at_5": 0.42795, + "map_at_10": 0.47834, + "map_at_20": 0.50739, + "map_at_100": 0.52986, + "map_at_1000": 0.52986, + "recall_at_1": 0.24472, + "recall_at_3": 0.44732, + "recall_at_5": 0.53849, + "recall_at_10": 0.67176, + "recall_at_20": 0.78224, + "recall_at_100": 0.93268, + "recall_at_1000": 0.93268, + "accuracy": 0.24472, + "precision_at_1": 0.60057, + "precision_at_3": 0.41858, + "precision_at_5": 0.32644, + "precision_at_10": 0.2296, + "precision_at_20": 0.14928, + "precision_at_100": 0.03937, + "precision_at_1000": 0.00394, + "mrr_at_1": 0.600575, + "mrr_at_3": 0.69205, + "mrr_at_5": 0.707423, + "mrr_at_10": 0.714828, + "mrr_at_20": 0.717322, + "mrr_at_100": 0.718324, + "mrr_at_1000": 0.718324, + "nauc_ndcg_at_1_max": 0.406032, + "nauc_ndcg_at_1_std": 0.234453, + "nauc_ndcg_at_1_diff1": 0.456253, + "nauc_ndcg_at_3_max": 0.407342, + "nauc_ndcg_at_3_std": 0.258084, + "nauc_ndcg_at_3_diff1": 0.375286, + "nauc_ndcg_at_5_max": 0.406275, + "nauc_ndcg_at_5_std": 0.265504, + "nauc_ndcg_at_5_diff1": 0.402306, + "nauc_ndcg_at_10_max": 0.421982, + "nauc_ndcg_at_10_std": 0.285801, + "nauc_ndcg_at_10_diff1": 0.417323, + "nauc_ndcg_at_20_max": 0.430302, + "nauc_ndcg_at_20_std": 0.294777, + "nauc_ndcg_at_20_diff1": 0.42129, + "nauc_ndcg_at_100_max": 0.428815, + "nauc_ndcg_at_100_std": 0.29177, + "nauc_ndcg_at_100_diff1": 0.426179, + "nauc_ndcg_at_1000_max": 0.428815, + "nauc_ndcg_at_1000_std": 0.29177, + "nauc_ndcg_at_1000_diff1": 0.426179, + "nauc_map_at_1_max": 0.217091, + "nauc_map_at_1_std": 0.054749, + "nauc_map_at_1_diff1": 0.443757, + "nauc_map_at_3_max": 0.30347, + "nauc_map_at_3_std": 0.146332, + "nauc_map_at_3_diff1": 0.37735, + "nauc_map_at_5_max": 0.323155, + "nauc_map_at_5_std": 0.174648, + "nauc_map_at_5_diff1": 0.381437, + "nauc_map_at_10_max": 0.36022, + "nauc_map_at_10_std": 0.220157, + "nauc_map_at_10_diff1": 0.385906, + "nauc_map_at_20_max": 0.376193, + "nauc_map_at_20_std": 0.241519, + "nauc_map_at_20_diff1": 0.388872, + "nauc_map_at_100_max": 0.376196, + "nauc_map_at_100_std": 0.243383, + "nauc_map_at_100_diff1": 0.390528, + "nauc_map_at_1000_max": 0.376196, + "nauc_map_at_1000_std": 0.243383, + "nauc_map_at_1000_diff1": 0.390528, + "nauc_recall_at_1_max": 0.217091, + "nauc_recall_at_1_std": 0.054749, + "nauc_recall_at_1_diff1": 0.443757, + "nauc_recall_at_3_max": 0.276032, + "nauc_recall_at_3_std": 0.150424, + "nauc_recall_at_3_diff1": 0.296712, + "nauc_recall_at_5_max": 0.26788, + "nauc_recall_at_5_std": 0.168626, + "nauc_recall_at_5_diff1": 0.283722, + "nauc_recall_at_10_max": 0.310749, + "nauc_recall_at_10_std": 0.226456, + "nauc_recall_at_10_diff1": 0.306694, + "nauc_recall_at_20_max": 0.332594, + "nauc_recall_at_20_std": 0.272197, + "nauc_recall_at_20_diff1": 0.304927, + "nauc_recall_at_100_max": 0.389833, + "nauc_recall_at_100_std": 0.355247, + "nauc_recall_at_100_diff1": 0.330005, + "nauc_recall_at_1000_max": 0.389833, + "nauc_recall_at_1000_std": 0.355247, + "nauc_recall_at_1000_diff1": 0.330005, + "nauc_precision_at_1_max": 0.403659, + "nauc_precision_at_1_std": 0.220995, + "nauc_precision_at_1_diff1": 0.471179, + "nauc_precision_at_3_max": 0.360719, + "nauc_precision_at_3_std": 0.299992, + "nauc_precision_at_3_diff1": 0.128237, + "nauc_precision_at_5_max": 0.311763, + "nauc_precision_at_5_std": 0.302767, + "nauc_precision_at_5_diff1": 0.070516, + "nauc_precision_at_10_max": 0.24412, + "nauc_precision_at_10_std": 0.290799, + "nauc_precision_at_10_diff1": -0.005393, + "nauc_precision_at_20_max": 0.132211, + "nauc_precision_at_20_std": 0.200961, + "nauc_precision_at_20_diff1": -0.067294, + "nauc_precision_at_100_max": -0.002059, + "nauc_precision_at_100_std": 0.082646, + "nauc_precision_at_100_diff1": -0.131227, + "nauc_precision_at_1000_max": -0.002059, + "nauc_precision_at_1000_std": 0.082646, + "nauc_precision_at_1000_diff1": -0.131227, + "nauc_mrr_at_1_max": 0.403659, + "nauc_mrr_at_1_std": 0.220995, + "nauc_mrr_at_1_diff1": 0.471179, + "nauc_mrr_at_3_max": 0.457069, + "nauc_mrr_at_3_std": 0.283751, + "nauc_mrr_at_3_diff1": 0.451828, + "nauc_mrr_at_5_max": 0.450304, + "nauc_mrr_at_5_std": 0.277212, + "nauc_mrr_at_5_diff1": 0.455886, + "nauc_mrr_at_10_max": 0.449485, + "nauc_mrr_at_10_std": 0.274741, + "nauc_mrr_at_10_diff1": 0.460339, + "nauc_mrr_at_20_max": 0.446411, + "nauc_mrr_at_20_std": 0.271256, + "nauc_mrr_at_20_diff1": 0.459666, + "nauc_mrr_at_100_max": 0.445527, + "nauc_mrr_at_100_std": 0.270128, + "nauc_mrr_at_100_diff1": 0.460025, + "nauc_mrr_at_1000_max": 0.445527, + "nauc_mrr_at_1000_std": 0.270128, + "nauc_mrr_at_1000_diff1": 0.460025, + "hit_rate_at_1": 0.60057, + "hit_rate_at_3": 0.8046, + "hit_rate_at_5": 0.87356, + "hit_rate_at_10": 0.92816, + "hit_rate_at_20": 0.96264, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.60432, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 2280.434517058893, + "kg_co2_emissions": null, + "date": 1792441180.167892, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 1.758697908371687e-05, + "end": 31.403032931964844, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 31.40305731794797, + "end": 31.403059000964276, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries and documents", + "start": 31.407974813017063, + "end": 408.603709753952, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 408.6037479409715, + "end": 408.65567881893367, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries and documents", + "start": 408.66773684101645, + "end": 783.4941108989296, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 783.4941510389326, + "end": 783.541837287019, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries and documents", + "start": 783.5623214379884, + "end": 1157.781307014986, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 1157.7813524729572, + "end": 1157.8288717119722, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries and documents", + "start": 1157.841399385943, + "end": 1531.4483183659613, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 1531.4483643339481, + "end": 1531.4974202249432, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries and documents", + "start": 1531.512371985009, + "end": 1905.4323064169148, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 1905.4323520449689, + "end": 1905.4801349620102, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries and documents", + "start": 1905.4939746749587, + "end": 2280.3771224640077, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 2280.3771488929633, + "end": 2280.4243111260002, + "split": "test", + "subset": "portuguese" + } + ] +} diff --git a/results/BAAI__bge-reranker-v2-m3/953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e/run_settings.jsonl b/results/BAAI__bge-reranker-v2-m3/953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e/run_settings.jsonl index a490db5b28..62bc31e1ef 100644 --- a/results/BAAI__bge-reranker-v2-m3/953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e/run_settings.jsonl +++ b/results/BAAI__bge-reranker-v2-m3/953dc6f6f85a1b2dbfca4c34a2796e7dde08d41e/run_settings.jsonl @@ -38,3 +38,5 @@ {"task": "Vidore3PhysicsNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.0", "torch": "2.14.1", "sentence-transformers": "5.7.0", "flash-attn": null, "transformers": "4.51.3"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "5.7.0", "flash-attn": null, "transformers": "4.51.3"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "5.7.0", "flash-attn": null, "transformers": "4.51.3"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "5.7.0", "flash-attn": null, "transformers": "4.51.3"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "5.7.0", "flash-attn": null, "transformers": "4.51.3"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} diff --git a/results/Qwen__Qwen3-VL-Reranker-2B/4bd860ac4f15ad1897a214615cccc700f8f71818/Vidore3NuclearNemotron35Embed8BReranking.v2.json b/results/Qwen__Qwen3-VL-Reranker-2B/4bd860ac4f15ad1897a214615cccc700f8f71818/Vidore3NuclearNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..8ec73dab8d --- /dev/null +++ b/results/Qwen__Qwen3-VL-Reranker-2B/4bd860ac4f15ad1897a214615cccc700f8f71818/Vidore3NuclearNemotron35Embed8BReranking.v2.json @@ -0,0 +1,992 @@ +{ + "dataset_revision": "97ff9e10e265e64abece589ff40c552215754f26", + "task_name": "Vidore3NuclearNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.4503, + "ndcg_at_3": 0.49075, + "ndcg_at_5": 0.49555, + "ndcg_at_10": 0.52199, + "ndcg_at_20": 0.55987, + "ndcg_at_100": 0.60292, + "ndcg_at_1000": 0.60292, + "map_at_1": 0.21676, + "map_at_3": 0.33834, + "map_at_5": 0.37532, + "map_at_10": 0.41258, + "map_at_20": 0.43801, + "map_at_100": 0.45567, + "map_at_1000": 0.45567, + "recall_at_1": 0.21676, + "recall_at_3": 0.4104, + "recall_at_5": 0.4806, + "recall_at_10": 0.58387, + "recall_at_20": 0.68492, + "recall_at_100": 0.81132, + "recall_at_1000": 0.81132, + "accuracy": 0.21676, + "precision_at_1": 0.46687, + "precision_at_3": 0.36245, + "precision_at_5": 0.28434, + "precision_at_10": 0.19548, + "precision_at_20": 0.12846, + "precision_at_100": 0.0338, + "precision_at_1000": 0.00338, + "mrr_at_1": 0.466867, + "mrr_at_3": 0.586847, + "mrr_at_5": 0.600552, + "mrr_at_10": 0.608773, + "mrr_at_20": 0.612285, + "mrr_at_100": 0.61366, + "mrr_at_1000": 0.61366, + "nauc_ndcg_at_1_max": 0.271487, + "nauc_ndcg_at_1_std": 0.171952, + "nauc_ndcg_at_1_diff1": 0.421378, + "nauc_ndcg_at_3_max": 0.384263, + "nauc_ndcg_at_3_std": 0.248563, + "nauc_ndcg_at_3_diff1": 0.362234, + "nauc_ndcg_at_5_max": 0.39371, + "nauc_ndcg_at_5_std": 0.282123, + "nauc_ndcg_at_5_diff1": 0.374885, + "nauc_ndcg_at_10_max": 0.399528, + "nauc_ndcg_at_10_std": 0.314325, + "nauc_ndcg_at_10_diff1": 0.385734, + "nauc_ndcg_at_20_max": 0.39971, + "nauc_ndcg_at_20_std": 0.349992, + "nauc_ndcg_at_20_diff1": 0.381083, + "nauc_ndcg_at_100_max": 0.393408, + "nauc_ndcg_at_100_std": 0.357751, + "nauc_ndcg_at_100_diff1": 0.376664, + "nauc_ndcg_at_1000_max": 0.393408, + "nauc_ndcg_at_1000_std": 0.357751, + "nauc_ndcg_at_1000_diff1": 0.376664, + "nauc_map_at_1_max": 0.198617, + "nauc_map_at_1_std": -0.003893, + "nauc_map_at_1_diff1": 0.459419, + "nauc_map_at_3_max": 0.311429, + "nauc_map_at_3_std": 0.126676, + "nauc_map_at_3_diff1": 0.393511, + "nauc_map_at_5_max": 0.344762, + "nauc_map_at_5_std": 0.197812, + "nauc_map_at_5_diff1": 0.379583, + "nauc_map_at_10_max": 0.353228, + "nauc_map_at_10_std": 0.247042, + "nauc_map_at_10_diff1": 0.377827, + "nauc_map_at_20_max": 0.352407, + "nauc_map_at_20_std": 0.275549, + "nauc_map_at_20_diff1": 0.3766, + "nauc_map_at_100_max": 0.349201, + "nauc_map_at_100_std": 0.2829, + "nauc_map_at_100_diff1": 0.373601, + "nauc_map_at_1000_max": 0.349201, + "nauc_map_at_1000_std": 0.2829, + "nauc_map_at_1000_diff1": 0.373601, + "nauc_recall_at_1_max": 0.198617, + "nauc_recall_at_1_std": -0.003893, + "nauc_recall_at_1_diff1": 0.459419, + "nauc_recall_at_3_max": 0.344541, + "nauc_recall_at_3_std": 0.178725, + "nauc_recall_at_3_diff1": 0.312917, + "nauc_recall_at_5_max": 0.384223, + "nauc_recall_at_5_std": 0.282318, + "nauc_recall_at_5_diff1": 0.301786, + "nauc_recall_at_10_max": 0.400964, + "nauc_recall_at_10_std": 0.376908, + "nauc_recall_at_10_diff1": 0.288169, + "nauc_recall_at_20_max": 0.391181, + "nauc_recall_at_20_std": 0.466823, + "nauc_recall_at_20_diff1": 0.266711, + "nauc_recall_at_100_max": 0.416183, + "nauc_recall_at_100_std": 0.553501, + "nauc_recall_at_100_diff1": 0.217795, + "nauc_recall_at_1000_max": 0.416183, + "nauc_recall_at_1000_std": 0.553501, + "nauc_recall_at_1000_diff1": 0.217795, + "nauc_precision_at_1_max": 0.272236, + "nauc_precision_at_1_std": 0.199801, + "nauc_precision_at_1_diff1": 0.411796, + "nauc_precision_at_3_max": 0.352231, + "nauc_precision_at_3_std": 0.384309, + "nauc_precision_at_3_diff1": 0.150032, + "nauc_precision_at_5_max": 0.297204, + "nauc_precision_at_5_std": 0.432691, + "nauc_precision_at_5_diff1": 0.084518, + "nauc_precision_at_10_max": 0.170948, + "nauc_precision_at_10_std": 0.392576, + "nauc_precision_at_10_diff1": 0.035556, + "nauc_precision_at_20_max": 0.074824, + "nauc_precision_at_20_std": 0.331402, + "nauc_precision_at_20_diff1": -0.017305, + "nauc_precision_at_100_max": -0.046589, + "nauc_precision_at_100_std": 0.205319, + "nauc_precision_at_100_diff1": -0.092474, + "nauc_precision_at_1000_max": -0.046589, + "nauc_precision_at_1000_std": 0.205319, + "nauc_precision_at_1000_diff1": -0.092474, + "nauc_mrr_at_1_max": 0.272236, + "nauc_mrr_at_1_std": 0.199801, + "nauc_mrr_at_1_diff1": 0.411796, + "nauc_mrr_at_3_max": 0.37763, + "nauc_mrr_at_3_std": 0.319895, + "nauc_mrr_at_3_diff1": 0.386183, + "nauc_mrr_at_5_max": 0.359614, + "nauc_mrr_at_5_std": 0.315973, + "nauc_mrr_at_5_diff1": 0.388497, + "nauc_mrr_at_10_max": 0.360239, + "nauc_mrr_at_10_std": 0.321268, + "nauc_mrr_at_10_diff1": 0.389891, + "nauc_mrr_at_20_max": 0.3579, + "nauc_mrr_at_20_std": 0.31851, + "nauc_mrr_at_20_diff1": 0.387813, + "nauc_mrr_at_100_max": 0.358129, + "nauc_mrr_at_100_std": 0.316616, + "nauc_mrr_at_100_diff1": 0.388495, + "nauc_mrr_at_1000_max": 0.358129, + "nauc_mrr_at_1000_std": 0.316616, + "nauc_mrr_at_1000_diff1": 0.388495, + "hit_rate_at_1": 0.46687, + "hit_rate_at_3": 0.73795, + "hit_rate_at_5": 0.79819, + "hit_rate_at_10": 0.85843, + "hit_rate_at_20": 0.90663, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.52199, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.4262, + "ndcg_at_3": 0.45325, + "ndcg_at_5": 0.46559, + "ndcg_at_10": 0.4927, + "ndcg_at_20": 0.53161, + "ndcg_at_100": 0.57709, + "ndcg_at_1000": 0.57709, + "map_at_1": 0.20595, + "map_at_3": 0.30673, + "map_at_5": 0.34565, + "map_at_10": 0.38151, + "map_at_20": 0.40677, + "map_at_100": 0.42454, + "map_at_1000": 0.42454, + "recall_at_1": 0.20595, + "recall_at_3": 0.37078, + "recall_at_5": 0.45324, + "recall_at_10": 0.55419, + "recall_at_20": 0.65696, + "recall_at_100": 0.79964, + "recall_at_1000": 0.79964, + "accuracy": 0.20595, + "precision_at_1": 0.43976, + "precision_at_3": 0.32631, + "precision_at_5": 0.26627, + "precision_at_10": 0.18675, + "precision_at_20": 0.1244, + "precision_at_100": 0.0328, + "precision_at_1000": 0.00328, + "mrr_at_1": 0.439759, + "mrr_at_3": 0.555221, + "mrr_at_5": 0.571787, + "mrr_at_10": 0.581352, + "mrr_at_20": 0.584322, + "mrr_at_100": 0.586659, + "mrr_at_1000": 0.586659, + "nauc_ndcg_at_1_max": 0.238866, + "nauc_ndcg_at_1_std": 0.20085, + "nauc_ndcg_at_1_diff1": 0.377898, + "nauc_ndcg_at_3_max": 0.322361, + "nauc_ndcg_at_3_std": 0.286006, + "nauc_ndcg_at_3_diff1": 0.318427, + "nauc_ndcg_at_5_max": 0.343124, + "nauc_ndcg_at_5_std": 0.325655, + "nauc_ndcg_at_5_diff1": 0.324589, + "nauc_ndcg_at_10_max": 0.343519, + "nauc_ndcg_at_10_std": 0.359253, + "nauc_ndcg_at_10_diff1": 0.327216, + "nauc_ndcg_at_20_max": 0.341506, + "nauc_ndcg_at_20_std": 0.398537, + "nauc_ndcg_at_20_diff1": 0.332739, + "nauc_ndcg_at_100_max": 0.324704, + "nauc_ndcg_at_100_std": 0.396465, + "nauc_ndcg_at_100_diff1": 0.320274, + "nauc_ndcg_at_1000_max": 0.324704, + "nauc_ndcg_at_1000_std": 0.396465, + "nauc_ndcg_at_1000_diff1": 0.320274, + "nauc_map_at_1_max": 0.212138, + "nauc_map_at_1_std": 0.121414, + "nauc_map_at_1_diff1": 0.353329, + "nauc_map_at_3_max": 0.286384, + "nauc_map_at_3_std": 0.208881, + "nauc_map_at_3_diff1": 0.341173, + "nauc_map_at_5_max": 0.299944, + "nauc_map_at_5_std": 0.261136, + "nauc_map_at_5_diff1": 0.330293, + "nauc_map_at_10_max": 0.303379, + "nauc_map_at_10_std": 0.301156, + "nauc_map_at_10_diff1": 0.32524, + "nauc_map_at_20_max": 0.300983, + "nauc_map_at_20_std": 0.326351, + "nauc_map_at_20_diff1": 0.325432, + "nauc_map_at_100_max": 0.295195, + "nauc_map_at_100_std": 0.331731, + "nauc_map_at_100_diff1": 0.321573, + "nauc_map_at_1000_max": 0.295195, + "nauc_map_at_1000_std": 0.331731, + "nauc_map_at_1000_diff1": 0.321573, + "nauc_recall_at_1_max": 0.212138, + "nauc_recall_at_1_std": 0.121414, + "nauc_recall_at_1_diff1": 0.353329, + "nauc_recall_at_3_max": 0.32463, + "nauc_recall_at_3_std": 0.246812, + "nauc_recall_at_3_diff1": 0.269118, + "nauc_recall_at_5_max": 0.344629, + "nauc_recall_at_5_std": 0.338337, + "nauc_recall_at_5_diff1": 0.25869, + "nauc_recall_at_10_max": 0.345441, + "nauc_recall_at_10_std": 0.426861, + "nauc_recall_at_10_diff1": 0.237577, + "nauc_recall_at_20_max": 0.352742, + "nauc_recall_at_20_std": 0.536761, + "nauc_recall_at_20_diff1": 0.261968, + "nauc_recall_at_100_max": 0.307199, + "nauc_recall_at_100_std": 0.582233, + "nauc_recall_at_100_diff1": 0.196355, + "nauc_recall_at_1000_max": 0.307199, + "nauc_recall_at_1000_std": 0.582233, + "nauc_recall_at_1000_diff1": 0.196355, + "nauc_precision_at_1_max": 0.255735, + "nauc_precision_at_1_std": 0.227311, + "nauc_precision_at_1_diff1": 0.373993, + "nauc_precision_at_3_max": 0.267491, + "nauc_precision_at_3_std": 0.34538, + "nauc_precision_at_3_diff1": 0.195367, + "nauc_precision_at_5_max": 0.210907, + "nauc_precision_at_5_std": 0.356982, + "nauc_precision_at_5_diff1": 0.121455, + "nauc_precision_at_10_max": 0.122875, + "nauc_precision_at_10_std": 0.327484, + "nauc_precision_at_10_diff1": 0.036453, + "nauc_precision_at_20_max": 0.033006, + "nauc_precision_at_20_std": 0.276465, + "nauc_precision_at_20_diff1": -0.008717, + "nauc_precision_at_100_max": -0.057228, + "nauc_precision_at_100_std": 0.14992, + "nauc_precision_at_100_diff1": -0.086491, + "nauc_precision_at_1000_max": -0.057228, + "nauc_precision_at_1000_std": 0.14992, + "nauc_precision_at_1000_diff1": -0.086491, + "nauc_mrr_at_1_max": 0.255735, + "nauc_mrr_at_1_std": 0.227311, + "nauc_mrr_at_1_diff1": 0.373993, + "nauc_mrr_at_3_max": 0.336822, + "nauc_mrr_at_3_std": 0.327996, + "nauc_mrr_at_3_diff1": 0.343703, + "nauc_mrr_at_5_max": 0.33223, + "nauc_mrr_at_5_std": 0.336861, + "nauc_mrr_at_5_diff1": 0.349696, + "nauc_mrr_at_10_max": 0.328104, + "nauc_mrr_at_10_std": 0.33246, + "nauc_mrr_at_10_diff1": 0.345816, + "nauc_mrr_at_20_max": 0.325877, + "nauc_mrr_at_20_std": 0.332089, + "nauc_mrr_at_20_diff1": 0.350902, + "nauc_mrr_at_100_max": 0.326122, + "nauc_mrr_at_100_std": 0.330386, + "nauc_mrr_at_100_diff1": 0.350933, + "nauc_mrr_at_1000_max": 0.326122, + "nauc_mrr_at_1000_std": 0.330386, + "nauc_mrr_at_1000_diff1": 0.350933, + "hit_rate_at_1": 0.43976, + "hit_rate_at_3": 0.69277, + "hit_rate_at_5": 0.76506, + "hit_rate_at_10": 0.84036, + "hit_rate_at_20": 0.88253, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.4927, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.43675, + "ndcg_at_3": 0.455, + "ndcg_at_5": 0.4589, + "ndcg_at_10": 0.4899, + "ndcg_at_20": 0.53561, + "ndcg_at_100": 0.57917, + "ndcg_at_1000": 0.57917, + "map_at_1": 0.20772, + "map_at_3": 0.31039, + "map_at_5": 0.34414, + "map_at_10": 0.38036, + "map_at_20": 0.40855, + "map_at_100": 0.42556, + "map_at_1000": 0.42556, + "recall_at_1": 0.20772, + "recall_at_3": 0.37301, + "recall_at_5": 0.43696, + "recall_at_10": 0.5446, + "recall_at_20": 0.67121, + "recall_at_100": 0.80754, + "recall_at_1000": 0.80754, + "accuracy": 0.20772, + "precision_at_1": 0.45482, + "precision_at_3": 0.32731, + "precision_at_5": 0.25663, + "precision_at_10": 0.18434, + "precision_at_20": 0.12636, + "precision_at_100": 0.03304, + "precision_at_1000": 0.0033, + "mrr_at_1": 0.454819, + "mrr_at_3": 0.560241, + "mrr_at_5": 0.571084, + "mrr_at_10": 0.583752, + "mrr_at_20": 0.588583, + "mrr_at_100": 0.590434, + "mrr_at_1000": 0.590434, + "nauc_ndcg_at_1_max": 0.30144, + "nauc_ndcg_at_1_std": 0.277922, + "nauc_ndcg_at_1_diff1": 0.425765, + "nauc_ndcg_at_3_max": 0.340249, + "nauc_ndcg_at_3_std": 0.312726, + "nauc_ndcg_at_3_diff1": 0.347726, + "nauc_ndcg_at_5_max": 0.35195, + "nauc_ndcg_at_5_std": 0.332387, + "nauc_ndcg_at_5_diff1": 0.355431, + "nauc_ndcg_at_10_max": 0.353031, + "nauc_ndcg_at_10_std": 0.355766, + "nauc_ndcg_at_10_diff1": 0.367309, + "nauc_ndcg_at_20_max": 0.361584, + "nauc_ndcg_at_20_std": 0.41458, + "nauc_ndcg_at_20_diff1": 0.362558, + "nauc_ndcg_at_100_max": 0.35708, + "nauc_ndcg_at_100_std": 0.408376, + "nauc_ndcg_at_100_diff1": 0.363007, + "nauc_ndcg_at_1000_max": 0.35708, + "nauc_ndcg_at_1000_std": 0.408376, + "nauc_ndcg_at_1000_diff1": 0.363007, + "nauc_map_at_1_max": 0.244974, + "nauc_map_at_1_std": 0.15623, + "nauc_map_at_1_diff1": 0.39609, + "nauc_map_at_3_max": 0.298237, + "nauc_map_at_3_std": 0.222331, + "nauc_map_at_3_diff1": 0.361179, + "nauc_map_at_5_max": 0.313383, + "nauc_map_at_5_std": 0.265328, + "nauc_map_at_5_diff1": 0.343958, + "nauc_map_at_10_max": 0.320829, + "nauc_map_at_10_std": 0.304582, + "nauc_map_at_10_diff1": 0.345795, + "nauc_map_at_20_max": 0.32222, + "nauc_map_at_20_std": 0.338255, + "nauc_map_at_20_diff1": 0.343945, + "nauc_map_at_100_max": 0.319541, + "nauc_map_at_100_std": 0.340563, + "nauc_map_at_100_diff1": 0.341438, + "nauc_map_at_1000_max": 0.319541, + "nauc_map_at_1000_std": 0.340563, + "nauc_map_at_1000_diff1": 0.341438, + "nauc_recall_at_1_max": 0.244974, + "nauc_recall_at_1_std": 0.15623, + "nauc_recall_at_1_diff1": 0.39609, + "nauc_recall_at_3_max": 0.324386, + "nauc_recall_at_3_std": 0.25456, + "nauc_recall_at_3_diff1": 0.297461, + "nauc_recall_at_5_max": 0.341162, + "nauc_recall_at_5_std": 0.313694, + "nauc_recall_at_5_diff1": 0.29321, + "nauc_recall_at_10_max": 0.324815, + "nauc_recall_at_10_std": 0.350794, + "nauc_recall_at_10_diff1": 0.273842, + "nauc_recall_at_20_max": 0.341193, + "nauc_recall_at_20_std": 0.52393, + "nauc_recall_at_20_diff1": 0.264392, + "nauc_recall_at_100_max": 0.357708, + "nauc_recall_at_100_std": 0.555507, + "nauc_recall_at_100_diff1": 0.258857, + "nauc_recall_at_1000_max": 0.357708, + "nauc_recall_at_1000_std": 0.555507, + "nauc_recall_at_1000_diff1": 0.258857, + "nauc_precision_at_1_max": 0.318281, + "nauc_precision_at_1_std": 0.296781, + "nauc_precision_at_1_diff1": 0.429578, + "nauc_precision_at_3_max": 0.252808, + "nauc_precision_at_3_std": 0.330422, + "nauc_precision_at_3_diff1": 0.194976, + "nauc_precision_at_5_max": 0.207855, + "nauc_precision_at_5_std": 0.332959, + "nauc_precision_at_5_diff1": 0.107493, + "nauc_precision_at_10_max": 0.118224, + "nauc_precision_at_10_std": 0.310631, + "nauc_precision_at_10_diff1": 0.044541, + "nauc_precision_at_20_max": 0.033916, + "nauc_precision_at_20_std": 0.276286, + "nauc_precision_at_20_diff1": -0.010692, + "nauc_precision_at_100_max": -0.055897, + "nauc_precision_at_100_std": 0.140212, + "nauc_precision_at_100_diff1": -0.086669, + "nauc_precision_at_1000_max": -0.055897, + "nauc_precision_at_1000_std": 0.140212, + "nauc_precision_at_1000_diff1": -0.086669, + "nauc_mrr_at_1_max": 0.318281, + "nauc_mrr_at_1_std": 0.296781, + "nauc_mrr_at_1_diff1": 0.429578, + "nauc_mrr_at_3_max": 0.373068, + "nauc_mrr_at_3_std": 0.380814, + "nauc_mrr_at_3_diff1": 0.41846, + "nauc_mrr_at_5_max": 0.364016, + "nauc_mrr_at_5_std": 0.378813, + "nauc_mrr_at_5_diff1": 0.429126, + "nauc_mrr_at_10_max": 0.359854, + "nauc_mrr_at_10_std": 0.379767, + "nauc_mrr_at_10_diff1": 0.424892, + "nauc_mrr_at_20_max": 0.359442, + "nauc_mrr_at_20_std": 0.380701, + "nauc_mrr_at_20_diff1": 0.425147, + "nauc_mrr_at_100_max": 0.359762, + "nauc_mrr_at_100_std": 0.378752, + "nauc_mrr_at_100_diff1": 0.425166, + "nauc_mrr_at_1000_max": 0.359762, + "nauc_mrr_at_1000_std": 0.378752, + "nauc_mrr_at_1000_diff1": 0.425166, + "hit_rate_at_1": 0.45482, + "hit_rate_at_3": 0.68373, + "hit_rate_at_5": 0.73193, + "hit_rate_at_10": 0.82831, + "hit_rate_at_20": 0.89759, + "hit_rate_at_100": 0.95181, + "hit_rate_at_1000": 0.95181, + "main_score": 0.4899, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.42922, + "ndcg_at_3": 0.45157, + "ndcg_at_5": 0.46686, + "ndcg_at_10": 0.48934, + "ndcg_at_20": 0.53511, + "ndcg_at_100": 0.58003, + "ndcg_at_1000": 0.58003, + "map_at_1": 0.20363, + "map_at_3": 0.30325, + "map_at_5": 0.34611, + "map_at_10": 0.38015, + "map_at_20": 0.40795, + "map_at_100": 0.42553, + "map_at_1000": 0.42553, + "recall_at_1": 0.20363, + "recall_at_3": 0.36657, + "recall_at_5": 0.45153, + "recall_at_10": 0.54545, + "recall_at_20": 0.67021, + "recall_at_100": 0.80792, + "recall_at_1000": 0.80792, + "accuracy": 0.20363, + "precision_at_1": 0.44578, + "precision_at_3": 0.3253, + "precision_at_5": 0.26807, + "precision_at_10": 0.18434, + "precision_at_20": 0.1262, + "precision_at_100": 0.03334, + "precision_at_1000": 0.00333, + "mrr_at_1": 0.445783, + "mrr_at_3": 0.563755, + "mrr_at_5": 0.575954, + "mrr_at_10": 0.583557, + "mrr_at_20": 0.588137, + "mrr_at_100": 0.590494, + "mrr_at_1000": 0.590494, + "nauc_ndcg_at_1_max": 0.222917, + "nauc_ndcg_at_1_std": 0.198341, + "nauc_ndcg_at_1_diff1": 0.440912, + "nauc_ndcg_at_3_max": 0.310176, + "nauc_ndcg_at_3_std": 0.307852, + "nauc_ndcg_at_3_diff1": 0.352645, + "nauc_ndcg_at_5_max": 0.315112, + "nauc_ndcg_at_5_std": 0.32639, + "nauc_ndcg_at_5_diff1": 0.354121, + "nauc_ndcg_at_10_max": 0.303372, + "nauc_ndcg_at_10_std": 0.354951, + "nauc_ndcg_at_10_diff1": 0.355587, + "nauc_ndcg_at_20_max": 0.29881, + "nauc_ndcg_at_20_std": 0.396518, + "nauc_ndcg_at_20_diff1": 0.363596, + "nauc_ndcg_at_100_max": 0.288601, + "nauc_ndcg_at_100_std": 0.385285, + "nauc_ndcg_at_100_diff1": 0.368729, + "nauc_ndcg_at_1000_max": 0.288601, + "nauc_ndcg_at_1000_std": 0.385285, + "nauc_ndcg_at_1000_diff1": 0.368729, + "nauc_map_at_1_max": 0.170332, + "nauc_map_at_1_std": 0.099619, + "nauc_map_at_1_diff1": 0.447495, + "nauc_map_at_3_max": 0.266779, + "nauc_map_at_3_std": 0.228643, + "nauc_map_at_3_diff1": 0.385882, + "nauc_map_at_5_max": 0.276136, + "nauc_map_at_5_std": 0.264928, + "nauc_map_at_5_diff1": 0.362766, + "nauc_map_at_10_max": 0.277129, + "nauc_map_at_10_std": 0.304891, + "nauc_map_at_10_diff1": 0.355248, + "nauc_map_at_20_max": 0.273139, + "nauc_map_at_20_std": 0.332568, + "nauc_map_at_20_diff1": 0.351461, + "nauc_map_at_100_max": 0.265902, + "nauc_map_at_100_std": 0.334315, + "nauc_map_at_100_diff1": 0.350299, + "nauc_map_at_1000_max": 0.265902, + "nauc_map_at_1000_std": 0.334315, + "nauc_map_at_1000_diff1": 0.350299, + "nauc_recall_at_1_max": 0.170332, + "nauc_recall_at_1_std": 0.099619, + "nauc_recall_at_1_diff1": 0.447495, + "nauc_recall_at_3_max": 0.30386, + "nauc_recall_at_3_std": 0.270516, + "nauc_recall_at_3_diff1": 0.311558, + "nauc_recall_at_5_max": 0.306484, + "nauc_recall_at_5_std": 0.312955, + "nauc_recall_at_5_diff1": 0.271531, + "nauc_recall_at_10_max": 0.287356, + "nauc_recall_at_10_std": 0.393941, + "nauc_recall_at_10_diff1": 0.257359, + "nauc_recall_at_20_max": 0.262398, + "nauc_recall_at_20_std": 0.505276, + "nauc_recall_at_20_diff1": 0.277053, + "nauc_recall_at_100_max": 0.25794, + "nauc_recall_at_100_std": 0.503847, + "nauc_recall_at_100_diff1": 0.28048, + "nauc_recall_at_1000_max": 0.25794, + "nauc_recall_at_1000_std": 0.503847, + "nauc_recall_at_1000_diff1": 0.28048, + "nauc_precision_at_1_max": 0.240297, + "nauc_precision_at_1_std": 0.23078, + "nauc_precision_at_1_diff1": 0.427416, + "nauc_precision_at_3_max": 0.26717, + "nauc_precision_at_3_std": 0.366361, + "nauc_precision_at_3_diff1": 0.144708, + "nauc_precision_at_5_max": 0.195583, + "nauc_precision_at_5_std": 0.340445, + "nauc_precision_at_5_diff1": 0.056678, + "nauc_precision_at_10_max": 0.095071, + "nauc_precision_at_10_std": 0.326855, + "nauc_precision_at_10_diff1": -0.00571, + "nauc_precision_at_20_max": 0.011107, + "nauc_precision_at_20_std": 0.272317, + "nauc_precision_at_20_diff1": -0.053011, + "nauc_precision_at_100_max": -0.082543, + "nauc_precision_at_100_std": 0.133791, + "nauc_precision_at_100_diff1": -0.105813, + "nauc_precision_at_1000_max": -0.082543, + "nauc_precision_at_1000_std": 0.133791, + "nauc_precision_at_1000_diff1": -0.105813, + "nauc_mrr_at_1_max": 0.240297, + "nauc_mrr_at_1_std": 0.23078, + "nauc_mrr_at_1_diff1": 0.427416, + "nauc_mrr_at_3_max": 0.311969, + "nauc_mrr_at_3_std": 0.341406, + "nauc_mrr_at_3_diff1": 0.382565, + "nauc_mrr_at_5_max": 0.308289, + "nauc_mrr_at_5_std": 0.343933, + "nauc_mrr_at_5_diff1": 0.378595, + "nauc_mrr_at_10_max": 0.30228, + "nauc_mrr_at_10_std": 0.343199, + "nauc_mrr_at_10_diff1": 0.380858, + "nauc_mrr_at_20_max": 0.301489, + "nauc_mrr_at_20_std": 0.343055, + "nauc_mrr_at_20_diff1": 0.381882, + "nauc_mrr_at_100_max": 0.303308, + "nauc_mrr_at_100_std": 0.341637, + "nauc_mrr_at_100_diff1": 0.383882, + "nauc_mrr_at_1000_max": 0.303308, + "nauc_mrr_at_1000_std": 0.341637, + "nauc_mrr_at_1000_diff1": 0.383882, + "hit_rate_at_1": 0.44578, + "hit_rate_at_3": 0.70482, + "hit_rate_at_5": 0.75904, + "hit_rate_at_10": 0.81928, + "hit_rate_at_20": 0.88253, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.48934, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.4488, + "ndcg_at_3": 0.46613, + "ndcg_at_5": 0.47928, + "ndcg_at_10": 0.50183, + "ndcg_at_20": 0.5432, + "ndcg_at_100": 0.58691, + "ndcg_at_1000": 0.58691, + "map_at_1": 0.20631, + "map_at_3": 0.31504, + "map_at_5": 0.35556, + "map_at_10": 0.39049, + "map_at_20": 0.4172, + "map_at_100": 0.43462, + "map_at_1000": 0.43462, + "recall_at_1": 0.20631, + "recall_at_3": 0.38586, + "recall_at_5": 0.46742, + "recall_at_10": 0.55834, + "recall_at_20": 0.67379, + "recall_at_100": 0.80575, + "recall_at_1000": 0.80575, + "accuracy": 0.20631, + "precision_at_1": 0.46386, + "precision_at_3": 0.34036, + "precision_at_5": 0.27651, + "precision_at_10": 0.19006, + "precision_at_20": 0.12711, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.463855, + "mrr_at_3": 0.569779, + "mrr_at_5": 0.584237, + "mrr_at_10": 0.593338, + "mrr_at_20": 0.597458, + "mrr_at_100": 0.599457, + "mrr_at_1000": 0.599457, + "nauc_ndcg_at_1_max": 0.267342, + "nauc_ndcg_at_1_std": 0.219435, + "nauc_ndcg_at_1_diff1": 0.408857, + "nauc_ndcg_at_3_max": 0.363702, + "nauc_ndcg_at_3_std": 0.298717, + "nauc_ndcg_at_3_diff1": 0.330634, + "nauc_ndcg_at_5_max": 0.377684, + "nauc_ndcg_at_5_std": 0.327071, + "nauc_ndcg_at_5_diff1": 0.346784, + "nauc_ndcg_at_10_max": 0.386043, + "nauc_ndcg_at_10_std": 0.360656, + "nauc_ndcg_at_10_diff1": 0.36712, + "nauc_ndcg_at_20_max": 0.381093, + "nauc_ndcg_at_20_std": 0.410705, + "nauc_ndcg_at_20_diff1": 0.358395, + "nauc_ndcg_at_100_max": 0.361052, + "nauc_ndcg_at_100_std": 0.398504, + "nauc_ndcg_at_100_diff1": 0.3496, + "nauc_ndcg_at_1000_max": 0.361052, + "nauc_ndcg_at_1000_std": 0.398504, + "nauc_ndcg_at_1000_diff1": 0.3496, + "nauc_map_at_1_max": 0.258617, + "nauc_map_at_1_std": 0.144603, + "nauc_map_at_1_diff1": 0.421204, + "nauc_map_at_3_max": 0.327254, + "nauc_map_at_3_std": 0.218999, + "nauc_map_at_3_diff1": 0.353551, + "nauc_map_at_5_max": 0.347562, + "nauc_map_at_5_std": 0.269672, + "nauc_map_at_5_diff1": 0.33981, + "nauc_map_at_10_max": 0.356188, + "nauc_map_at_10_std": 0.311837, + "nauc_map_at_10_diff1": 0.344022, + "nauc_map_at_20_max": 0.348778, + "nauc_map_at_20_std": 0.34038, + "nauc_map_at_20_diff1": 0.339713, + "nauc_map_at_100_max": 0.340564, + "nauc_map_at_100_std": 0.341406, + "nauc_map_at_100_diff1": 0.336452, + "nauc_map_at_1000_max": 0.340564, + "nauc_map_at_1000_std": 0.341406, + "nauc_map_at_1000_diff1": 0.336452, + "nauc_recall_at_1_max": 0.258617, + "nauc_recall_at_1_std": 0.144603, + "nauc_recall_at_1_diff1": 0.421204, + "nauc_recall_at_3_max": 0.337558, + "nauc_recall_at_3_std": 0.22789, + "nauc_recall_at_3_diff1": 0.262734, + "nauc_recall_at_5_max": 0.362458, + "nauc_recall_at_5_std": 0.303988, + "nauc_recall_at_5_diff1": 0.263786, + "nauc_recall_at_10_max": 0.370271, + "nauc_recall_at_10_std": 0.386963, + "nauc_recall_at_10_diff1": 0.286151, + "nauc_recall_at_20_max": 0.3594, + "nauc_recall_at_20_std": 0.532175, + "nauc_recall_at_20_diff1": 0.255713, + "nauc_recall_at_100_max": 0.301953, + "nauc_recall_at_100_std": 0.526691, + "nauc_recall_at_100_diff1": 0.205635, + "nauc_recall_at_1000_max": 0.301953, + "nauc_recall_at_1000_std": 0.526691, + "nauc_recall_at_1000_diff1": 0.205635, + "nauc_precision_at_1_max": 0.276091, + "nauc_precision_at_1_std": 0.242276, + "nauc_precision_at_1_diff1": 0.401368, + "nauc_precision_at_3_max": 0.276622, + "nauc_precision_at_3_std": 0.326433, + "nauc_precision_at_3_diff1": 0.164898, + "nauc_precision_at_5_max": 0.214093, + "nauc_precision_at_5_std": 0.330877, + "nauc_precision_at_5_diff1": 0.091597, + "nauc_precision_at_10_max": 0.126708, + "nauc_precision_at_10_std": 0.308455, + "nauc_precision_at_10_diff1": 0.04612, + "nauc_precision_at_20_max": 0.021969, + "nauc_precision_at_20_std": 0.263964, + "nauc_precision_at_20_diff1": -0.014712, + "nauc_precision_at_100_max": -0.082503, + "nauc_precision_at_100_std": 0.12942, + "nauc_precision_at_100_diff1": -0.087072, + "nauc_precision_at_1000_max": -0.082503, + "nauc_precision_at_1000_std": 0.12942, + "nauc_precision_at_1000_diff1": -0.087072, + "nauc_mrr_at_1_max": 0.276091, + "nauc_mrr_at_1_std": 0.242276, + "nauc_mrr_at_1_diff1": 0.401368, + "nauc_mrr_at_3_max": 0.373342, + "nauc_mrr_at_3_std": 0.332348, + "nauc_mrr_at_3_diff1": 0.371803, + "nauc_mrr_at_5_max": 0.362159, + "nauc_mrr_at_5_std": 0.336752, + "nauc_mrr_at_5_diff1": 0.380137, + "nauc_mrr_at_10_max": 0.360581, + "nauc_mrr_at_10_std": 0.338057, + "nauc_mrr_at_10_diff1": 0.378624, + "nauc_mrr_at_20_max": 0.360341, + "nauc_mrr_at_20_std": 0.340065, + "nauc_mrr_at_20_diff1": 0.378317, + "nauc_mrr_at_100_max": 0.360198, + "nauc_mrr_at_100_std": 0.337602, + "nauc_mrr_at_100_diff1": 0.378256, + "nauc_mrr_at_1000_max": 0.360198, + "nauc_mrr_at_1000_std": 0.337602, + "nauc_mrr_at_1000_diff1": 0.378256, + "hit_rate_at_1": 0.46386, + "hit_rate_at_3": 0.70482, + "hit_rate_at_5": 0.76807, + "hit_rate_at_10": 0.83133, + "hit_rate_at_20": 0.88855, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.50183, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.45181, + "ndcg_at_3": 0.46524, + "ndcg_at_5": 0.47464, + "ndcg_at_10": 0.50258, + "ndcg_at_20": 0.54252, + "ndcg_at_100": 0.58717, + "ndcg_at_1000": 0.58717, + "map_at_1": 0.20998, + "map_at_3": 0.3138, + "map_at_5": 0.35169, + "map_at_10": 0.39088, + "map_at_20": 0.41702, + "map_at_100": 0.43443, + "map_at_1000": 0.43443, + "recall_at_1": 0.20998, + "recall_at_3": 0.37736, + "recall_at_5": 0.45159, + "recall_at_10": 0.56228, + "recall_at_20": 0.66996, + "recall_at_100": 0.80599, + "recall_at_1000": 0.80599, + "accuracy": 0.20998, + "precision_at_1": 0.46687, + "precision_at_3": 0.33333, + "precision_at_5": 0.26988, + "precision_at_10": 0.18886, + "precision_at_20": 0.12711, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.466867, + "mrr_at_3": 0.573795, + "mrr_at_5": 0.586295, + "mrr_at_10": 0.596077, + "mrr_at_20": 0.599767, + "mrr_at_100": 0.602132, + "mrr_at_1000": 0.602132, + "nauc_ndcg_at_1_max": 0.291352, + "nauc_ndcg_at_1_std": 0.211059, + "nauc_ndcg_at_1_diff1": 0.346402, + "nauc_ndcg_at_3_max": 0.326183, + "nauc_ndcg_at_3_std": 0.276097, + "nauc_ndcg_at_3_diff1": 0.249844, + "nauc_ndcg_at_5_max": 0.356676, + "nauc_ndcg_at_5_std": 0.309932, + "nauc_ndcg_at_5_diff1": 0.272496, + "nauc_ndcg_at_10_max": 0.354551, + "nauc_ndcg_at_10_std": 0.354472, + "nauc_ndcg_at_10_diff1": 0.279618, + "nauc_ndcg_at_20_max": 0.354229, + "nauc_ndcg_at_20_std": 0.397587, + "nauc_ndcg_at_20_diff1": 0.27042, + "nauc_ndcg_at_100_max": 0.344138, + "nauc_ndcg_at_100_std": 0.388643, + "nauc_ndcg_at_100_diff1": 0.274278, + "nauc_ndcg_at_1000_max": 0.344138, + "nauc_ndcg_at_1000_std": 0.388643, + "nauc_ndcg_at_1000_diff1": 0.274278, + "nauc_map_at_1_max": 0.226398, + "nauc_map_at_1_std": 0.111525, + "nauc_map_at_1_diff1": 0.356592, + "nauc_map_at_3_max": 0.26829, + "nauc_map_at_3_std": 0.193263, + "nauc_map_at_3_diff1": 0.260372, + "nauc_map_at_5_max": 0.306047, + "nauc_map_at_5_std": 0.247094, + "nauc_map_at_5_diff1": 0.265907, + "nauc_map_at_10_max": 0.30989, + "nauc_map_at_10_std": 0.294666, + "nauc_map_at_10_diff1": 0.257047, + "nauc_map_at_20_max": 0.308346, + "nauc_map_at_20_std": 0.322804, + "nauc_map_at_20_diff1": 0.250179, + "nauc_map_at_100_max": 0.302819, + "nauc_map_at_100_std": 0.324004, + "nauc_map_at_100_diff1": 0.250627, + "nauc_map_at_1000_max": 0.302819, + "nauc_map_at_1000_std": 0.324004, + "nauc_map_at_1000_diff1": 0.250627, + "nauc_recall_at_1_max": 0.226398, + "nauc_recall_at_1_std": 0.111525, + "nauc_recall_at_1_diff1": 0.356592, + "nauc_recall_at_3_max": 0.272824, + "nauc_recall_at_3_std": 0.203543, + "nauc_recall_at_3_diff1": 0.187579, + "nauc_recall_at_5_max": 0.330694, + "nauc_recall_at_5_std": 0.293334, + "nauc_recall_at_5_diff1": 0.206119, + "nauc_recall_at_10_max": 0.328405, + "nauc_recall_at_10_std": 0.400526, + "nauc_recall_at_10_diff1": 0.194737, + "nauc_recall_at_20_max": 0.337017, + "nauc_recall_at_20_std": 0.52691, + "nauc_recall_at_20_diff1": 0.168158, + "nauc_recall_at_100_max": 0.30484, + "nauc_recall_at_100_std": 0.541553, + "nauc_recall_at_100_diff1": 0.149425, + "nauc_recall_at_1000_max": 0.30484, + "nauc_recall_at_1000_std": 0.541553, + "nauc_recall_at_1000_diff1": 0.149425, + "nauc_precision_at_1_max": 0.306477, + "nauc_precision_at_1_std": 0.24545, + "nauc_precision_at_1_diff1": 0.346299, + "nauc_precision_at_3_max": 0.264388, + "nauc_precision_at_3_std": 0.347797, + "nauc_precision_at_3_diff1": 0.103474, + "nauc_precision_at_5_max": 0.239301, + "nauc_precision_at_5_std": 0.35087, + "nauc_precision_at_5_diff1": 0.08327, + "nauc_precision_at_10_max": 0.142801, + "nauc_precision_at_10_std": 0.345634, + "nauc_precision_at_10_diff1": 0.021249, + "nauc_precision_at_20_max": 0.051595, + "nauc_precision_at_20_std": 0.289182, + "nauc_precision_at_20_diff1": -0.032355, + "nauc_precision_at_100_max": -0.040946, + "nauc_precision_at_100_std": 0.153704, + "nauc_precision_at_100_diff1": -0.064863, + "nauc_precision_at_1000_max": -0.040946, + "nauc_precision_at_1000_std": 0.153704, + "nauc_precision_at_1000_diff1": -0.064863, + "nauc_mrr_at_1_max": 0.306477, + "nauc_mrr_at_1_std": 0.24545, + "nauc_mrr_at_1_diff1": 0.346299, + "nauc_mrr_at_3_max": 0.376571, + "nauc_mrr_at_3_std": 0.334601, + "nauc_mrr_at_3_diff1": 0.327184, + "nauc_mrr_at_5_max": 0.370368, + "nauc_mrr_at_5_std": 0.3467, + "nauc_mrr_at_5_diff1": 0.33071, + "nauc_mrr_at_10_max": 0.366965, + "nauc_mrr_at_10_std": 0.345678, + "nauc_mrr_at_10_diff1": 0.332364, + "nauc_mrr_at_20_max": 0.367095, + "nauc_mrr_at_20_std": 0.345962, + "nauc_mrr_at_20_diff1": 0.331627, + "nauc_mrr_at_100_max": 0.367428, + "nauc_mrr_at_100_std": 0.343975, + "nauc_mrr_at_100_diff1": 0.331344, + "nauc_mrr_at_1000_max": 0.367428, + "nauc_mrr_at_1000_std": 0.343975, + "nauc_mrr_at_1000_diff1": 0.331344, + "hit_rate_at_1": 0.46687, + "hit_rate_at_3": 0.70181, + "hit_rate_at_5": 0.75602, + "hit_rate_at_10": 0.8253, + "hit_rate_at_20": 0.87952, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.50258, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 7255.896583425114, + "kg_co2_emissions": null, + "date": 1792441331.8321, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 2.3416010662913322e-05, + "end": 31.746840140083805, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 31.746881042025052, + "end": 31.74688488803804, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries and documents", + "start": 31.751660208101384, + "end": 1238.0487513580592, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 1238.0488066819962, + "end": 1238.0983484190656, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries and documents", + "start": 1238.10928400408, + "end": 2419.543030148023, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 2419.5431108700577, + "end": 2419.5907383901067, + "split": "test", + "subset": "spanish" + } + ] +} diff --git a/results/Qwen__Qwen3-VL-Reranker-2B/4bd860ac4f15ad1897a214615cccc700f8f71818/Vidore3TelecomNemotron35Embed8BReranking.v2.json b/results/Qwen__Qwen3-VL-Reranker-2B/4bd860ac4f15ad1897a214615cccc700f8f71818/Vidore3TelecomNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..b90e4cde60 --- /dev/null +++ b/results/Qwen__Qwen3-VL-Reranker-2B/4bd860ac4f15ad1897a214615cccc700f8f71818/Vidore3TelecomNemotron35Embed8BReranking.v2.json @@ -0,0 +1,992 @@ +{ + "dataset_revision": "47c9b703527792f3fbf0a082f4589ff40c1aa8cc", + "task_name": "Vidore3TelecomNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.75, + "ndcg_at_3": 0.69109, + "ndcg_at_5": 0.69665, + "ndcg_at_10": 0.72484, + "ndcg_at_20": 0.75936, + "ndcg_at_100": 0.78635, + "ndcg_at_1000": 0.78635, + "map_at_1": 0.33748, + "map_at_3": 0.50152, + "map_at_5": 0.55534, + "map_at_10": 0.61097, + "map_at_20": 0.64169, + "map_at_100": 0.65707, + "map_at_1000": 0.65707, + "recall_at_1": 0.33748, + "recall_at_3": 0.5374, + "recall_at_5": 0.6287, + "recall_at_10": 0.75204, + "recall_at_20": 0.8561, + "recall_at_100": 0.92961, + "recall_at_1000": 0.92961, + "accuracy": 0.33748, + "precision_at_1": 0.77299, + "precision_at_3": 0.50383, + "precision_at_5": 0.39023, + "precision_at_10": 0.26322, + "precision_at_20": 0.16494, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.772989, + "mrr_at_3": 0.824234, + "mrr_at_5": 0.832998, + "mrr_at_10": 0.838362, + "mrr_at_20": 0.840162, + "mrr_at_100": 0.840435, + "mrr_at_1000": 0.840435, + "nauc_ndcg_at_1_max": 0.484574, + "nauc_ndcg_at_1_std": 0.202736, + "nauc_ndcg_at_1_diff1": 0.511898, + "nauc_ndcg_at_3_max": 0.495101, + "nauc_ndcg_at_3_std": 0.242072, + "nauc_ndcg_at_3_diff1": 0.410956, + "nauc_ndcg_at_5_max": 0.480159, + "nauc_ndcg_at_5_std": 0.266526, + "nauc_ndcg_at_5_diff1": 0.413908, + "nauc_ndcg_at_10_max": 0.50933, + "nauc_ndcg_at_10_std": 0.304268, + "nauc_ndcg_at_10_diff1": 0.425081, + "nauc_ndcg_at_20_max": 0.513889, + "nauc_ndcg_at_20_std": 0.329288, + "nauc_ndcg_at_20_diff1": 0.435225, + "nauc_ndcg_at_100_max": 0.505219, + "nauc_ndcg_at_100_std": 0.298582, + "nauc_ndcg_at_100_diff1": 0.437541, + "nauc_ndcg_at_1000_max": 0.505219, + "nauc_ndcg_at_1000_std": 0.298582, + "nauc_ndcg_at_1000_diff1": 0.437541, + "nauc_map_at_1_max": 0.345812, + "nauc_map_at_1_std": -0.012544, + "nauc_map_at_1_diff1": 0.481959, + "nauc_map_at_3_max": 0.42723, + "nauc_map_at_3_std": 0.127243, + "nauc_map_at_3_diff1": 0.386173, + "nauc_map_at_5_max": 0.426269, + "nauc_map_at_5_std": 0.179109, + "nauc_map_at_5_diff1": 0.372152, + "nauc_map_at_10_max": 0.44794, + "nauc_map_at_10_std": 0.245395, + "nauc_map_at_10_diff1": 0.365062, + "nauc_map_at_20_max": 0.457829, + "nauc_map_at_20_std": 0.265511, + "nauc_map_at_20_diff1": 0.373361, + "nauc_map_at_100_max": 0.456735, + "nauc_map_at_100_std": 0.258728, + "nauc_map_at_100_diff1": 0.372001, + "nauc_map_at_1000_max": 0.456735, + "nauc_map_at_1000_std": 0.258728, + "nauc_map_at_1000_diff1": 0.372001, + "nauc_recall_at_1_max": 0.345812, + "nauc_recall_at_1_std": -0.012544, + "nauc_recall_at_1_diff1": 0.481959, + "nauc_recall_at_3_max": 0.428224, + "nauc_recall_at_3_std": 0.145195, + "nauc_recall_at_3_diff1": 0.372503, + "nauc_recall_at_5_max": 0.417182, + "nauc_recall_at_5_std": 0.228231, + "nauc_recall_at_5_diff1": 0.350284, + "nauc_recall_at_10_max": 0.491734, + "nauc_recall_at_10_std": 0.372647, + "nauc_recall_at_10_diff1": 0.347696, + "nauc_recall_at_20_max": 0.518505, + "nauc_recall_at_20_std": 0.507755, + "nauc_recall_at_20_diff1": 0.368802, + "nauc_recall_at_100_max": 0.537169, + "nauc_recall_at_100_std": 0.498747, + "nauc_recall_at_100_diff1": 0.36653, + "nauc_recall_at_1000_max": 0.537169, + "nauc_recall_at_1000_std": 0.498747, + "nauc_recall_at_1000_diff1": 0.36653, + "nauc_precision_at_1_max": 0.539961, + "nauc_precision_at_1_std": 0.235499, + "nauc_precision_at_1_diff1": 0.502064, + "nauc_precision_at_3_max": 0.2293, + "nauc_precision_at_3_std": 0.28249, + "nauc_precision_at_3_diff1": 0.013736, + "nauc_precision_at_5_max": 0.065856, + "nauc_precision_at_5_std": 0.273216, + "nauc_precision_at_5_diff1": -0.087987, + "nauc_precision_at_10_max": -0.025863, + "nauc_precision_at_10_std": 0.235034, + "nauc_precision_at_10_diff1": -0.163612, + "nauc_precision_at_20_max": -0.109674, + "nauc_precision_at_20_std": 0.137336, + "nauc_precision_at_20_diff1": -0.187802, + "nauc_precision_at_100_max": -0.166779, + "nauc_precision_at_100_std": 0.02247, + "nauc_precision_at_100_diff1": -0.218532, + "nauc_precision_at_1000_max": -0.166779, + "nauc_precision_at_1000_std": 0.02247, + "nauc_precision_at_1000_diff1": -0.218532, + "nauc_mrr_at_1_max": 0.539961, + "nauc_mrr_at_1_std": 0.235499, + "nauc_mrr_at_1_diff1": 0.502064, + "nauc_mrr_at_3_max": 0.592451, + "nauc_mrr_at_3_std": 0.277369, + "nauc_mrr_at_3_diff1": 0.537053, + "nauc_mrr_at_5_max": 0.591992, + "nauc_mrr_at_5_std": 0.281281, + "nauc_mrr_at_5_diff1": 0.543775, + "nauc_mrr_at_10_max": 0.592968, + "nauc_mrr_at_10_std": 0.282854, + "nauc_mrr_at_10_diff1": 0.538345, + "nauc_mrr_at_20_max": 0.590665, + "nauc_mrr_at_20_std": 0.284401, + "nauc_mrr_at_20_diff1": 0.535041, + "nauc_mrr_at_100_max": 0.590321, + "nauc_mrr_at_100_std": 0.283081, + "nauc_mrr_at_100_diff1": 0.535113, + "nauc_mrr_at_1000_max": 0.590321, + "nauc_mrr_at_1000_std": 0.283081, + "nauc_mrr_at_1000_diff1": 0.535113, + "hit_rate_at_1": 0.77299, + "hit_rate_at_3": 0.88506, + "hit_rate_at_5": 0.92241, + "hit_rate_at_10": 0.96264, + "hit_rate_at_20": 0.98851, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.72484, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.70115, + "ndcg_at_3": 0.66137, + "ndcg_at_5": 0.67, + "ndcg_at_10": 0.69765, + "ndcg_at_20": 0.73539, + "ndcg_at_100": 0.76353, + "ndcg_at_1000": 0.76353, + "map_at_1": 0.31865, + "map_at_3": 0.47795, + "map_at_5": 0.53034, + "map_at_10": 0.58053, + "map_at_20": 0.61123, + "map_at_100": 0.62789, + "map_at_1000": 0.62789, + "recall_at_1": 0.31865, + "recall_at_3": 0.52035, + "recall_at_5": 0.61215, + "recall_at_10": 0.73082, + "recall_at_20": 0.84251, + "recall_at_100": 0.92187, + "recall_at_1000": 0.92187, + "accuracy": 0.31865, + "precision_at_1": 0.72701, + "precision_at_3": 0.47414, + "precision_at_5": 0.37414, + "precision_at_10": 0.25345, + "precision_at_20": 0.16063, + "precision_at_100": 0.03882, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.727011, + "mrr_at_3": 0.793103, + "mrr_at_5": 0.807184, + "mrr_at_10": 0.811134, + "mrr_at_20": 0.812948, + "mrr_at_100": 0.813138, + "mrr_at_1000": 0.813138, + "nauc_ndcg_at_1_max": 0.410418, + "nauc_ndcg_at_1_std": 0.096506, + "nauc_ndcg_at_1_diff1": 0.519024, + "nauc_ndcg_at_3_max": 0.427675, + "nauc_ndcg_at_3_std": 0.186233, + "nauc_ndcg_at_3_diff1": 0.366616, + "nauc_ndcg_at_5_max": 0.440911, + "nauc_ndcg_at_5_std": 0.253744, + "nauc_ndcg_at_5_diff1": 0.374694, + "nauc_ndcg_at_10_max": 0.471119, + "nauc_ndcg_at_10_std": 0.279872, + "nauc_ndcg_at_10_diff1": 0.441353, + "nauc_ndcg_at_20_max": 0.461131, + "nauc_ndcg_at_20_std": 0.276815, + "nauc_ndcg_at_20_diff1": 0.441576, + "nauc_ndcg_at_100_max": 0.458969, + "nauc_ndcg_at_100_std": 0.257905, + "nauc_ndcg_at_100_diff1": 0.439142, + "nauc_ndcg_at_1000_max": 0.458969, + "nauc_ndcg_at_1000_std": 0.257905, + "nauc_ndcg_at_1000_diff1": 0.439142, + "nauc_map_at_1_max": 0.28586, + "nauc_map_at_1_std": -0.042486, + "nauc_map_at_1_diff1": 0.493811, + "nauc_map_at_3_max": 0.35975, + "nauc_map_at_3_std": 0.094599, + "nauc_map_at_3_diff1": 0.374563, + "nauc_map_at_5_max": 0.387885, + "nauc_map_at_5_std": 0.168148, + "nauc_map_at_5_diff1": 0.365521, + "nauc_map_at_10_max": 0.408576, + "nauc_map_at_10_std": 0.223894, + "nauc_map_at_10_diff1": 0.38297, + "nauc_map_at_20_max": 0.409392, + "nauc_map_at_20_std": 0.234996, + "nauc_map_at_20_diff1": 0.381793, + "nauc_map_at_100_max": 0.408986, + "nauc_map_at_100_std": 0.234504, + "nauc_map_at_100_diff1": 0.37669, + "nauc_map_at_1000_max": 0.408986, + "nauc_map_at_1000_std": 0.234504, + "nauc_map_at_1000_diff1": 0.37669, + "nauc_recall_at_1_max": 0.28586, + "nauc_recall_at_1_std": -0.042486, + "nauc_recall_at_1_diff1": 0.493811, + "nauc_recall_at_3_max": 0.358382, + "nauc_recall_at_3_std": 0.141558, + "nauc_recall_at_3_diff1": 0.335406, + "nauc_recall_at_5_max": 0.389182, + "nauc_recall_at_5_std": 0.259247, + "nauc_recall_at_5_diff1": 0.327769, + "nauc_recall_at_10_max": 0.451962, + "nauc_recall_at_10_std": 0.383185, + "nauc_recall_at_10_diff1": 0.401203, + "nauc_recall_at_20_max": 0.438684, + "nauc_recall_at_20_std": 0.422932, + "nauc_recall_at_20_diff1": 0.392954, + "nauc_recall_at_100_max": 0.491855, + "nauc_recall_at_100_std": 0.471284, + "nauc_recall_at_100_diff1": 0.407785, + "nauc_recall_at_1000_max": 0.491855, + "nauc_recall_at_1000_std": 0.471284, + "nauc_recall_at_1000_diff1": 0.407785, + "nauc_precision_at_1_max": 0.448063, + "nauc_precision_at_1_std": 0.130658, + "nauc_precision_at_1_diff1": 0.503794, + "nauc_precision_at_3_max": 0.239966, + "nauc_precision_at_3_std": 0.251225, + "nauc_precision_at_3_diff1": -0.019987, + "nauc_precision_at_5_max": 0.147768, + "nauc_precision_at_5_std": 0.304446, + "nauc_precision_at_5_diff1": -0.121629, + "nauc_precision_at_10_max": 0.039134, + "nauc_precision_at_10_std": 0.256282, + "nauc_precision_at_10_diff1": -0.135127, + "nauc_precision_at_20_max": -0.069622, + "nauc_precision_at_20_std": 0.144656, + "nauc_precision_at_20_diff1": -0.198995, + "nauc_precision_at_100_max": -0.1201, + "nauc_precision_at_100_std": 0.043729, + "nauc_precision_at_100_diff1": -0.237792, + "nauc_precision_at_1000_max": -0.1201, + "nauc_precision_at_1000_std": 0.043729, + "nauc_precision_at_1000_diff1": -0.237792, + "nauc_mrr_at_1_max": 0.448063, + "nauc_mrr_at_1_std": 0.130658, + "nauc_mrr_at_1_diff1": 0.503794, + "nauc_mrr_at_3_max": 0.51058, + "nauc_mrr_at_3_std": 0.224898, + "nauc_mrr_at_3_diff1": 0.497655, + "nauc_mrr_at_5_max": 0.506882, + "nauc_mrr_at_5_std": 0.21382, + "nauc_mrr_at_5_diff1": 0.504433, + "nauc_mrr_at_10_max": 0.505213, + "nauc_mrr_at_10_std": 0.207332, + "nauc_mrr_at_10_diff1": 0.507238, + "nauc_mrr_at_20_max": 0.502501, + "nauc_mrr_at_20_std": 0.203302, + "nauc_mrr_at_20_diff1": 0.505462, + "nauc_mrr_at_100_max": 0.502026, + "nauc_mrr_at_100_std": 0.202695, + "nauc_mrr_at_100_diff1": 0.505325, + "nauc_mrr_at_1000_max": 0.502026, + "nauc_mrr_at_1000_std": 0.202695, + "nauc_mrr_at_1000_diff1": 0.505325, + "hit_rate_at_1": 0.72701, + "hit_rate_at_3": 0.86782, + "hit_rate_at_5": 0.92816, + "hit_rate_at_10": 0.95977, + "hit_rate_at_20": 0.98563, + "hit_rate_at_100": 0.99138, + "hit_rate_at_1000": 0.99138, + "main_score": 0.69765, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.71983, + "ndcg_at_3": 0.66618, + "ndcg_at_5": 0.66904, + "ndcg_at_10": 0.70268, + "ndcg_at_20": 0.73614, + "ndcg_at_100": 0.76746, + "ndcg_at_1000": 0.76746, + "map_at_1": 0.32176, + "map_at_3": 0.47742, + "map_at_5": 0.52867, + "map_at_10": 0.58606, + "map_at_20": 0.61437, + "map_at_100": 0.6314, + "map_at_1000": 0.6314, + "recall_at_1": 0.32176, + "recall_at_3": 0.51685, + "recall_at_5": 0.60847, + "recall_at_10": 0.74046, + "recall_at_20": 0.83812, + "recall_at_100": 0.93008, + "recall_at_1000": 0.93008, + "accuracy": 0.32176, + "precision_at_1": 0.73851, + "precision_at_3": 0.48372, + "precision_at_5": 0.37011, + "precision_at_10": 0.25661, + "precision_at_20": 0.16149, + "precision_at_100": 0.03905, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.738506, + "mrr_at_3": 0.801245, + "mrr_at_5": 0.810297, + "mrr_at_10": 0.815735, + "mrr_at_20": 0.817461, + "mrr_at_100": 0.81826, + "mrr_at_1000": 0.81826, + "nauc_ndcg_at_1_max": 0.455765, + "nauc_ndcg_at_1_std": 0.188497, + "nauc_ndcg_at_1_diff1": 0.508134, + "nauc_ndcg_at_3_max": 0.478864, + "nauc_ndcg_at_3_std": 0.224882, + "nauc_ndcg_at_3_diff1": 0.370782, + "nauc_ndcg_at_5_max": 0.483744, + "nauc_ndcg_at_5_std": 0.292918, + "nauc_ndcg_at_5_diff1": 0.388066, + "nauc_ndcg_at_10_max": 0.521024, + "nauc_ndcg_at_10_std": 0.341813, + "nauc_ndcg_at_10_diff1": 0.444882, + "nauc_ndcg_at_20_max": 0.511317, + "nauc_ndcg_at_20_std": 0.342427, + "nauc_ndcg_at_20_diff1": 0.443765, + "nauc_ndcg_at_100_max": 0.498403, + "nauc_ndcg_at_100_std": 0.313661, + "nauc_ndcg_at_100_diff1": 0.426532, + "nauc_ndcg_at_1000_max": 0.498403, + "nauc_ndcg_at_1000_std": 0.313661, + "nauc_ndcg_at_1000_diff1": 0.426532, + "nauc_map_at_1_max": 0.336642, + "nauc_map_at_1_std": 0.021534, + "nauc_map_at_1_diff1": 0.500465, + "nauc_map_at_3_max": 0.41642, + "nauc_map_at_3_std": 0.13252, + "nauc_map_at_3_diff1": 0.400914, + "nauc_map_at_5_max": 0.432817, + "nauc_map_at_5_std": 0.206422, + "nauc_map_at_5_diff1": 0.395191, + "nauc_map_at_10_max": 0.456376, + "nauc_map_at_10_std": 0.27207, + "nauc_map_at_10_diff1": 0.392884, + "nauc_map_at_20_max": 0.458748, + "nauc_map_at_20_std": 0.286459, + "nauc_map_at_20_diff1": 0.389133, + "nauc_map_at_100_max": 0.456519, + "nauc_map_at_100_std": 0.279938, + "nauc_map_at_100_diff1": 0.381082, + "nauc_map_at_1000_max": 0.456519, + "nauc_map_at_1000_std": 0.279938, + "nauc_map_at_1000_diff1": 0.381082, + "nauc_recall_at_1_max": 0.336642, + "nauc_recall_at_1_std": 0.021534, + "nauc_recall_at_1_diff1": 0.500465, + "nauc_recall_at_3_max": 0.421784, + "nauc_recall_at_3_std": 0.144882, + "nauc_recall_at_3_diff1": 0.374911, + "nauc_recall_at_5_max": 0.437466, + "nauc_recall_at_5_std": 0.284107, + "nauc_recall_at_5_diff1": 0.364252, + "nauc_recall_at_10_max": 0.504461, + "nauc_recall_at_10_std": 0.438024, + "nauc_recall_at_10_diff1": 0.401694, + "nauc_recall_at_20_max": 0.510564, + "nauc_recall_at_20_std": 0.521342, + "nauc_recall_at_20_diff1": 0.386295, + "nauc_recall_at_100_max": 0.503931, + "nauc_recall_at_100_std": 0.54897, + "nauc_recall_at_100_diff1": 0.273933, + "nauc_recall_at_1000_max": 0.503931, + "nauc_recall_at_1000_std": 0.54897, + "nauc_recall_at_1000_diff1": 0.273933, + "nauc_precision_at_1_max": 0.497899, + "nauc_precision_at_1_std": 0.214095, + "nauc_precision_at_1_diff1": 0.524647, + "nauc_precision_at_3_max": 0.250701, + "nauc_precision_at_3_std": 0.250386, + "nauc_precision_at_3_diff1": -0.015452, + "nauc_precision_at_5_max": 0.132737, + "nauc_precision_at_5_std": 0.294834, + "nauc_precision_at_5_diff1": -0.093646, + "nauc_precision_at_10_max": 0.036847, + "nauc_precision_at_10_std": 0.260064, + "nauc_precision_at_10_diff1": -0.1481, + "nauc_precision_at_20_max": -0.086864, + "nauc_precision_at_20_std": 0.130928, + "nauc_precision_at_20_diff1": -0.206808, + "nauc_precision_at_100_max": -0.157806, + "nauc_precision_at_100_std": 0.012341, + "nauc_precision_at_100_diff1": -0.257926, + "nauc_precision_at_1000_max": -0.157806, + "nauc_precision_at_1000_std": 0.012341, + "nauc_precision_at_1000_diff1": -0.257926, + "nauc_mrr_at_1_max": 0.497899, + "nauc_mrr_at_1_std": 0.214095, + "nauc_mrr_at_1_diff1": 0.524647, + "nauc_mrr_at_3_max": 0.555269, + "nauc_mrr_at_3_std": 0.271826, + "nauc_mrr_at_3_diff1": 0.536332, + "nauc_mrr_at_5_max": 0.554478, + "nauc_mrr_at_5_std": 0.287806, + "nauc_mrr_at_5_diff1": 0.533275, + "nauc_mrr_at_10_max": 0.556151, + "nauc_mrr_at_10_std": 0.287987, + "nauc_mrr_at_10_diff1": 0.541827, + "nauc_mrr_at_20_max": 0.55361, + "nauc_mrr_at_20_std": 0.283997, + "nauc_mrr_at_20_diff1": 0.538565, + "nauc_mrr_at_100_max": 0.551756, + "nauc_mrr_at_100_std": 0.281036, + "nauc_mrr_at_100_diff1": 0.537951, + "nauc_mrr_at_1000_max": 0.551756, + "nauc_mrr_at_1000_std": 0.281036, + "nauc_mrr_at_1000_diff1": 0.537951, + "hit_rate_at_1": 0.73851, + "hit_rate_at_3": 0.87356, + "hit_rate_at_5": 0.91092, + "hit_rate_at_10": 0.95115, + "hit_rate_at_20": 0.97414, + "hit_rate_at_100": 0.99713, + "hit_rate_at_1000": 0.99713, + "main_score": 0.70268, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.70115, + "ndcg_at_3": 0.66046, + "ndcg_at_5": 0.66477, + "ndcg_at_10": 0.69657, + "ndcg_at_20": 0.73334, + "ndcg_at_100": 0.76344, + "ndcg_at_1000": 0.76344, + "map_at_1": 0.31064, + "map_at_3": 0.47208, + "map_at_5": 0.52174, + "map_at_10": 0.57549, + "map_at_20": 0.60631, + "map_at_100": 0.62277, + "map_at_1000": 0.62277, + "recall_at_1": 0.31064, + "recall_at_3": 0.52208, + "recall_at_5": 0.61016, + "recall_at_10": 0.73597, + "recall_at_20": 0.8448, + "recall_at_100": 0.93189, + "recall_at_1000": 0.93189, + "accuracy": 0.31064, + "precision_at_1": 0.72414, + "precision_at_3": 0.47893, + "precision_at_5": 0.37069, + "precision_at_10": 0.25575, + "precision_at_20": 0.16164, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.724138, + "mrr_at_3": 0.791667, + "mrr_at_5": 0.803161, + "mrr_at_10": 0.80804, + "mrr_at_20": 0.80952, + "mrr_at_100": 0.810048, + "mrr_at_1000": 0.810048, + "nauc_ndcg_at_1_max": 0.544556, + "nauc_ndcg_at_1_std": 0.231321, + "nauc_ndcg_at_1_diff1": 0.493033, + "nauc_ndcg_at_3_max": 0.524734, + "nauc_ndcg_at_3_std": 0.302393, + "nauc_ndcg_at_3_diff1": 0.38045, + "nauc_ndcg_at_5_max": 0.534593, + "nauc_ndcg_at_5_std": 0.331547, + "nauc_ndcg_at_5_diff1": 0.372878, + "nauc_ndcg_at_10_max": 0.548565, + "nauc_ndcg_at_10_std": 0.343929, + "nauc_ndcg_at_10_diff1": 0.388894, + "nauc_ndcg_at_20_max": 0.546462, + "nauc_ndcg_at_20_std": 0.357112, + "nauc_ndcg_at_20_diff1": 0.405457, + "nauc_ndcg_at_100_max": 0.541496, + "nauc_ndcg_at_100_std": 0.334731, + "nauc_ndcg_at_100_diff1": 0.41781, + "nauc_ndcg_at_1000_max": 0.541496, + "nauc_ndcg_at_1000_std": 0.334731, + "nauc_ndcg_at_1000_diff1": 0.41781, + "nauc_map_at_1_max": 0.35193, + "nauc_map_at_1_std": 0.028753, + "nauc_map_at_1_diff1": 0.463549, + "nauc_map_at_3_max": 0.435232, + "nauc_map_at_3_std": 0.175262, + "nauc_map_at_3_diff1": 0.372242, + "nauc_map_at_5_max": 0.464308, + "nauc_map_at_5_std": 0.238479, + "nauc_map_at_5_diff1": 0.353855, + "nauc_map_at_10_max": 0.484478, + "nauc_map_at_10_std": 0.285963, + "nauc_map_at_10_diff1": 0.350737, + "nauc_map_at_20_max": 0.487103, + "nauc_map_at_20_std": 0.308471, + "nauc_map_at_20_diff1": 0.359747, + "nauc_map_at_100_max": 0.485869, + "nauc_map_at_100_std": 0.304264, + "nauc_map_at_100_diff1": 0.364433, + "nauc_map_at_1000_max": 0.485869, + "nauc_map_at_1000_std": 0.304264, + "nauc_map_at_1000_diff1": 0.364433, + "nauc_recall_at_1_max": 0.35193, + "nauc_recall_at_1_std": 0.028753, + "nauc_recall_at_1_diff1": 0.463549, + "nauc_recall_at_3_max": 0.420969, + "nauc_recall_at_3_std": 0.20925, + "nauc_recall_at_3_diff1": 0.31696, + "nauc_recall_at_5_max": 0.461666, + "nauc_recall_at_5_std": 0.299186, + "nauc_recall_at_5_diff1": 0.288986, + "nauc_recall_at_10_max": 0.487528, + "nauc_recall_at_10_std": 0.382772, + "nauc_recall_at_10_diff1": 0.263598, + "nauc_recall_at_20_max": 0.512892, + "nauc_recall_at_20_std": 0.493859, + "nauc_recall_at_20_diff1": 0.305394, + "nauc_recall_at_100_max": 0.555008, + "nauc_recall_at_100_std": 0.542099, + "nauc_recall_at_100_diff1": 0.318644, + "nauc_recall_at_1000_max": 0.555008, + "nauc_recall_at_1000_std": 0.542099, + "nauc_recall_at_1000_diff1": 0.318644, + "nauc_precision_at_1_max": 0.565818, + "nauc_precision_at_1_std": 0.26222, + "nauc_precision_at_1_diff1": 0.492525, + "nauc_precision_at_3_max": 0.298848, + "nauc_precision_at_3_std": 0.350477, + "nauc_precision_at_3_diff1": 0.047816, + "nauc_precision_at_5_max": 0.191692, + "nauc_precision_at_5_std": 0.355488, + "nauc_precision_at_5_diff1": -0.055646, + "nauc_precision_at_10_max": 0.023398, + "nauc_precision_at_10_std": 0.237892, + "nauc_precision_at_10_diff1": -0.121738, + "nauc_precision_at_20_max": -0.094983, + "nauc_precision_at_20_std": 0.13875, + "nauc_precision_at_20_diff1": -0.133009, + "nauc_precision_at_100_max": -0.167032, + "nauc_precision_at_100_std": 0.014437, + "nauc_precision_at_100_diff1": -0.15157, + "nauc_precision_at_1000_max": -0.167032, + "nauc_precision_at_1000_std": 0.014437, + "nauc_precision_at_1000_diff1": -0.15157, + "nauc_mrr_at_1_max": 0.565818, + "nauc_mrr_at_1_std": 0.26222, + "nauc_mrr_at_1_diff1": 0.492525, + "nauc_mrr_at_3_max": 0.598331, + "nauc_mrr_at_3_std": 0.332273, + "nauc_mrr_at_3_diff1": 0.476155, + "nauc_mrr_at_5_max": 0.601573, + "nauc_mrr_at_5_std": 0.328866, + "nauc_mrr_at_5_diff1": 0.490604, + "nauc_mrr_at_10_max": 0.600566, + "nauc_mrr_at_10_std": 0.320613, + "nauc_mrr_at_10_diff1": 0.490295, + "nauc_mrr_at_20_max": 0.597633, + "nauc_mrr_at_20_std": 0.318229, + "nauc_mrr_at_20_diff1": 0.48955, + "nauc_mrr_at_100_max": 0.597642, + "nauc_mrr_at_100_std": 0.318402, + "nauc_mrr_at_100_diff1": 0.489462, + "nauc_mrr_at_1000_max": 0.597642, + "nauc_mrr_at_1000_std": 0.318402, + "nauc_mrr_at_1000_diff1": 0.489462, + "hit_rate_at_1": 0.72414, + "hit_rate_at_3": 0.87356, + "hit_rate_at_5": 0.92529, + "hit_rate_at_10": 0.95977, + "hit_rate_at_20": 0.97989, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.69657, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.68966, + "ndcg_at_3": 0.65691, + "ndcg_at_5": 0.66336, + "ndcg_at_10": 0.69278, + "ndcg_at_20": 0.73115, + "ndcg_at_100": 0.7607, + "ndcg_at_1000": 0.7607, + "map_at_1": 0.30421, + "map_at_3": 0.4663, + "map_at_5": 0.51968, + "map_at_10": 0.57087, + "map_at_20": 0.60182, + "map_at_100": 0.61837, + "map_at_1000": 0.61837, + "recall_at_1": 0.30421, + "recall_at_3": 0.51878, + "recall_at_5": 0.61019, + "recall_at_10": 0.73247, + "recall_at_20": 0.84678, + "recall_at_100": 0.93007, + "recall_at_1000": 0.93007, + "accuracy": 0.30421, + "precision_at_1": 0.70977, + "precision_at_3": 0.47989, + "precision_at_5": 0.37644, + "precision_at_10": 0.25546, + "precision_at_20": 0.16207, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.70977, + "mrr_at_3": 0.79023, + "mrr_at_5": 0.797989, + "mrr_at_10": 0.802957, + "mrr_at_20": 0.805356, + "mrr_at_100": 0.80556, + "mrr_at_1000": 0.80556, + "nauc_ndcg_at_1_max": 0.425559, + "nauc_ndcg_at_1_std": 0.168308, + "nauc_ndcg_at_1_diff1": 0.494861, + "nauc_ndcg_at_3_max": 0.480744, + "nauc_ndcg_at_3_std": 0.228087, + "nauc_ndcg_at_3_diff1": 0.386741, + "nauc_ndcg_at_5_max": 0.494619, + "nauc_ndcg_at_5_std": 0.279011, + "nauc_ndcg_at_5_diff1": 0.354483, + "nauc_ndcg_at_10_max": 0.506562, + "nauc_ndcg_at_10_std": 0.300731, + "nauc_ndcg_at_10_diff1": 0.401678, + "nauc_ndcg_at_20_max": 0.502939, + "nauc_ndcg_at_20_std": 0.308563, + "nauc_ndcg_at_20_diff1": 0.406529, + "nauc_ndcg_at_100_max": 0.494931, + "nauc_ndcg_at_100_std": 0.283445, + "nauc_ndcg_at_100_diff1": 0.415359, + "nauc_ndcg_at_1000_max": 0.494931, + "nauc_ndcg_at_1000_std": 0.283445, + "nauc_ndcg_at_1000_diff1": 0.415359, + "nauc_map_at_1_max": 0.330845, + "nauc_map_at_1_std": -0.027274, + "nauc_map_at_1_diff1": 0.519023, + "nauc_map_at_3_max": 0.401665, + "nauc_map_at_3_std": 0.110715, + "nauc_map_at_3_diff1": 0.373094, + "nauc_map_at_5_max": 0.426416, + "nauc_map_at_5_std": 0.170383, + "nauc_map_at_5_diff1": 0.346331, + "nauc_map_at_10_max": 0.450874, + "nauc_map_at_10_std": 0.229214, + "nauc_map_at_10_diff1": 0.355764, + "nauc_map_at_20_max": 0.453645, + "nauc_map_at_20_std": 0.2477, + "nauc_map_at_20_diff1": 0.357582, + "nauc_map_at_100_max": 0.452915, + "nauc_map_at_100_std": 0.243294, + "nauc_map_at_100_diff1": 0.359888, + "nauc_map_at_1000_max": 0.452915, + "nauc_map_at_1000_std": 0.243294, + "nauc_map_at_1000_diff1": 0.359888, + "nauc_recall_at_1_max": 0.330845, + "nauc_recall_at_1_std": -0.027274, + "nauc_recall_at_1_diff1": 0.519023, + "nauc_recall_at_3_max": 0.37669, + "nauc_recall_at_3_std": 0.132678, + "nauc_recall_at_3_diff1": 0.323699, + "nauc_recall_at_5_max": 0.423076, + "nauc_recall_at_5_std": 0.244909, + "nauc_recall_at_5_diff1": 0.274258, + "nauc_recall_at_10_max": 0.471558, + "nauc_recall_at_10_std": 0.363376, + "nauc_recall_at_10_diff1": 0.316271, + "nauc_recall_at_20_max": 0.483295, + "nauc_recall_at_20_std": 0.470291, + "nauc_recall_at_20_diff1": 0.310438, + "nauc_recall_at_100_max": 0.50231, + "nauc_recall_at_100_std": 0.49555, + "nauc_recall_at_100_diff1": 0.336767, + "nauc_recall_at_1000_max": 0.50231, + "nauc_recall_at_1000_std": 0.49555, + "nauc_recall_at_1000_diff1": 0.336767, + "nauc_precision_at_1_max": 0.439675, + "nauc_precision_at_1_std": 0.183751, + "nauc_precision_at_1_diff1": 0.492544, + "nauc_precision_at_3_max": 0.269659, + "nauc_precision_at_3_std": 0.293355, + "nauc_precision_at_3_diff1": -0.005011, + "nauc_precision_at_5_max": 0.170063, + "nauc_precision_at_5_std": 0.313045, + "nauc_precision_at_5_diff1": -0.12035, + "nauc_precision_at_10_max": 0.060055, + "nauc_precision_at_10_std": 0.261077, + "nauc_precision_at_10_diff1": -0.122621, + "nauc_precision_at_20_max": -0.051928, + "nauc_precision_at_20_std": 0.167842, + "nauc_precision_at_20_diff1": -0.155783, + "nauc_precision_at_100_max": -0.120591, + "nauc_precision_at_100_std": 0.05964, + "nauc_precision_at_100_diff1": -0.181141, + "nauc_precision_at_1000_max": -0.120591, + "nauc_precision_at_1000_std": 0.05964, + "nauc_precision_at_1000_diff1": -0.181141, + "nauc_mrr_at_1_max": 0.439675, + "nauc_mrr_at_1_std": 0.183751, + "nauc_mrr_at_1_diff1": 0.492544, + "nauc_mrr_at_3_max": 0.501197, + "nauc_mrr_at_3_std": 0.258568, + "nauc_mrr_at_3_diff1": 0.486755, + "nauc_mrr_at_5_max": 0.513323, + "nauc_mrr_at_5_std": 0.276625, + "nauc_mrr_at_5_diff1": 0.484472, + "nauc_mrr_at_10_max": 0.503665, + "nauc_mrr_at_10_std": 0.262652, + "nauc_mrr_at_10_diff1": 0.488159, + "nauc_mrr_at_20_max": 0.500192, + "nauc_mrr_at_20_std": 0.258739, + "nauc_mrr_at_20_diff1": 0.486418, + "nauc_mrr_at_100_max": 0.499741, + "nauc_mrr_at_100_std": 0.25818, + "nauc_mrr_at_100_diff1": 0.486141, + "nauc_mrr_at_1000_max": 0.499741, + "nauc_mrr_at_1000_std": 0.25818, + "nauc_mrr_at_1000_diff1": 0.486141, + "hit_rate_at_1": 0.70977, + "hit_rate_at_3": 0.88506, + "hit_rate_at_5": 0.91954, + "hit_rate_at_10": 0.9569, + "hit_rate_at_20": 0.98851, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.69278, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.72701, + "ndcg_at_3": 0.66981, + "ndcg_at_5": 0.67598, + "ndcg_at_10": 0.70167, + "ndcg_at_20": 0.74093, + "ndcg_at_100": 0.77119, + "ndcg_at_1000": 0.77119, + "map_at_1": 0.32177, + "map_at_3": 0.47995, + "map_at_5": 0.53184, + "map_at_10": 0.58306, + "map_at_20": 0.61478, + "map_at_100": 0.63218, + "map_at_1000": 0.63218, + "recall_at_1": 0.32177, + "recall_at_3": 0.52171, + "recall_at_5": 0.61391, + "recall_at_10": 0.73164, + "recall_at_20": 0.84678, + "recall_at_100": 0.93268, + "recall_at_1000": 0.93268, + "accuracy": 0.32177, + "precision_at_1": 0.74713, + "precision_at_3": 0.48372, + "precision_at_5": 0.37586, + "precision_at_10": 0.25374, + "precision_at_20": 0.1625, + "precision_at_100": 0.03937, + "precision_at_1000": 0.00394, + "mrr_at_1": 0.747126, + "mrr_at_3": 0.809387, + "mrr_at_5": 0.819588, + "mrr_at_10": 0.823626, + "mrr_at_20": 0.825506, + "mrr_at_100": 0.825799, + "mrr_at_1000": 0.825799, + "nauc_ndcg_at_1_max": 0.372869, + "nauc_ndcg_at_1_std": 0.073199, + "nauc_ndcg_at_1_diff1": 0.463391, + "nauc_ndcg_at_3_max": 0.424932, + "nauc_ndcg_at_3_std": 0.185447, + "nauc_ndcg_at_3_diff1": 0.351518, + "nauc_ndcg_at_5_max": 0.459079, + "nauc_ndcg_at_5_std": 0.225095, + "nauc_ndcg_at_5_diff1": 0.363705, + "nauc_ndcg_at_10_max": 0.457142, + "nauc_ndcg_at_10_std": 0.237296, + "nauc_ndcg_at_10_diff1": 0.381989, + "nauc_ndcg_at_20_max": 0.452072, + "nauc_ndcg_at_20_std": 0.244418, + "nauc_ndcg_at_20_diff1": 0.392469, + "nauc_ndcg_at_100_max": 0.447148, + "nauc_ndcg_at_100_std": 0.22738, + "nauc_ndcg_at_100_diff1": 0.404059, + "nauc_ndcg_at_1000_max": 0.447148, + "nauc_ndcg_at_1000_std": 0.22738, + "nauc_ndcg_at_1000_diff1": 0.404059, + "nauc_map_at_1_max": 0.28381, + "nauc_map_at_1_std": -0.07122, + "nauc_map_at_1_diff1": 0.42466, + "nauc_map_at_3_max": 0.368336, + "nauc_map_at_3_std": 0.067822, + "nauc_map_at_3_diff1": 0.339701, + "nauc_map_at_5_max": 0.39481, + "nauc_map_at_5_std": 0.119218, + "nauc_map_at_5_diff1": 0.340197, + "nauc_map_at_10_max": 0.40443, + "nauc_map_at_10_std": 0.169079, + "nauc_map_at_10_diff1": 0.335454, + "nauc_map_at_20_max": 0.408033, + "nauc_map_at_20_std": 0.186104, + "nauc_map_at_20_diff1": 0.342686, + "nauc_map_at_100_max": 0.407921, + "nauc_map_at_100_std": 0.185455, + "nauc_map_at_100_diff1": 0.34383, + "nauc_map_at_1000_max": 0.407921, + "nauc_map_at_1000_std": 0.185455, + "nauc_map_at_1000_diff1": 0.34383, + "nauc_recall_at_1_max": 0.28381, + "nauc_recall_at_1_std": -0.07122, + "nauc_recall_at_1_diff1": 0.42466, + "nauc_recall_at_3_max": 0.367686, + "nauc_recall_at_3_std": 0.106421, + "nauc_recall_at_3_diff1": 0.303366, + "nauc_recall_at_5_max": 0.416573, + "nauc_recall_at_5_std": 0.204084, + "nauc_recall_at_5_diff1": 0.309217, + "nauc_recall_at_10_max": 0.42979, + "nauc_recall_at_10_std": 0.300264, + "nauc_recall_at_10_diff1": 0.315592, + "nauc_recall_at_20_max": 0.432594, + "nauc_recall_at_20_std": 0.387577, + "nauc_recall_at_20_diff1": 0.311231, + "nauc_recall_at_100_max": 0.468547, + "nauc_recall_at_100_std": 0.461687, + "nauc_recall_at_100_diff1": 0.359964, + "nauc_recall_at_1000_max": 0.468547, + "nauc_recall_at_1000_std": 0.461687, + "nauc_recall_at_1000_diff1": 0.359964, + "nauc_precision_at_1_max": 0.417963, + "nauc_precision_at_1_std": 0.100825, + "nauc_precision_at_1_diff1": 0.476533, + "nauc_precision_at_3_max": 0.221866, + "nauc_precision_at_3_std": 0.260889, + "nauc_precision_at_3_diff1": 0.021267, + "nauc_precision_at_5_max": 0.147849, + "nauc_precision_at_5_std": 0.288887, + "nauc_precision_at_5_diff1": -0.038938, + "nauc_precision_at_10_max": 0.02198, + "nauc_precision_at_10_std": 0.239015, + "nauc_precision_at_10_diff1": -0.100225, + "nauc_precision_at_20_max": -0.0702, + "nauc_precision_at_20_std": 0.154083, + "nauc_precision_at_20_diff1": -0.138943, + "nauc_precision_at_100_max": -0.120166, + "nauc_precision_at_100_std": 0.064863, + "nauc_precision_at_100_diff1": -0.159577, + "nauc_precision_at_1000_max": -0.120166, + "nauc_precision_at_1000_std": 0.064863, + "nauc_precision_at_1000_diff1": -0.159577, + "nauc_mrr_at_1_max": 0.417963, + "nauc_mrr_at_1_std": 0.100825, + "nauc_mrr_at_1_diff1": 0.476533, + "nauc_mrr_at_3_max": 0.477535, + "nauc_mrr_at_3_std": 0.182147, + "nauc_mrr_at_3_diff1": 0.484054, + "nauc_mrr_at_5_max": 0.494331, + "nauc_mrr_at_5_std": 0.200263, + "nauc_mrr_at_5_diff1": 0.49608, + "nauc_mrr_at_10_max": 0.488439, + "nauc_mrr_at_10_std": 0.189005, + "nauc_mrr_at_10_diff1": 0.494425, + "nauc_mrr_at_20_max": 0.485488, + "nauc_mrr_at_20_std": 0.188049, + "nauc_mrr_at_20_diff1": 0.493001, + "nauc_mrr_at_100_max": 0.484786, + "nauc_mrr_at_100_std": 0.187037, + "nauc_mrr_at_100_diff1": 0.492313, + "nauc_mrr_at_1000_max": 0.484786, + "nauc_mrr_at_1000_std": 0.187037, + "nauc_mrr_at_1000_diff1": 0.492313, + "hit_rate_at_1": 0.74713, + "hit_rate_at_3": 0.88506, + "hit_rate_at_5": 0.92816, + "hit_rate_at_10": 0.95977, + "hit_rate_at_20": 0.98563, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.70167, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 7044.036697290954, + "kg_co2_emissions": null, + "date": 1792441235.025611, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 1.6245059669017792e-05, + "end": 31.799947780091316, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 31.799986940110102, + "end": 31.799994431086816, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries and documents", + "start": 31.804811884998344, + "end": 1193.1125979321077, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 1193.1126599160489, + "end": 1193.1648225460667, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries and documents", + "start": 1193.1781422760105, + "end": 2322.692625354044, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 2322.692741981009, + "end": 2322.7376590030035, + "split": "test", + "subset": "spanish" + } + ] +} diff --git a/results/Qwen__Qwen3-VL-Reranker-2B/4bd860ac4f15ad1897a214615cccc700f8f71818/run_settings.jsonl b/results/Qwen__Qwen3-VL-Reranker-2B/4bd860ac4f15ad1897a214615cccc700f8f71818/run_settings.jsonl index 0218287258..40a92d06c5 100644 --- a/results/Qwen__Qwen3-VL-Reranker-2B/4bd860ac4f15ad1897a214615cccc700f8f71818/run_settings.jsonl +++ b/results/Qwen__Qwen3-VL-Reranker-2B/4bd860ac4f15ad1897a214615cccc700f8f71818/run_settings.jsonl @@ -38,3 +38,9 @@ {"task": "Vidore3FinanceFrNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.22.5", "torch": "2.14.0a0+b2c75dd062.nv26.9.68203377", "sentence-transformers": "6.1.0", "flash-attn": "2.8.3.post1+git91ffc2ac7c.63064690", "transformers": "5.17.0"}, "encode_kwargs": {"batch_size": 4}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.0"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.0"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["french", "spanish"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["english", "german"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["italian", "portuguese"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["french", "spanish"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["english", "german"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["italian", "portuguese"]} diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/Vidore3NuclearNemotron35Embed8BReranking.v2.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/Vidore3NuclearNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..f3028eced2 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/Vidore3NuclearNemotron35Embed8BReranking.v2.json @@ -0,0 +1,1132 @@ +{ + "dataset_revision": "97ff9e10e265e64abece589ff40c552215754f26", + "task_name": "Vidore3NuclearNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.2741, + "ndcg_at_3": 0.30083, + "ndcg_at_5": 0.3277, + "ndcg_at_10": 0.3628, + "ndcg_at_20": 0.41209, + "ndcg_at_100": 0.47867, + "ndcg_at_1000": 0.47867, + "map_at_1": 0.11204, + "map_at_3": 0.18409, + "map_at_5": 0.21788, + "map_at_10": 0.25083, + "map_at_20": 0.27771, + "map_at_100": 0.30133, + "map_at_1000": 0.30133, + "recall_at_1": 0.11204, + "recall_at_3": 0.23918, + "recall_at_5": 0.33461, + "recall_at_10": 0.45554, + "recall_at_20": 0.59715, + "recall_at_100": 0.81132, + "recall_at_1000": 0.81132, + "accuracy": 0.11204, + "precision_at_1": 0.29217, + "precision_at_3": 0.24197, + "precision_at_5": 0.21145, + "precision_at_10": 0.15572, + "precision_at_20": 0.11175, + "precision_at_100": 0.0338, + "precision_at_1000": 0.00338, + "mrr_at_1": 0.292169, + "mrr_at_3": 0.402108, + "mrr_at_5": 0.430572, + "mrr_at_10": 0.444202, + "mrr_at_20": 0.449669, + "mrr_at_100": 0.453168, + "mrr_at_1000": 0.453168, + "nauc_ndcg_at_1_max": 0.079195, + "nauc_ndcg_at_1_std": 0.085946, + "nauc_ndcg_at_1_diff1": 0.212084, + "nauc_ndcg_at_3_max": 0.089372, + "nauc_ndcg_at_3_std": 0.078382, + "nauc_ndcg_at_3_diff1": 0.160301, + "nauc_ndcg_at_5_max": 0.108497, + "nauc_ndcg_at_5_std": 0.125117, + "nauc_ndcg_at_5_diff1": 0.177293, + "nauc_ndcg_at_10_max": 0.122632, + "nauc_ndcg_at_10_std": 0.135693, + "nauc_ndcg_at_10_diff1": 0.172986, + "nauc_ndcg_at_20_max": 0.146886, + "nauc_ndcg_at_20_std": 0.17261, + "nauc_ndcg_at_20_diff1": 0.214195, + "nauc_ndcg_at_100_max": 0.11884, + "nauc_ndcg_at_100_std": 0.16871, + "nauc_ndcg_at_100_diff1": 0.205219, + "nauc_ndcg_at_1000_max": 0.11884, + "nauc_ndcg_at_1000_std": 0.16871, + "nauc_ndcg_at_1000_diff1": 0.205219, + "nauc_map_at_1_max": 0.054773, + "nauc_map_at_1_std": -0.049927, + "nauc_map_at_1_diff1": 0.236822, + "nauc_map_at_3_max": 0.077021, + "nauc_map_at_3_std": 0.026567, + "nauc_map_at_3_diff1": 0.209295, + "nauc_map_at_5_max": 0.094304, + "nauc_map_at_5_std": 0.083681, + "nauc_map_at_5_diff1": 0.205176, + "nauc_map_at_10_max": 0.110567, + "nauc_map_at_10_std": 0.10996, + "nauc_map_at_10_diff1": 0.200349, + "nauc_map_at_20_max": 0.125564, + "nauc_map_at_20_std": 0.133793, + "nauc_map_at_20_diff1": 0.224696, + "nauc_map_at_100_max": 0.122332, + "nauc_map_at_100_std": 0.136052, + "nauc_map_at_100_diff1": 0.223594, + "nauc_map_at_1000_max": 0.122332, + "nauc_map_at_1000_std": 0.136052, + "nauc_map_at_1000_diff1": 0.223594, + "nauc_recall_at_1_max": 0.054773, + "nauc_recall_at_1_std": -0.049927, + "nauc_recall_at_1_diff1": 0.236822, + "nauc_recall_at_3_max": 0.049121, + "nauc_recall_at_3_std": 0.029376, + "nauc_recall_at_3_diff1": 0.153301, + "nauc_recall_at_5_max": 0.066104, + "nauc_recall_at_5_std": 0.120988, + "nauc_recall_at_5_diff1": 0.140327, + "nauc_recall_at_10_max": 0.094445, + "nauc_recall_at_10_std": 0.140027, + "nauc_recall_at_10_diff1": 0.106762, + "nauc_recall_at_20_max": 0.14215, + "nauc_recall_at_20_std": 0.242549, + "nauc_recall_at_20_diff1": 0.198769, + "nauc_recall_at_100_max": 0.026954, + "nauc_recall_at_100_std": 0.309669, + "nauc_recall_at_100_diff1": 0.203423, + "nauc_recall_at_1000_max": 0.026954, + "nauc_recall_at_1000_std": 0.309669, + "nauc_recall_at_1000_diff1": 0.203423, + "nauc_precision_at_1_max": 0.094312, + "nauc_precision_at_1_std": 0.117369, + "nauc_precision_at_1_diff1": 0.226546, + "nauc_precision_at_3_max": 0.132096, + "nauc_precision_at_3_std": 0.195073, + "nauc_precision_at_3_diff1": 0.11396, + "nauc_precision_at_5_max": 0.161629, + "nauc_precision_at_5_std": 0.256347, + "nauc_precision_at_5_diff1": 0.095978, + "nauc_precision_at_10_max": 0.170367, + "nauc_precision_at_10_std": 0.232521, + "nauc_precision_at_10_diff1": 0.054746, + "nauc_precision_at_20_max": 0.168283, + "nauc_precision_at_20_std": 0.237483, + "nauc_precision_at_20_diff1": 0.100804, + "nauc_precision_at_100_max": 0.086074, + "nauc_precision_at_100_std": 0.150481, + "nauc_precision_at_100_diff1": 0.029528, + "nauc_precision_at_1000_max": 0.086074, + "nauc_precision_at_1000_std": 0.150481, + "nauc_precision_at_1000_diff1": 0.029528, + "nauc_mrr_at_1_max": 0.094312, + "nauc_mrr_at_1_std": 0.117369, + "nauc_mrr_at_1_diff1": 0.226546, + "nauc_mrr_at_3_max": 0.116501, + "nauc_mrr_at_3_std": 0.143388, + "nauc_mrr_at_3_diff1": 0.179158, + "nauc_mrr_at_5_max": 0.120384, + "nauc_mrr_at_5_std": 0.153943, + "nauc_mrr_at_5_diff1": 0.183538, + "nauc_mrr_at_10_max": 0.121748, + "nauc_mrr_at_10_std": 0.147502, + "nauc_mrr_at_10_diff1": 0.179734, + "nauc_mrr_at_20_max": 0.120775, + "nauc_mrr_at_20_std": 0.150788, + "nauc_mrr_at_20_diff1": 0.180482, + "nauc_mrr_at_100_max": 0.117568, + "nauc_mrr_at_100_std": 0.149898, + "nauc_mrr_at_100_diff1": 0.180776, + "nauc_mrr_at_1000_max": 0.117568, + "nauc_mrr_at_1000_std": 0.149898, + "nauc_mrr_at_1000_diff1": 0.180776, + "hit_rate_at_1": 0.29217, + "hit_rate_at_3": 0.53614, + "hit_rate_at_5": 0.65964, + "hit_rate_at_10": 0.76506, + "hit_rate_at_20": 0.84337, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.3628, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.16717, + "ndcg_at_3": 0.18812, + "ndcg_at_5": 0.19855, + "ndcg_at_10": 0.2485, + "ndcg_at_20": 0.31176, + "ndcg_at_100": 0.39584, + "ndcg_at_1000": 0.39584, + "map_at_1": 0.06753, + "map_at_3": 0.11202, + "map_at_5": 0.12803, + "map_at_10": 0.15849, + "map_at_20": 0.18541, + "map_at_100": 0.21283, + "map_at_1000": 0.21283, + "recall_at_1": 0.06753, + "recall_at_3": 0.15208, + "recall_at_5": 0.20171, + "recall_at_10": 0.33989, + "recall_at_20": 0.52208, + "recall_at_100": 0.79964, + "recall_at_1000": 0.79964, + "accuracy": 0.06753, + "precision_at_1": 0.18373, + "precision_at_3": 0.15462, + "precision_at_5": 0.12952, + "precision_at_10": 0.11687, + "precision_at_20": 0.09488, + "precision_at_100": 0.0328, + "precision_at_1000": 0.00328, + "mrr_at_1": 0.183735, + "mrr_at_3": 0.261546, + "mrr_at_5": 0.282932, + "mrr_at_10": 0.307097, + "mrr_at_20": 0.31751, + "mrr_at_100": 0.322737, + "mrr_at_1000": 0.322737, + "nauc_ndcg_at_1_max": 0.028731, + "nauc_ndcg_at_1_std": 0.127453, + "nauc_ndcg_at_1_diff1": 0.02731, + "nauc_ndcg_at_3_max": 0.074854, + "nauc_ndcg_at_3_std": 0.137743, + "nauc_ndcg_at_3_diff1": 0.045606, + "nauc_ndcg_at_5_max": 0.072193, + "nauc_ndcg_at_5_std": 0.144373, + "nauc_ndcg_at_5_diff1": 0.065426, + "nauc_ndcg_at_10_max": 0.101489, + "nauc_ndcg_at_10_std": 0.186043, + "nauc_ndcg_at_10_diff1": 0.082514, + "nauc_ndcg_at_20_max": 0.107983, + "nauc_ndcg_at_20_std": 0.178554, + "nauc_ndcg_at_20_diff1": 0.109782, + "nauc_ndcg_at_100_max": 0.074813, + "nauc_ndcg_at_100_std": 0.18207, + "nauc_ndcg_at_100_diff1": 0.097653, + "nauc_ndcg_at_1000_max": 0.074813, + "nauc_ndcg_at_1000_std": 0.18207, + "nauc_ndcg_at_1000_diff1": 0.097653, + "nauc_map_at_1_max": 0.115459, + "nauc_map_at_1_std": 0.133232, + "nauc_map_at_1_diff1": 0.079771, + "nauc_map_at_3_max": 0.126636, + "nauc_map_at_3_std": 0.15445, + "nauc_map_at_3_diff1": 0.051863, + "nauc_map_at_5_max": 0.121999, + "nauc_map_at_5_std": 0.166603, + "nauc_map_at_5_diff1": 0.064843, + "nauc_map_at_10_max": 0.128484, + "nauc_map_at_10_std": 0.189236, + "nauc_map_at_10_diff1": 0.078476, + "nauc_map_at_20_max": 0.132988, + "nauc_map_at_20_std": 0.196537, + "nauc_map_at_20_diff1": 0.092365, + "nauc_map_at_100_max": 0.124716, + "nauc_map_at_100_std": 0.19187, + "nauc_map_at_100_diff1": 0.09422, + "nauc_map_at_1000_max": 0.124716, + "nauc_map_at_1000_std": 0.19187, + "nauc_map_at_1000_diff1": 0.09422, + "nauc_recall_at_1_max": 0.115459, + "nauc_recall_at_1_std": 0.133232, + "nauc_recall_at_1_diff1": 0.079771, + "nauc_recall_at_3_max": 0.103217, + "nauc_recall_at_3_std": 0.121552, + "nauc_recall_at_3_diff1": 0.009794, + "nauc_recall_at_5_max": 0.079843, + "nauc_recall_at_5_std": 0.119987, + "nauc_recall_at_5_diff1": 0.029702, + "nauc_recall_at_10_max": 0.105399, + "nauc_recall_at_10_std": 0.175126, + "nauc_recall_at_10_diff1": 0.07324, + "nauc_recall_at_20_max": 0.118804, + "nauc_recall_at_20_std": 0.159069, + "nauc_recall_at_20_diff1": 0.132398, + "nauc_recall_at_100_max": 0.027933, + "nauc_recall_at_100_std": 0.225194, + "nauc_recall_at_100_diff1": 0.153281, + "nauc_recall_at_1000_max": 0.027933, + "nauc_recall_at_1000_std": 0.225194, + "nauc_recall_at_1000_diff1": 0.153281, + "nauc_precision_at_1_max": 0.017351, + "nauc_precision_at_1_std": 0.117336, + "nauc_precision_at_1_diff1": 0.044221, + "nauc_precision_at_3_max": 0.040961, + "nauc_precision_at_3_std": 0.180928, + "nauc_precision_at_3_diff1": 0.071005, + "nauc_precision_at_5_max": 0.025038, + "nauc_precision_at_5_std": 0.175776, + "nauc_precision_at_5_diff1": 0.084792, + "nauc_precision_at_10_max": 0.082566, + "nauc_precision_at_10_std": 0.201578, + "nauc_precision_at_10_diff1": 0.09646, + "nauc_precision_at_20_max": 0.055857, + "nauc_precision_at_20_std": 0.123833, + "nauc_precision_at_20_diff1": 0.102288, + "nauc_precision_at_100_max": -0.028091, + "nauc_precision_at_100_std": 0.050547, + "nauc_precision_at_100_diff1": 0.032846, + "nauc_precision_at_1000_max": -0.028091, + "nauc_precision_at_1000_std": 0.050547, + "nauc_precision_at_1000_diff1": 0.032846, + "nauc_mrr_at_1_max": 0.017351, + "nauc_mrr_at_1_std": 0.117336, + "nauc_mrr_at_1_diff1": 0.044221, + "nauc_mrr_at_3_max": 0.034901, + "nauc_mrr_at_3_std": 0.126766, + "nauc_mrr_at_3_diff1": 0.052627, + "nauc_mrr_at_5_max": 0.027603, + "nauc_mrr_at_5_std": 0.127404, + "nauc_mrr_at_5_diff1": 0.049337, + "nauc_mrr_at_10_max": 0.036933, + "nauc_mrr_at_10_std": 0.134862, + "nauc_mrr_at_10_diff1": 0.060387, + "nauc_mrr_at_20_max": 0.037334, + "nauc_mrr_at_20_std": 0.127774, + "nauc_mrr_at_20_diff1": 0.061729, + "nauc_mrr_at_100_max": 0.034267, + "nauc_mrr_at_100_std": 0.129094, + "nauc_mrr_at_100_diff1": 0.058149, + "nauc_mrr_at_1000_max": 0.034267, + "nauc_mrr_at_1000_std": 0.129094, + "nauc_mrr_at_1000_diff1": 0.058149, + "hit_rate_at_1": 0.18373, + "hit_rate_at_3": 0.36446, + "hit_rate_at_5": 0.45783, + "hit_rate_at_10": 0.63855, + "hit_rate_at_20": 0.78916, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.2485, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.19127, + "ndcg_at_3": 0.20264, + "ndcg_at_5": 0.21513, + "ndcg_at_10": 0.26096, + "ndcg_at_20": 0.32428, + "ndcg_at_100": 0.40695, + "ndcg_at_1000": 0.40695, + "map_at_1": 0.06826, + "map_at_3": 0.11651, + "map_at_5": 0.13767, + "map_at_10": 0.16719, + "map_at_20": 0.19479, + "map_at_100": 0.2222, + "map_at_1000": 0.2222, + "recall_at_1": 0.06826, + "recall_at_3": 0.15303, + "recall_at_5": 0.21307, + "recall_at_10": 0.34576, + "recall_at_20": 0.53058, + "recall_at_100": 0.80754, + "recall_at_1000": 0.80754, + "accuracy": 0.06826, + "precision_at_1": 0.20783, + "precision_at_3": 0.16867, + "precision_at_5": 0.14699, + "precision_at_10": 0.1238, + "precision_at_20": 0.09864, + "precision_at_100": 0.03304, + "precision_at_1000": 0.0033, + "mrr_at_1": 0.207831, + "mrr_at_3": 0.280622, + "mrr_at_5": 0.305171, + "mrr_at_10": 0.329358, + "mrr_at_20": 0.338727, + "mrr_at_100": 0.343505, + "mrr_at_1000": 0.343505, + "nauc_ndcg_at_1_max": 0.063566, + "nauc_ndcg_at_1_std": 0.095021, + "nauc_ndcg_at_1_diff1": 0.096311, + "nauc_ndcg_at_3_max": 0.100813, + "nauc_ndcg_at_3_std": 0.14271, + "nauc_ndcg_at_3_diff1": 0.112318, + "nauc_ndcg_at_5_max": 0.116522, + "nauc_ndcg_at_5_std": 0.157378, + "nauc_ndcg_at_5_diff1": 0.094223, + "nauc_ndcg_at_10_max": 0.120101, + "nauc_ndcg_at_10_std": 0.183895, + "nauc_ndcg_at_10_diff1": 0.099855, + "nauc_ndcg_at_20_max": 0.091467, + "nauc_ndcg_at_20_std": 0.224804, + "nauc_ndcg_at_20_diff1": 0.097796, + "nauc_ndcg_at_100_max": 0.093439, + "nauc_ndcg_at_100_std": 0.239691, + "nauc_ndcg_at_100_diff1": 0.12495, + "nauc_ndcg_at_1000_max": 0.093439, + "nauc_ndcg_at_1000_std": 0.239691, + "nauc_ndcg_at_1000_diff1": 0.12495, + "nauc_map_at_1_max": 0.091712, + "nauc_map_at_1_std": 0.002226, + "nauc_map_at_1_diff1": 0.082409, + "nauc_map_at_3_max": 0.118382, + "nauc_map_at_3_std": 0.106849, + "nauc_map_at_3_diff1": 0.087633, + "nauc_map_at_5_max": 0.126854, + "nauc_map_at_5_std": 0.132193, + "nauc_map_at_5_diff1": 0.077708, + "nauc_map_at_10_max": 0.134454, + "nauc_map_at_10_std": 0.169646, + "nauc_map_at_10_diff1": 0.106079, + "nauc_map_at_20_max": 0.124771, + "nauc_map_at_20_std": 0.204937, + "nauc_map_at_20_diff1": 0.111594, + "nauc_map_at_100_max": 0.130011, + "nauc_map_at_100_std": 0.213185, + "nauc_map_at_100_diff1": 0.126318, + "nauc_map_at_1000_max": 0.130011, + "nauc_map_at_1000_std": 0.213185, + "nauc_map_at_1000_diff1": 0.126318, + "nauc_recall_at_1_max": 0.091712, + "nauc_recall_at_1_std": 0.002226, + "nauc_recall_at_1_diff1": 0.082409, + "nauc_recall_at_3_max": 0.112112, + "nauc_recall_at_3_std": 0.108728, + "nauc_recall_at_3_diff1": 0.061671, + "nauc_recall_at_5_max": 0.105972, + "nauc_recall_at_5_std": 0.113163, + "nauc_recall_at_5_diff1": 0.023394, + "nauc_recall_at_10_max": 0.07968, + "nauc_recall_at_10_std": 0.173115, + "nauc_recall_at_10_diff1": 0.045146, + "nauc_recall_at_20_max": -0.017495, + "nauc_recall_at_20_std": 0.225145, + "nauc_recall_at_20_diff1": 0.016592, + "nauc_recall_at_100_max": -0.070301, + "nauc_recall_at_100_std": 0.384804, + "nauc_recall_at_100_diff1": 0.082676, + "nauc_recall_at_1000_max": -0.070301, + "nauc_recall_at_1000_std": 0.384804, + "nauc_recall_at_1000_diff1": 0.082676, + "nauc_precision_at_1_max": 0.063123, + "nauc_precision_at_1_std": 0.109892, + "nauc_precision_at_1_diff1": 0.116524, + "nauc_precision_at_3_max": 0.106154, + "nauc_precision_at_3_std": 0.224639, + "nauc_precision_at_3_diff1": 0.138779, + "nauc_precision_at_5_max": 0.145205, + "nauc_precision_at_5_std": 0.27483, + "nauc_precision_at_5_diff1": 0.124207, + "nauc_precision_at_10_max": 0.179621, + "nauc_precision_at_10_std": 0.296566, + "nauc_precision_at_10_diff1": 0.156633, + "nauc_precision_at_20_max": 0.099843, + "nauc_precision_at_20_std": 0.299274, + "nauc_precision_at_20_diff1": 0.134689, + "nauc_precision_at_100_max": 0.07396, + "nauc_precision_at_100_std": 0.198543, + "nauc_precision_at_100_diff1": 0.124897, + "nauc_precision_at_1000_max": 0.07396, + "nauc_precision_at_1000_std": 0.198543, + "nauc_precision_at_1000_diff1": 0.124897, + "nauc_mrr_at_1_max": 0.063123, + "nauc_mrr_at_1_std": 0.109892, + "nauc_mrr_at_1_diff1": 0.116524, + "nauc_mrr_at_3_max": 0.087999, + "nauc_mrr_at_3_std": 0.156223, + "nauc_mrr_at_3_diff1": 0.124875, + "nauc_mrr_at_5_max": 0.095372, + "nauc_mrr_at_5_std": 0.155235, + "nauc_mrr_at_5_diff1": 0.111675, + "nauc_mrr_at_10_max": 0.087308, + "nauc_mrr_at_10_std": 0.154425, + "nauc_mrr_at_10_diff1": 0.109644, + "nauc_mrr_at_20_max": 0.082124, + "nauc_mrr_at_20_std": 0.155842, + "nauc_mrr_at_20_diff1": 0.109596, + "nauc_mrr_at_100_max": 0.083226, + "nauc_mrr_at_100_std": 0.15531, + "nauc_mrr_at_100_diff1": 0.110274, + "nauc_mrr_at_1000_max": 0.083226, + "nauc_mrr_at_1000_std": 0.15531, + "nauc_mrr_at_1000_diff1": 0.110274, + "hit_rate_at_1": 0.20783, + "hit_rate_at_3": 0.37651, + "hit_rate_at_5": 0.48494, + "hit_rate_at_10": 0.66867, + "hit_rate_at_20": 0.81024, + "hit_rate_at_100": 0.95181, + "hit_rate_at_1000": 0.95181, + "main_score": 0.26096, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.20482, + "ndcg_at_3": 0.23596, + "ndcg_at_5": 0.25065, + "ndcg_at_10": 0.28636, + "ndcg_at_20": 0.34295, + "ndcg_at_100": 0.4261, + "ndcg_at_1000": 0.4261, + "map_at_1": 0.07589, + "map_at_3": 0.14101, + "map_at_5": 0.16431, + "map_at_10": 0.18995, + "map_at_20": 0.21623, + "map_at_100": 0.24373, + "map_at_1000": 0.24373, + "recall_at_1": 0.07589, + "recall_at_3": 0.19151, + "recall_at_5": 0.25734, + "recall_at_10": 0.37199, + "recall_at_20": 0.527, + "recall_at_100": 0.80792, + "recall_at_1000": 0.80792, + "accuracy": 0.07589, + "precision_at_1": 0.22289, + "precision_at_3": 0.20382, + "precision_at_5": 0.17048, + "precision_at_10": 0.12922, + "precision_at_20": 0.1009, + "precision_at_100": 0.03334, + "precision_at_1000": 0.00333, + "mrr_at_1": 0.222892, + "mrr_at_3": 0.314759, + "mrr_at_5": 0.335843, + "mrr_at_10": 0.3563, + "mrr_at_20": 0.363356, + "mrr_at_100": 0.368575, + "mrr_at_1000": 0.368575, + "nauc_ndcg_at_1_max": 0.106924, + "nauc_ndcg_at_1_std": 0.047114, + "nauc_ndcg_at_1_diff1": 0.100082, + "nauc_ndcg_at_3_max": 0.07439, + "nauc_ndcg_at_3_std": 0.119011, + "nauc_ndcg_at_3_diff1": 0.088767, + "nauc_ndcg_at_5_max": 0.074726, + "nauc_ndcg_at_5_std": 0.165976, + "nauc_ndcg_at_5_diff1": 0.071111, + "nauc_ndcg_at_10_max": 0.062144, + "nauc_ndcg_at_10_std": 0.163837, + "nauc_ndcg_at_10_diff1": 0.080132, + "nauc_ndcg_at_20_max": 0.046948, + "nauc_ndcg_at_20_std": 0.166352, + "nauc_ndcg_at_20_diff1": 0.08163, + "nauc_ndcg_at_100_max": 0.037923, + "nauc_ndcg_at_100_std": 0.179648, + "nauc_ndcg_at_100_diff1": 0.092244, + "nauc_ndcg_at_1000_max": 0.037923, + "nauc_ndcg_at_1000_std": 0.179648, + "nauc_ndcg_at_1000_diff1": 0.092244, + "nauc_map_at_1_max": 0.157461, + "nauc_map_at_1_std": 0.014674, + "nauc_map_at_1_diff1": 0.065468, + "nauc_map_at_3_max": 0.091761, + "nauc_map_at_3_std": 0.124048, + "nauc_map_at_3_diff1": 0.066183, + "nauc_map_at_5_max": 0.090233, + "nauc_map_at_5_std": 0.16282, + "nauc_map_at_5_diff1": 0.066496, + "nauc_map_at_10_max": 0.088345, + "nauc_map_at_10_std": 0.171864, + "nauc_map_at_10_diff1": 0.079836, + "nauc_map_at_20_max": 0.079585, + "nauc_map_at_20_std": 0.182238, + "nauc_map_at_20_diff1": 0.08659, + "nauc_map_at_100_max": 0.079, + "nauc_map_at_100_std": 0.183328, + "nauc_map_at_100_diff1": 0.090094, + "nauc_map_at_1000_max": 0.079, + "nauc_map_at_1000_std": 0.183328, + "nauc_map_at_1000_diff1": 0.090094, + "nauc_recall_at_1_max": 0.157461, + "nauc_recall_at_1_std": 0.014674, + "nauc_recall_at_1_diff1": 0.065468, + "nauc_recall_at_3_max": 0.054053, + "nauc_recall_at_3_std": 0.122123, + "nauc_recall_at_3_diff1": 0.042549, + "nauc_recall_at_5_max": 0.043738, + "nauc_recall_at_5_std": 0.179518, + "nauc_recall_at_5_diff1": 0.023416, + "nauc_recall_at_10_max": 8.6e-05, + "nauc_recall_at_10_std": 0.144808, + "nauc_recall_at_10_diff1": 0.028101, + "nauc_recall_at_20_max": -0.060304, + "nauc_recall_at_20_std": 0.140531, + "nauc_recall_at_20_diff1": 0.01099, + "nauc_recall_at_100_max": -0.158821, + "nauc_recall_at_100_std": 0.249306, + "nauc_recall_at_100_diff1": 0.043552, + "nauc_recall_at_1000_max": -0.158821, + "nauc_recall_at_1000_std": 0.249306, + "nauc_recall_at_1000_diff1": 0.043552, + "nauc_precision_at_1_max": 0.093193, + "nauc_precision_at_1_std": 0.072768, + "nauc_precision_at_1_diff1": 0.094861, + "nauc_precision_at_3_max": 0.037407, + "nauc_precision_at_3_std": 0.182704, + "nauc_precision_at_3_diff1": 0.089438, + "nauc_precision_at_5_max": 0.056426, + "nauc_precision_at_5_std": 0.263298, + "nauc_precision_at_5_diff1": 0.071682, + "nauc_precision_at_10_max": 0.057052, + "nauc_precision_at_10_std": 0.222682, + "nauc_precision_at_10_diff1": 0.131676, + "nauc_precision_at_20_max": 0.037837, + "nauc_precision_at_20_std": 0.150992, + "nauc_precision_at_20_diff1": 0.115158, + "nauc_precision_at_100_max": 0.025747, + "nauc_precision_at_100_std": 0.086619, + "nauc_precision_at_100_diff1": 0.083141, + "nauc_precision_at_1000_max": 0.025747, + "nauc_precision_at_1000_std": 0.086619, + "nauc_precision_at_1000_diff1": 0.083141, + "nauc_mrr_at_1_max": 0.093193, + "nauc_mrr_at_1_std": 0.072768, + "nauc_mrr_at_1_diff1": 0.094861, + "nauc_mrr_at_3_max": 0.068246, + "nauc_mrr_at_3_std": 0.107038, + "nauc_mrr_at_3_diff1": 0.086239, + "nauc_mrr_at_5_max": 0.064057, + "nauc_mrr_at_5_std": 0.123385, + "nauc_mrr_at_5_diff1": 0.070027, + "nauc_mrr_at_10_max": 0.05318, + "nauc_mrr_at_10_std": 0.113896, + "nauc_mrr_at_10_diff1": 0.084426, + "nauc_mrr_at_20_max": 0.054318, + "nauc_mrr_at_20_std": 0.110974, + "nauc_mrr_at_20_diff1": 0.079436, + "nauc_mrr_at_100_max": 0.05428, + "nauc_mrr_at_100_std": 0.113226, + "nauc_mrr_at_100_diff1": 0.078408, + "nauc_mrr_at_1000_max": 0.05428, + "nauc_mrr_at_1000_std": 0.113226, + "nauc_mrr_at_1000_diff1": 0.078408, + "hit_rate_at_1": 0.22289, + "hit_rate_at_3": 0.44277, + "hit_rate_at_5": 0.53614, + "hit_rate_at_10": 0.69277, + "hit_rate_at_20": 0.79518, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.28636, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.13705, + "ndcg_at_3": 0.17472, + "ndcg_at_5": 0.19846, + "ndcg_at_10": 0.24227, + "ndcg_at_20": 0.30222, + "ndcg_at_100": 0.39315, + "ndcg_at_1000": 0.39315, + "map_at_1": 0.06194, + "map_at_3": 0.10661, + "map_at_5": 0.12929, + "map_at_10": 0.15533, + "map_at_20": 0.18148, + "map_at_100": 0.21102, + "map_at_1000": 0.21102, + "recall_at_1": 0.06194, + "recall_at_3": 0.14491, + "recall_at_5": 0.21249, + "recall_at_10": 0.32899, + "recall_at_20": 0.50283, + "recall_at_100": 0.80575, + "recall_at_1000": 0.80575, + "accuracy": 0.06194, + "precision_at_1": 0.15361, + "precision_at_3": 0.1506, + "precision_at_5": 0.13855, + "precision_at_10": 0.11687, + "precision_at_20": 0.09428, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.153614, + "mrr_at_3": 0.241968, + "mrr_at_5": 0.272239, + "mrr_at_10": 0.292162, + "mrr_at_20": 0.301669, + "mrr_at_100": 0.307444, + "mrr_at_1000": 0.307444, + "nauc_ndcg_at_1_max": 0.021411, + "nauc_ndcg_at_1_std": 0.123879, + "nauc_ndcg_at_1_diff1": 0.092732, + "nauc_ndcg_at_3_max": 0.040028, + "nauc_ndcg_at_3_std": 0.116877, + "nauc_ndcg_at_3_diff1": 0.062941, + "nauc_ndcg_at_5_max": 0.067978, + "nauc_ndcg_at_5_std": 0.11995, + "nauc_ndcg_at_5_diff1": 0.070745, + "nauc_ndcg_at_10_max": 0.09142, + "nauc_ndcg_at_10_std": 0.172449, + "nauc_ndcg_at_10_diff1": 0.066245, + "nauc_ndcg_at_20_max": 0.088418, + "nauc_ndcg_at_20_std": 0.183376, + "nauc_ndcg_at_20_diff1": 0.083286, + "nauc_ndcg_at_100_max": 0.042974, + "nauc_ndcg_at_100_std": 0.197873, + "nauc_ndcg_at_100_diff1": 0.115844, + "nauc_ndcg_at_1000_max": 0.042974, + "nauc_ndcg_at_1000_std": 0.197873, + "nauc_ndcg_at_1000_diff1": 0.115844, + "nauc_map_at_1_max": 0.089513, + "nauc_map_at_1_std": 0.144556, + "nauc_map_at_1_diff1": 0.083976, + "nauc_map_at_3_max": 0.081844, + "nauc_map_at_3_std": 0.157073, + "nauc_map_at_3_diff1": 0.066473, + "nauc_map_at_5_max": 0.101882, + "nauc_map_at_5_std": 0.169755, + "nauc_map_at_5_diff1": 0.073165, + "nauc_map_at_10_max": 0.109913, + "nauc_map_at_10_std": 0.199208, + "nauc_map_at_10_diff1": 0.070774, + "nauc_map_at_20_max": 0.108584, + "nauc_map_at_20_std": 0.209632, + "nauc_map_at_20_diff1": 0.077835, + "nauc_map_at_100_max": 0.092097, + "nauc_map_at_100_std": 0.216394, + "nauc_map_at_100_diff1": 0.093102, + "nauc_map_at_1000_max": 0.092097, + "nauc_map_at_1000_std": 0.216394, + "nauc_map_at_1000_diff1": 0.093102, + "nauc_recall_at_1_max": 0.089513, + "nauc_recall_at_1_std": 0.144556, + "nauc_recall_at_1_diff1": 0.083976, + "nauc_recall_at_3_max": 0.052304, + "nauc_recall_at_3_std": 0.110179, + "nauc_recall_at_3_diff1": 0.035966, + "nauc_recall_at_5_max": 0.074903, + "nauc_recall_at_5_std": 0.108748, + "nauc_recall_at_5_diff1": 0.053219, + "nauc_recall_at_10_max": 0.113381, + "nauc_recall_at_10_std": 0.157311, + "nauc_recall_at_10_diff1": 0.028603, + "nauc_recall_at_20_max": 0.111172, + "nauc_recall_at_20_std": 0.139671, + "nauc_recall_at_20_diff1": 0.024596, + "nauc_recall_at_100_max": 0.01517, + "nauc_recall_at_100_std": 0.206179, + "nauc_recall_at_100_diff1": 0.107326, + "nauc_recall_at_1000_max": 0.01517, + "nauc_recall_at_1000_std": 0.206179, + "nauc_recall_at_1000_diff1": 0.107326, + "nauc_precision_at_1_max": 0.03953, + "nauc_precision_at_1_std": 0.138358, + "nauc_precision_at_1_diff1": 0.068166, + "nauc_precision_at_3_max": 0.039706, + "nauc_precision_at_3_std": 0.122271, + "nauc_precision_at_3_diff1": 0.058004, + "nauc_precision_at_5_max": 0.072939, + "nauc_precision_at_5_std": 0.120649, + "nauc_precision_at_5_diff1": 0.064637, + "nauc_precision_at_10_max": 0.053259, + "nauc_precision_at_10_std": 0.173336, + "nauc_precision_at_10_diff1": 0.049964, + "nauc_precision_at_20_max": 0.006614, + "nauc_precision_at_20_std": 0.16986, + "nauc_precision_at_20_diff1": 0.072787, + "nauc_precision_at_100_max": -0.060661, + "nauc_precision_at_100_std": 0.119372, + "nauc_precision_at_100_diff1": 0.093017, + "nauc_precision_at_1000_max": -0.060661, + "nauc_precision_at_1000_std": 0.119372, + "nauc_precision_at_1000_diff1": 0.093017, + "nauc_mrr_at_1_max": 0.03953, + "nauc_mrr_at_1_std": 0.138358, + "nauc_mrr_at_1_diff1": 0.068166, + "nauc_mrr_at_3_max": 0.04087, + "nauc_mrr_at_3_std": 0.124824, + "nauc_mrr_at_3_diff1": 0.079047, + "nauc_mrr_at_5_max": 0.040356, + "nauc_mrr_at_5_std": 0.117691, + "nauc_mrr_at_5_diff1": 0.079935, + "nauc_mrr_at_10_max": 0.039786, + "nauc_mrr_at_10_std": 0.131118, + "nauc_mrr_at_10_diff1": 0.081035, + "nauc_mrr_at_20_max": 0.037884, + "nauc_mrr_at_20_std": 0.128964, + "nauc_mrr_at_20_diff1": 0.083186, + "nauc_mrr_at_100_max": 0.035616, + "nauc_mrr_at_100_std": 0.130611, + "nauc_mrr_at_100_diff1": 0.082191, + "nauc_mrr_at_1000_max": 0.035616, + "nauc_mrr_at_1000_std": 0.130611, + "nauc_mrr_at_1000_diff1": 0.082191, + "hit_rate_at_1": 0.15361, + "hit_rate_at_3": 0.35542, + "hit_rate_at_5": 0.49096, + "hit_rate_at_10": 0.64458, + "hit_rate_at_20": 0.7741, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.24227, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.19729, + "ndcg_at_3": 0.22692, + "ndcg_at_5": 0.24587, + "ndcg_at_10": 0.28493, + "ndcg_at_20": 0.34154, + "ndcg_at_100": 0.42299, + "ndcg_at_1000": 0.42299, + "map_at_1": 0.0738, + "map_at_3": 0.12943, + "map_at_5": 0.15552, + "map_at_10": 0.18534, + "map_at_20": 0.21219, + "map_at_100": 0.23932, + "map_at_1000": 0.23932, + "recall_at_1": 0.0738, + "recall_at_3": 0.17381, + "recall_at_5": 0.25016, + "recall_at_10": 0.36784, + "recall_at_20": 0.53672, + "recall_at_100": 0.80599, + "recall_at_1000": 0.80599, + "accuracy": 0.0738, + "precision_at_1": 0.21687, + "precision_at_3": 0.19478, + "precision_at_5": 0.17108, + "precision_at_10": 0.13554, + "precision_at_20": 0.10196, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.216867, + "mrr_at_3": 0.314257, + "mrr_at_5": 0.338504, + "mrr_at_10": 0.356289, + "mrr_at_20": 0.36504, + "mrr_at_100": 0.370071, + "mrr_at_1000": 0.370071, + "nauc_ndcg_at_1_max": 0.062583, + "nauc_ndcg_at_1_std": 0.140876, + "nauc_ndcg_at_1_diff1": 0.14654, + "nauc_ndcg_at_3_max": 0.098586, + "nauc_ndcg_at_3_std": 0.180985, + "nauc_ndcg_at_3_diff1": 0.216209, + "nauc_ndcg_at_5_max": 0.096656, + "nauc_ndcg_at_5_std": 0.179591, + "nauc_ndcg_at_5_diff1": 0.220919, + "nauc_ndcg_at_10_max": 0.085974, + "nauc_ndcg_at_10_std": 0.200546, + "nauc_ndcg_at_10_diff1": 0.22645, + "nauc_ndcg_at_20_max": 0.079708, + "nauc_ndcg_at_20_std": 0.235669, + "nauc_ndcg_at_20_diff1": 0.21199, + "nauc_ndcg_at_100_max": 0.067731, + "nauc_ndcg_at_100_std": 0.237761, + "nauc_ndcg_at_100_diff1": 0.212851, + "nauc_ndcg_at_1000_max": 0.067731, + "nauc_ndcg_at_1000_std": 0.237761, + "nauc_ndcg_at_1000_diff1": 0.212851, + "nauc_map_at_1_max": 0.116582, + "nauc_map_at_1_std": 0.090947, + "nauc_map_at_1_diff1": 0.263762, + "nauc_map_at_3_max": 0.11681, + "nauc_map_at_3_std": 0.164095, + "nauc_map_at_3_diff1": 0.250109, + "nauc_map_at_5_max": 0.114919, + "nauc_map_at_5_std": 0.177662, + "nauc_map_at_5_diff1": 0.253066, + "nauc_map_at_10_max": 0.095938, + "nauc_map_at_10_std": 0.199933, + "nauc_map_at_10_diff1": 0.256442, + "nauc_map_at_20_max": 0.098174, + "nauc_map_at_20_std": 0.222044, + "nauc_map_at_20_diff1": 0.25211, + "nauc_map_at_100_max": 0.099301, + "nauc_map_at_100_std": 0.222281, + "nauc_map_at_100_diff1": 0.257421, + "nauc_map_at_1000_max": 0.099301, + "nauc_map_at_1000_std": 0.222281, + "nauc_map_at_1000_diff1": 0.257421, + "nauc_recall_at_1_max": 0.116582, + "nauc_recall_at_1_std": 0.090947, + "nauc_recall_at_1_diff1": 0.263762, + "nauc_recall_at_3_max": 0.106639, + "nauc_recall_at_3_std": 0.139328, + "nauc_recall_at_3_diff1": 0.210646, + "nauc_recall_at_5_max": 0.083644, + "nauc_recall_at_5_std": 0.130484, + "nauc_recall_at_5_diff1": 0.186529, + "nauc_recall_at_10_max": 0.050296, + "nauc_recall_at_10_std": 0.164393, + "nauc_recall_at_10_diff1": 0.206152, + "nauc_recall_at_20_max": 0.000883, + "nauc_recall_at_20_std": 0.212417, + "nauc_recall_at_20_diff1": 0.130064, + "nauc_recall_at_100_max": -0.090903, + "nauc_recall_at_100_std": 0.324221, + "nauc_recall_at_100_diff1": 0.108549, + "nauc_recall_at_1000_max": -0.090903, + "nauc_recall_at_1000_std": 0.324221, + "nauc_recall_at_1000_diff1": 0.108549, + "nauc_precision_at_1_max": 0.068631, + "nauc_precision_at_1_std": 0.154, + "nauc_precision_at_1_diff1": 0.176546, + "nauc_precision_at_3_max": 0.095882, + "nauc_precision_at_3_std": 0.231567, + "nauc_precision_at_3_diff1": 0.213801, + "nauc_precision_at_5_max": 0.081312, + "nauc_precision_at_5_std": 0.227014, + "nauc_precision_at_5_diff1": 0.188354, + "nauc_precision_at_10_max": 0.047827, + "nauc_precision_at_10_std": 0.213532, + "nauc_precision_at_10_diff1": 0.16036, + "nauc_precision_at_20_max": 0.043296, + "nauc_precision_at_20_std": 0.224876, + "nauc_precision_at_20_diff1": 0.12089, + "nauc_precision_at_100_max": 0.024044, + "nauc_precision_at_100_std": 0.133058, + "nauc_precision_at_100_diff1": 0.077745, + "nauc_precision_at_1000_max": 0.024044, + "nauc_precision_at_1000_std": 0.133058, + "nauc_precision_at_1000_diff1": 0.077745, + "nauc_mrr_at_1_max": 0.068631, + "nauc_mrr_at_1_std": 0.154, + "nauc_mrr_at_1_diff1": 0.176546, + "nauc_mrr_at_3_max": 0.098434, + "nauc_mrr_at_3_std": 0.169015, + "nauc_mrr_at_3_diff1": 0.167876, + "nauc_mrr_at_5_max": 0.103657, + "nauc_mrr_at_5_std": 0.175543, + "nauc_mrr_at_5_diff1": 0.164445, + "nauc_mrr_at_10_max": 0.099647, + "nauc_mrr_at_10_std": 0.177712, + "nauc_mrr_at_10_diff1": 0.1717, + "nauc_mrr_at_20_max": 0.094531, + "nauc_mrr_at_20_std": 0.179328, + "nauc_mrr_at_20_diff1": 0.168599, + "nauc_mrr_at_100_max": 0.094442, + "nauc_mrr_at_100_std": 0.177917, + "nauc_mrr_at_100_diff1": 0.167676, + "nauc_mrr_at_1000_max": 0.094442, + "nauc_mrr_at_1000_std": 0.177917, + "nauc_mrr_at_1000_diff1": 0.167676, + "hit_rate_at_1": 0.21687, + "hit_rate_at_3": 0.44277, + "hit_rate_at_5": 0.5512, + "hit_rate_at_10": 0.67771, + "hit_rate_at_20": 0.80422, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.28493, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 416.69013203692157, + "kg_co2_emissions": null, + "date": 1792439873.940656, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 7.180962711572647e-06, + "end": 31.10690913000144, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 31.106938433949836, + "end": 31.106943240971304, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries", + "start": 31.130251934053376, + "end": 31.41482519498095, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding corpus", + "start": 31.428521133959293, + "end": 95.46715440100525, + "split": "test", + "subset": "french" + }, + { + "name": "Computing similarity", + "start": 95.46717542305123, + "end": 95.48834399494808, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 95.48990481905639, + "end": 95.53340255003422, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 95.56380533997435, + "end": 95.69061657600105, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding corpus", + "start": 95.70410954102408, + "end": 159.73711375403218, + "split": "test", + "subset": "spanish" + }, + { + "name": "Computing similarity", + "start": 159.7371802040143, + "end": 159.7577410689555, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 159.75914537895005, + "end": 159.80158106598537, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 159.83451464294922, + "end": 159.94676754099783, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding corpus", + "start": 159.9602635729825, + "end": 224.4263409289997, + "split": "test", + "subset": "english" + }, + { + "name": "Computing similarity", + "start": 224.426412204979, + "end": 224.44734295399394, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 224.44885760894977, + "end": 224.49079717404675, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 224.5249061289942, + "end": 224.65625013504177, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding corpus", + "start": 224.6699159500422, + "end": 288.4307746780105, + "split": "test", + "subset": "german" + }, + { + "name": "Computing similarity", + "start": 288.4308178520296, + "end": 288.4509253000142, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 288.45242694497574, + "end": 288.4943975160131, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 288.5272290200228, + "end": 288.6540188250365, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding corpus", + "start": 288.66748067503795, + "end": 352.6398481479846, + "split": "test", + "subset": "italian" + }, + { + "name": "Computing similarity", + "start": 352.6398911130382, + "end": 352.66167986404616, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 352.6635560999857, + "end": 352.70614697795827, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 352.73992774204817, + "end": 352.8671992570162, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Encoding corpus", + "start": 352.8806402559858, + "end": 416.6162909240229, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Computing similarity", + "start": 416.6163059170358, + "end": 416.63701819302514, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 416.63860959303565, + "end": 416.68030440900475, + "split": "test", + "subset": "portuguese" + } + ] +} diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/Vidore3TelecomNemotron35Embed8BReranking.v2.json b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/Vidore3TelecomNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..1b157b0a64 --- /dev/null +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/Vidore3TelecomNemotron35Embed8BReranking.v2.json @@ -0,0 +1,1132 @@ +{ + "dataset_revision": "47c9b703527792f3fbf0a082f4589ff40c1aa8cc", + "task_name": "Vidore3TelecomNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.5, + "ndcg_at_3": 0.47936, + "ndcg_at_5": 0.49798, + "ndcg_at_10": 0.53954, + "ndcg_at_20": 0.58427, + "ndcg_at_100": 0.64089, + "ndcg_at_1000": 0.64089, + "map_at_1": 0.21158, + "map_at_3": 0.31765, + "map_at_5": 0.36073, + "map_at_10": 0.40729, + "map_at_20": 0.43861, + "map_at_100": 0.46361, + "map_at_1000": 0.46361, + "recall_at_1": 0.21158, + "recall_at_3": 0.36879, + "recall_at_5": 0.47314, + "recall_at_10": 0.62123, + "recall_at_20": 0.75414, + "recall_at_100": 0.92961, + "recall_at_1000": 0.92961, + "accuracy": 0.21158, + "precision_at_1": 0.52874, + "precision_at_3": 0.36398, + "precision_at_5": 0.2954, + "precision_at_10": 0.21178, + "precision_at_20": 0.14195, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.528736, + "mrr_at_3": 0.620211, + "mrr_at_5": 0.640757, + "mrr_at_10": 0.652731, + "mrr_at_20": 0.655573, + "mrr_at_100": 0.656744, + "mrr_at_1000": 0.656744, + "nauc_ndcg_at_1_max": 0.314181, + "nauc_ndcg_at_1_std": 0.234914, + "nauc_ndcg_at_1_diff1": 0.355716, + "nauc_ndcg_at_3_max": 0.316843, + "nauc_ndcg_at_3_std": 0.27072, + "nauc_ndcg_at_3_diff1": 0.296352, + "nauc_ndcg_at_5_max": 0.330539, + "nauc_ndcg_at_5_std": 0.300836, + "nauc_ndcg_at_5_diff1": 0.30329, + "nauc_ndcg_at_10_max": 0.347809, + "nauc_ndcg_at_10_std": 0.313275, + "nauc_ndcg_at_10_diff1": 0.320907, + "nauc_ndcg_at_20_max": 0.376397, + "nauc_ndcg_at_20_std": 0.326604, + "nauc_ndcg_at_20_diff1": 0.339174, + "nauc_ndcg_at_100_max": 0.367134, + "nauc_ndcg_at_100_std": 0.309227, + "nauc_ndcg_at_100_diff1": 0.323936, + "nauc_ndcg_at_1000_max": 0.367134, + "nauc_ndcg_at_1000_std": 0.309227, + "nauc_ndcg_at_1000_diff1": 0.323936, + "nauc_map_at_1_max": 0.149992, + "nauc_map_at_1_std": 0.142074, + "nauc_map_at_1_diff1": 0.343137, + "nauc_map_at_3_max": 0.241812, + "nauc_map_at_3_std": 0.220462, + "nauc_map_at_3_diff1": 0.315529, + "nauc_map_at_5_max": 0.262011, + "nauc_map_at_5_std": 0.252393, + "nauc_map_at_5_diff1": 0.315854, + "nauc_map_at_10_max": 0.294863, + "nauc_map_at_10_std": 0.288919, + "nauc_map_at_10_diff1": 0.312689, + "nauc_map_at_20_max": 0.317725, + "nauc_map_at_20_std": 0.309144, + "nauc_map_at_20_diff1": 0.320456, + "nauc_map_at_100_max": 0.318843, + "nauc_map_at_100_std": 0.311469, + "nauc_map_at_100_diff1": 0.313839, + "nauc_map_at_1000_max": 0.318843, + "nauc_map_at_1000_std": 0.311469, + "nauc_map_at_1000_diff1": 0.313839, + "nauc_recall_at_1_max": 0.149992, + "nauc_recall_at_1_std": 0.142074, + "nauc_recall_at_1_diff1": 0.343137, + "nauc_recall_at_3_max": 0.228978, + "nauc_recall_at_3_std": 0.214886, + "nauc_recall_at_3_diff1": 0.281628, + "nauc_recall_at_5_max": 0.235279, + "nauc_recall_at_5_std": 0.235892, + "nauc_recall_at_5_diff1": 0.240072, + "nauc_recall_at_10_max": 0.264306, + "nauc_recall_at_10_std": 0.284731, + "nauc_recall_at_10_diff1": 0.218061, + "nauc_recall_at_20_max": 0.368261, + "nauc_recall_at_20_std": 0.333204, + "nauc_recall_at_20_diff1": 0.268124, + "nauc_recall_at_100_max": 0.508863, + "nauc_recall_at_100_std": 0.33195, + "nauc_recall_at_100_diff1": 0.244512, + "nauc_recall_at_1000_max": 0.508863, + "nauc_recall_at_1000_std": 0.33195, + "nauc_recall_at_1000_diff1": 0.244512, + "nauc_precision_at_1_max": 0.33847, + "nauc_precision_at_1_std": 0.251079, + "nauc_precision_at_1_diff1": 0.4033, + "nauc_precision_at_3_max": 0.331346, + "nauc_precision_at_3_std": 0.27585, + "nauc_precision_at_3_diff1": 0.165639, + "nauc_precision_at_5_max": 0.298178, + "nauc_precision_at_5_std": 0.296143, + "nauc_precision_at_5_diff1": 0.092296, + "nauc_precision_at_10_max": 0.25204, + "nauc_precision_at_10_std": 0.269625, + "nauc_precision_at_10_diff1": 0.036354, + "nauc_precision_at_20_max": 0.188577, + "nauc_precision_at_20_std": 0.206606, + "nauc_precision_at_20_diff1": -0.021192, + "nauc_precision_at_100_max": 0.05536, + "nauc_precision_at_100_std": 0.075241, + "nauc_precision_at_100_diff1": -0.102761, + "nauc_precision_at_1000_max": 0.05536, + "nauc_precision_at_1000_std": 0.075241, + "nauc_precision_at_1000_diff1": -0.102761, + "nauc_mrr_at_1_max": 0.33847, + "nauc_mrr_at_1_std": 0.251079, + "nauc_mrr_at_1_diff1": 0.4033, + "nauc_mrr_at_3_max": 0.353031, + "nauc_mrr_at_3_std": 0.261942, + "nauc_mrr_at_3_diff1": 0.39509, + "nauc_mrr_at_5_max": 0.359426, + "nauc_mrr_at_5_std": 0.264102, + "nauc_mrr_at_5_diff1": 0.392738, + "nauc_mrr_at_10_max": 0.357018, + "nauc_mrr_at_10_std": 0.263535, + "nauc_mrr_at_10_diff1": 0.398987, + "nauc_mrr_at_20_max": 0.355617, + "nauc_mrr_at_20_std": 0.262817, + "nauc_mrr_at_20_diff1": 0.398216, + "nauc_mrr_at_100_max": 0.354336, + "nauc_mrr_at_100_std": 0.261576, + "nauc_mrr_at_100_diff1": 0.39792, + "nauc_mrr_at_1000_max": 0.354336, + "nauc_mrr_at_1000_std": 0.261576, + "nauc_mrr_at_1000_diff1": 0.39792, + "hit_rate_at_1": 0.52874, + "hit_rate_at_3": 0.73563, + "hit_rate_at_5": 0.82759, + "hit_rate_at_10": 0.91667, + "hit_rate_at_20": 0.95977, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.53954, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.37069, + "ndcg_at_3": 0.3753, + "ndcg_at_5": 0.40987, + "ndcg_at_10": 0.4533, + "ndcg_at_20": 0.50723, + "ndcg_at_100": 0.57272, + "ndcg_at_1000": 0.57272, + "map_at_1": 0.14573, + "map_at_3": 0.24326, + "map_at_5": 0.2881, + "map_at_10": 0.32987, + "map_at_20": 0.36075, + "map_at_100": 0.38769, + "map_at_1000": 0.38769, + "recall_at_1": 0.14573, + "recall_at_3": 0.31055, + "recall_at_5": 0.41672, + "recall_at_10": 0.55501, + "recall_at_20": 0.71829, + "recall_at_100": 0.92187, + "recall_at_1000": 0.92187, + "accuracy": 0.14573, + "precision_at_1": 0.39943, + "precision_at_3": 0.30077, + "precision_at_5": 0.26207, + "precision_at_10": 0.18736, + "precision_at_20": 0.1329, + "precision_at_100": 0.03882, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.399425, + "mrr_at_3": 0.502395, + "mrr_at_5": 0.52955, + "mrr_at_10": 0.542967, + "mrr_at_20": 0.548974, + "mrr_at_100": 0.550625, + "mrr_at_1000": 0.550625, + "nauc_ndcg_at_1_max": 0.271062, + "nauc_ndcg_at_1_std": 0.263525, + "nauc_ndcg_at_1_diff1": 0.374891, + "nauc_ndcg_at_3_max": 0.261981, + "nauc_ndcg_at_3_std": 0.325482, + "nauc_ndcg_at_3_diff1": 0.321281, + "nauc_ndcg_at_5_max": 0.283751, + "nauc_ndcg_at_5_std": 0.328593, + "nauc_ndcg_at_5_diff1": 0.337043, + "nauc_ndcg_at_10_max": 0.310466, + "nauc_ndcg_at_10_std": 0.366272, + "nauc_ndcg_at_10_diff1": 0.355146, + "nauc_ndcg_at_20_max": 0.323125, + "nauc_ndcg_at_20_std": 0.365349, + "nauc_ndcg_at_20_diff1": 0.37134, + "nauc_ndcg_at_100_max": 0.300216, + "nauc_ndcg_at_100_std": 0.336719, + "nauc_ndcg_at_100_diff1": 0.359255, + "nauc_ndcg_at_1000_max": 0.300216, + "nauc_ndcg_at_1000_std": 0.336719, + "nauc_ndcg_at_1000_diff1": 0.359255, + "nauc_map_at_1_max": 0.160371, + "nauc_map_at_1_std": 0.180135, + "nauc_map_at_1_diff1": 0.391569, + "nauc_map_at_3_max": 0.231085, + "nauc_map_at_3_std": 0.289928, + "nauc_map_at_3_diff1": 0.343834, + "nauc_map_at_5_max": 0.25752, + "nauc_map_at_5_std": 0.300703, + "nauc_map_at_5_diff1": 0.347102, + "nauc_map_at_10_max": 0.293955, + "nauc_map_at_10_std": 0.33897, + "nauc_map_at_10_diff1": 0.349467, + "nauc_map_at_20_max": 0.302368, + "nauc_map_at_20_std": 0.347107, + "nauc_map_at_20_diff1": 0.353726, + "nauc_map_at_100_max": 0.293413, + "nauc_map_at_100_std": 0.335618, + "nauc_map_at_100_diff1": 0.347757, + "nauc_map_at_1000_max": 0.293413, + "nauc_map_at_1000_std": 0.335618, + "nauc_map_at_1000_diff1": 0.347757, + "nauc_recall_at_1_max": 0.160371, + "nauc_recall_at_1_std": 0.180135, + "nauc_recall_at_1_diff1": 0.391569, + "nauc_recall_at_3_max": 0.192291, + "nauc_recall_at_3_std": 0.268077, + "nauc_recall_at_3_diff1": 0.255938, + "nauc_recall_at_5_max": 0.203103, + "nauc_recall_at_5_std": 0.267859, + "nauc_recall_at_5_diff1": 0.245431, + "nauc_recall_at_10_max": 0.259228, + "nauc_recall_at_10_std": 0.331403, + "nauc_recall_at_10_diff1": 0.258328, + "nauc_recall_at_20_max": 0.324368, + "nauc_recall_at_20_std": 0.346695, + "nauc_recall_at_20_diff1": 0.270157, + "nauc_recall_at_100_max": 0.31888, + "nauc_recall_at_100_std": 0.271623, + "nauc_recall_at_100_diff1": 0.215903, + "nauc_recall_at_1000_max": 0.31888, + "nauc_recall_at_1000_std": 0.271623, + "nauc_recall_at_1000_diff1": 0.215903, + "nauc_precision_at_1_max": 0.283225, + "nauc_precision_at_1_std": 0.250045, + "nauc_precision_at_1_diff1": 0.371973, + "nauc_precision_at_3_max": 0.280395, + "nauc_precision_at_3_std": 0.3212, + "nauc_precision_at_3_diff1": 0.226411, + "nauc_precision_at_5_max": 0.281863, + "nauc_precision_at_5_std": 0.270609, + "nauc_precision_at_5_diff1": 0.188895, + "nauc_precision_at_10_max": 0.231166, + "nauc_precision_at_10_std": 0.260355, + "nauc_precision_at_10_diff1": 0.114224, + "nauc_precision_at_20_max": 0.132741, + "nauc_precision_at_20_std": 0.139406, + "nauc_precision_at_20_diff1": 0.0315, + "nauc_precision_at_100_max": -0.025994, + "nauc_precision_at_100_std": -0.031959, + "nauc_precision_at_100_diff1": -0.083347, + "nauc_precision_at_1000_max": -0.025994, + "nauc_precision_at_1000_std": -0.031959, + "nauc_precision_at_1000_diff1": -0.083347, + "nauc_mrr_at_1_max": 0.283225, + "nauc_mrr_at_1_std": 0.250045, + "nauc_mrr_at_1_diff1": 0.371973, + "nauc_mrr_at_3_max": 0.283798, + "nauc_mrr_at_3_std": 0.290235, + "nauc_mrr_at_3_diff1": 0.34502, + "nauc_mrr_at_5_max": 0.281208, + "nauc_mrr_at_5_std": 0.28641, + "nauc_mrr_at_5_diff1": 0.350803, + "nauc_mrr_at_10_max": 0.282226, + "nauc_mrr_at_10_std": 0.290103, + "nauc_mrr_at_10_diff1": 0.357365, + "nauc_mrr_at_20_max": 0.286278, + "nauc_mrr_at_20_std": 0.288967, + "nauc_mrr_at_20_diff1": 0.35579, + "nauc_mrr_at_100_max": 0.285683, + "nauc_mrr_at_100_std": 0.287758, + "nauc_mrr_at_100_diff1": 0.354646, + "nauc_mrr_at_1000_max": 0.285683, + "nauc_mrr_at_1000_std": 0.287758, + "nauc_mrr_at_1000_diff1": 0.354646, + "hit_rate_at_1": 0.39943, + "hit_rate_at_3": 0.64368, + "hit_rate_at_5": 0.75862, + "hit_rate_at_10": 0.85632, + "hit_rate_at_20": 0.94253, + "hit_rate_at_100": 0.99138, + "hit_rate_at_1000": 0.99138, + "main_score": 0.4533, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.43822, + "ndcg_at_3": 0.44295, + "ndcg_at_5": 0.45669, + "ndcg_at_10": 0.50308, + "ndcg_at_20": 0.54796, + "ndcg_at_100": 0.61325, + "ndcg_at_1000": 0.61325, + "map_at_1": 0.18418, + "map_at_3": 0.28857, + "map_at_5": 0.32681, + "map_at_10": 0.37412, + "map_at_20": 0.40436, + "map_at_100": 0.4315, + "map_at_1000": 0.4315, + "recall_at_1": 0.18418, + "recall_at_3": 0.34913, + "recall_at_5": 0.43864, + "recall_at_10": 0.58595, + "recall_at_20": 0.72096, + "recall_at_100": 0.93008, + "recall_at_1000": 0.93008, + "accuracy": 0.18418, + "precision_at_1": 0.46264, + "precision_at_3": 0.34195, + "precision_at_5": 0.27241, + "precision_at_10": 0.20201, + "precision_at_20": 0.13621, + "precision_at_100": 0.03905, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.462644, + "mrr_at_3": 0.567529, + "mrr_at_5": 0.584339, + "mrr_at_10": 0.600571, + "mrr_at_20": 0.604142, + "mrr_at_100": 0.606056, + "mrr_at_1000": 0.606056, + "nauc_ndcg_at_1_max": 0.14967, + "nauc_ndcg_at_1_std": 0.179233, + "nauc_ndcg_at_1_diff1": 0.390253, + "nauc_ndcg_at_3_max": 0.179077, + "nauc_ndcg_at_3_std": 0.252172, + "nauc_ndcg_at_3_diff1": 0.303408, + "nauc_ndcg_at_5_max": 0.176497, + "nauc_ndcg_at_5_std": 0.25196, + "nauc_ndcg_at_5_diff1": 0.2931, + "nauc_ndcg_at_10_max": 0.185254, + "nauc_ndcg_at_10_std": 0.282451, + "nauc_ndcg_at_10_diff1": 0.297924, + "nauc_ndcg_at_20_max": 0.220366, + "nauc_ndcg_at_20_std": 0.28815, + "nauc_ndcg_at_20_diff1": 0.313265, + "nauc_ndcg_at_100_max": 0.207923, + "nauc_ndcg_at_100_std": 0.277626, + "nauc_ndcg_at_100_diff1": 0.331431, + "nauc_ndcg_at_1000_max": 0.207923, + "nauc_ndcg_at_1000_std": 0.277626, + "nauc_ndcg_at_1000_diff1": 0.331431, + "nauc_map_at_1_max": 0.061082, + "nauc_map_at_1_std": 0.108117, + "nauc_map_at_1_diff1": 0.360889, + "nauc_map_at_3_max": 0.14483, + "nauc_map_at_3_std": 0.203651, + "nauc_map_at_3_diff1": 0.301325, + "nauc_map_at_5_max": 0.146898, + "nauc_map_at_5_std": 0.215806, + "nauc_map_at_5_diff1": 0.290934, + "nauc_map_at_10_max": 0.172339, + "nauc_map_at_10_std": 0.261593, + "nauc_map_at_10_diff1": 0.296045, + "nauc_map_at_20_max": 0.193508, + "nauc_map_at_20_std": 0.277059, + "nauc_map_at_20_diff1": 0.302307, + "nauc_map_at_100_max": 0.192447, + "nauc_map_at_100_std": 0.275578, + "nauc_map_at_100_diff1": 0.306559, + "nauc_map_at_1000_max": 0.192447, + "nauc_map_at_1000_std": 0.275578, + "nauc_map_at_1000_diff1": 0.306559, + "nauc_recall_at_1_max": 0.061082, + "nauc_recall_at_1_std": 0.108117, + "nauc_recall_at_1_diff1": 0.360889, + "nauc_recall_at_3_max": 0.152222, + "nauc_recall_at_3_std": 0.231232, + "nauc_recall_at_3_diff1": 0.22772, + "nauc_recall_at_5_max": 0.132213, + "nauc_recall_at_5_std": 0.220584, + "nauc_recall_at_5_diff1": 0.176507, + "nauc_recall_at_10_max": 0.172104, + "nauc_recall_at_10_std": 0.311593, + "nauc_recall_at_10_diff1": 0.171249, + "nauc_recall_at_20_max": 0.26778, + "nauc_recall_at_20_std": 0.308907, + "nauc_recall_at_20_diff1": 0.171256, + "nauc_recall_at_100_max": 0.383636, + "nauc_recall_at_100_std": 0.420052, + "nauc_recall_at_100_diff1": 0.169635, + "nauc_recall_at_1000_max": 0.383636, + "nauc_recall_at_1000_std": 0.420052, + "nauc_recall_at_1000_diff1": 0.169635, + "nauc_precision_at_1_max": 0.165866, + "nauc_precision_at_1_std": 0.195387, + "nauc_precision_at_1_diff1": 0.419234, + "nauc_precision_at_3_max": 0.230462, + "nauc_precision_at_3_std": 0.282004, + "nauc_precision_at_3_diff1": 0.183558, + "nauc_precision_at_5_max": 0.202345, + "nauc_precision_at_5_std": 0.250224, + "nauc_precision_at_5_diff1": 0.118382, + "nauc_precision_at_10_max": 0.174554, + "nauc_precision_at_10_std": 0.238605, + "nauc_precision_at_10_diff1": 0.073849, + "nauc_precision_at_20_max": 0.139692, + "nauc_precision_at_20_std": 0.154366, + "nauc_precision_at_20_diff1": 0.024555, + "nauc_precision_at_100_max": 0.024295, + "nauc_precision_at_100_std": 0.027063, + "nauc_precision_at_100_diff1": -0.033691, + "nauc_precision_at_1000_max": 0.024295, + "nauc_precision_at_1000_std": 0.027063, + "nauc_precision_at_1000_diff1": -0.033691, + "nauc_mrr_at_1_max": 0.165866, + "nauc_mrr_at_1_std": 0.195387, + "nauc_mrr_at_1_diff1": 0.419234, + "nauc_mrr_at_3_max": 0.194088, + "nauc_mrr_at_3_std": 0.238789, + "nauc_mrr_at_3_diff1": 0.399093, + "nauc_mrr_at_5_max": 0.189525, + "nauc_mrr_at_5_std": 0.241134, + "nauc_mrr_at_5_diff1": 0.394164, + "nauc_mrr_at_10_max": 0.186497, + "nauc_mrr_at_10_std": 0.2324, + "nauc_mrr_at_10_diff1": 0.396537, + "nauc_mrr_at_20_max": 0.185707, + "nauc_mrr_at_20_std": 0.228396, + "nauc_mrr_at_20_diff1": 0.397877, + "nauc_mrr_at_100_max": 0.184261, + "nauc_mrr_at_100_std": 0.228555, + "nauc_mrr_at_100_diff1": 0.398195, + "nauc_mrr_at_1000_max": 0.184261, + "nauc_mrr_at_1000_std": 0.228555, + "nauc_mrr_at_1000_diff1": 0.398195, + "hit_rate_at_1": 0.46264, + "hit_rate_at_3": 0.70115, + "hit_rate_at_5": 0.77586, + "hit_rate_at_10": 0.8908, + "hit_rate_at_20": 0.93966, + "hit_rate_at_100": 0.99713, + "hit_rate_at_1000": 0.99713, + "main_score": 0.50308, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.43822, + "ndcg_at_3": 0.43258, + "ndcg_at_5": 0.45283, + "ndcg_at_10": 0.49705, + "ndcg_at_20": 0.54416, + "ndcg_at_100": 0.60931, + "ndcg_at_1000": 0.60931, + "map_at_1": 0.1809, + "map_at_3": 0.28765, + "map_at_5": 0.32588, + "map_at_10": 0.37139, + "map_at_20": 0.40064, + "map_at_100": 0.42749, + "map_at_1000": 0.42749, + "recall_at_1": 0.1809, + "recall_at_3": 0.35189, + "recall_at_5": 0.44577, + "recall_at_10": 0.58732, + "recall_at_20": 0.72628, + "recall_at_100": 0.93189, + "recall_at_1000": 0.93189, + "accuracy": 0.1809, + "precision_at_1": 0.46839, + "precision_at_3": 0.33142, + "precision_at_5": 0.27184, + "precision_at_10": 0.19971, + "precision_at_20": 0.13621, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.468391, + "mrr_at_3": 0.562261, + "mrr_at_5": 0.585393, + "mrr_at_10": 0.596621, + "mrr_at_20": 0.600355, + "mrr_at_100": 0.602551, + "mrr_at_1000": 0.602551, + "nauc_ndcg_at_1_max": 0.076026, + "nauc_ndcg_at_1_std": 0.193633, + "nauc_ndcg_at_1_diff1": 0.350214, + "nauc_ndcg_at_3_max": 0.140378, + "nauc_ndcg_at_3_std": 0.256745, + "nauc_ndcg_at_3_diff1": 0.363738, + "nauc_ndcg_at_5_max": 0.142041, + "nauc_ndcg_at_5_std": 0.287249, + "nauc_ndcg_at_5_diff1": 0.361161, + "nauc_ndcg_at_10_max": 0.167989, + "nauc_ndcg_at_10_std": 0.331286, + "nauc_ndcg_at_10_diff1": 0.366943, + "nauc_ndcg_at_20_max": 0.197709, + "nauc_ndcg_at_20_std": 0.330824, + "nauc_ndcg_at_20_diff1": 0.379556, + "nauc_ndcg_at_100_max": 0.177916, + "nauc_ndcg_at_100_std": 0.302404, + "nauc_ndcg_at_100_diff1": 0.381513, + "nauc_ndcg_at_1000_max": 0.177916, + "nauc_ndcg_at_1000_std": 0.302404, + "nauc_ndcg_at_1000_diff1": 0.381513, + "nauc_map_at_1_max": 0.003197, + "nauc_map_at_1_std": 0.145595, + "nauc_map_at_1_diff1": 0.362788, + "nauc_map_at_3_max": 0.088065, + "nauc_map_at_3_std": 0.256998, + "nauc_map_at_3_diff1": 0.343936, + "nauc_map_at_5_max": 0.104878, + "nauc_map_at_5_std": 0.274756, + "nauc_map_at_5_diff1": 0.342521, + "nauc_map_at_10_max": 0.139274, + "nauc_map_at_10_std": 0.314789, + "nauc_map_at_10_diff1": 0.346928, + "nauc_map_at_20_max": 0.16011, + "nauc_map_at_20_std": 0.320183, + "nauc_map_at_20_diff1": 0.352391, + "nauc_map_at_100_max": 0.153941, + "nauc_map_at_100_std": 0.314586, + "nauc_map_at_100_diff1": 0.353583, + "nauc_map_at_1000_max": 0.153941, + "nauc_map_at_1000_std": 0.314586, + "nauc_map_at_1000_diff1": 0.353583, + "nauc_recall_at_1_max": 0.003197, + "nauc_recall_at_1_std": 0.145595, + "nauc_recall_at_1_diff1": 0.362788, + "nauc_recall_at_3_max": 0.107236, + "nauc_recall_at_3_std": 0.241408, + "nauc_recall_at_3_diff1": 0.314009, + "nauc_recall_at_5_max": 0.109952, + "nauc_recall_at_5_std": 0.296027, + "nauc_recall_at_5_diff1": 0.290278, + "nauc_recall_at_10_max": 0.174314, + "nauc_recall_at_10_std": 0.361392, + "nauc_recall_at_10_diff1": 0.300637, + "nauc_recall_at_20_max": 0.279018, + "nauc_recall_at_20_std": 0.386318, + "nauc_recall_at_20_diff1": 0.304828, + "nauc_recall_at_100_max": 0.428469, + "nauc_recall_at_100_std": 0.425291, + "nauc_recall_at_100_diff1": 0.391498, + "nauc_recall_at_1000_max": 0.428469, + "nauc_recall_at_1000_std": 0.425291, + "nauc_recall_at_1000_diff1": 0.391498, + "nauc_precision_at_1_max": 0.092949, + "nauc_precision_at_1_std": 0.197238, + "nauc_precision_at_1_diff1": 0.368157, + "nauc_precision_at_3_max": 0.21426, + "nauc_precision_at_3_std": 0.26878, + "nauc_precision_at_3_diff1": 0.273758, + "nauc_precision_at_5_max": 0.207927, + "nauc_precision_at_5_std": 0.239756, + "nauc_precision_at_5_diff1": 0.177364, + "nauc_precision_at_10_max": 0.208986, + "nauc_precision_at_10_std": 0.229789, + "nauc_precision_at_10_diff1": 0.100038, + "nauc_precision_at_20_max": 0.163228, + "nauc_precision_at_20_std": 0.111579, + "nauc_precision_at_20_diff1": 0.029447, + "nauc_precision_at_100_max": 0.02329, + "nauc_precision_at_100_std": -0.033822, + "nauc_precision_at_100_diff1": -0.056399, + "nauc_precision_at_1000_max": 0.02329, + "nauc_precision_at_1000_std": -0.033822, + "nauc_precision_at_1000_diff1": -0.056399, + "nauc_mrr_at_1_max": 0.092949, + "nauc_mrr_at_1_std": 0.197238, + "nauc_mrr_at_1_diff1": 0.368157, + "nauc_mrr_at_3_max": 0.131844, + "nauc_mrr_at_3_std": 0.200371, + "nauc_mrr_at_3_diff1": 0.381776, + "nauc_mrr_at_5_max": 0.123234, + "nauc_mrr_at_5_std": 0.22337, + "nauc_mrr_at_5_diff1": 0.376912, + "nauc_mrr_at_10_max": 0.121682, + "nauc_mrr_at_10_std": 0.225596, + "nauc_mrr_at_10_diff1": 0.385208, + "nauc_mrr_at_20_max": 0.120049, + "nauc_mrr_at_20_std": 0.220967, + "nauc_mrr_at_20_diff1": 0.382735, + "nauc_mrr_at_100_max": 0.11989, + "nauc_mrr_at_100_std": 0.219948, + "nauc_mrr_at_100_diff1": 0.381594, + "nauc_mrr_at_1000_max": 0.11989, + "nauc_mrr_at_1000_std": 0.219948, + "nauc_mrr_at_1000_diff1": 0.381594, + "hit_rate_at_1": 0.46839, + "hit_rate_at_3": 0.69253, + "hit_rate_at_5": 0.79598, + "hit_rate_at_10": 0.87931, + "hit_rate_at_20": 0.93391, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.49705, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.36782, + "ndcg_at_3": 0.36932, + "ndcg_at_5": 0.38955, + "ndcg_at_10": 0.44572, + "ndcg_at_20": 0.49623, + "ndcg_at_100": 0.56787, + "ndcg_at_1000": 0.56787, + "map_at_1": 0.14557, + "map_at_3": 0.23994, + "map_at_5": 0.27612, + "map_at_10": 0.32191, + "map_at_20": 0.34909, + "map_at_100": 0.37916, + "map_at_1000": 0.37916, + "recall_at_1": 0.14557, + "recall_at_3": 0.29981, + "recall_at_5": 0.38856, + "recall_at_10": 0.56014, + "recall_at_20": 0.70742, + "recall_at_100": 0.93007, + "recall_at_1000": 0.93007, + "accuracy": 0.14557, + "precision_at_1": 0.39368, + "precision_at_3": 0.29502, + "precision_at_5": 0.24023, + "precision_at_10": 0.18506, + "precision_at_20": 0.12917, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.393678, + "mrr_at_3": 0.488985, + "mrr_at_5": 0.508094, + "mrr_at_10": 0.530711, + "mrr_at_20": 0.536001, + "mrr_at_100": 0.537754, + "mrr_at_1000": 0.537754, + "nauc_ndcg_at_1_max": 0.220796, + "nauc_ndcg_at_1_std": 0.171225, + "nauc_ndcg_at_1_diff1": 0.220471, + "nauc_ndcg_at_3_max": 0.223291, + "nauc_ndcg_at_3_std": 0.21479, + "nauc_ndcg_at_3_diff1": 0.266016, + "nauc_ndcg_at_5_max": 0.237377, + "nauc_ndcg_at_5_std": 0.21699, + "nauc_ndcg_at_5_diff1": 0.257447, + "nauc_ndcg_at_10_max": 0.257534, + "nauc_ndcg_at_10_std": 0.261102, + "nauc_ndcg_at_10_diff1": 0.237937, + "nauc_ndcg_at_20_max": 0.258281, + "nauc_ndcg_at_20_std": 0.264418, + "nauc_ndcg_at_20_diff1": 0.25735, + "nauc_ndcg_at_100_max": 0.216826, + "nauc_ndcg_at_100_std": 0.210755, + "nauc_ndcg_at_100_diff1": 0.251827, + "nauc_ndcg_at_1000_max": 0.216826, + "nauc_ndcg_at_1000_std": 0.210755, + "nauc_ndcg_at_1000_diff1": 0.251827, + "nauc_map_at_1_max": 0.13438, + "nauc_map_at_1_std": 0.134994, + "nauc_map_at_1_diff1": 0.242943, + "nauc_map_at_3_max": 0.190322, + "nauc_map_at_3_std": 0.208904, + "nauc_map_at_3_diff1": 0.260026, + "nauc_map_at_5_max": 0.212836, + "nauc_map_at_5_std": 0.211934, + "nauc_map_at_5_diff1": 0.246531, + "nauc_map_at_10_max": 0.22954, + "nauc_map_at_10_std": 0.249724, + "nauc_map_at_10_diff1": 0.234423, + "nauc_map_at_20_max": 0.235669, + "nauc_map_at_20_std": 0.256247, + "nauc_map_at_20_diff1": 0.242629, + "nauc_map_at_100_max": 0.218572, + "nauc_map_at_100_std": 0.235306, + "nauc_map_at_100_diff1": 0.246839, + "nauc_map_at_1000_max": 0.218572, + "nauc_map_at_1000_std": 0.235306, + "nauc_map_at_1000_diff1": 0.246839, + "nauc_recall_at_1_max": 0.13438, + "nauc_recall_at_1_std": 0.134994, + "nauc_recall_at_1_diff1": 0.242943, + "nauc_recall_at_3_max": 0.197556, + "nauc_recall_at_3_std": 0.235114, + "nauc_recall_at_3_diff1": 0.285027, + "nauc_recall_at_5_max": 0.197753, + "nauc_recall_at_5_std": 0.203552, + "nauc_recall_at_5_diff1": 0.227724, + "nauc_recall_at_10_max": 0.244149, + "nauc_recall_at_10_std": 0.296738, + "nauc_recall_at_10_diff1": 0.160963, + "nauc_recall_at_20_max": 0.281622, + "nauc_recall_at_20_std": 0.319892, + "nauc_recall_at_20_diff1": 0.203633, + "nauc_recall_at_100_max": 0.263698, + "nauc_recall_at_100_std": 0.26239, + "nauc_recall_at_100_diff1": 0.250629, + "nauc_recall_at_1000_max": 0.263698, + "nauc_recall_at_1000_std": 0.26239, + "nauc_recall_at_1000_diff1": 0.250629, + "nauc_precision_at_1_max": 0.221723, + "nauc_precision_at_1_std": 0.168094, + "nauc_precision_at_1_diff1": 0.234833, + "nauc_precision_at_3_max": 0.238019, + "nauc_precision_at_3_std": 0.204245, + "nauc_precision_at_3_diff1": 0.195742, + "nauc_precision_at_5_max": 0.226569, + "nauc_precision_at_5_std": 0.174147, + "nauc_precision_at_5_diff1": 0.143845, + "nauc_precision_at_10_max": 0.161692, + "nauc_precision_at_10_std": 0.172577, + "nauc_precision_at_10_diff1": 0.062229, + "nauc_precision_at_20_max": 0.061029, + "nauc_precision_at_20_std": 0.079566, + "nauc_precision_at_20_diff1": 0.058386, + "nauc_precision_at_100_max": -0.087889, + "nauc_precision_at_100_std": -0.112676, + "nauc_precision_at_100_diff1": -0.016656, + "nauc_precision_at_1000_max": -0.087889, + "nauc_precision_at_1000_std": -0.112676, + "nauc_precision_at_1000_diff1": -0.016656, + "nauc_mrr_at_1_max": 0.221723, + "nauc_mrr_at_1_std": 0.168094, + "nauc_mrr_at_1_diff1": 0.234833, + "nauc_mrr_at_3_max": 0.256079, + "nauc_mrr_at_3_std": 0.19188, + "nauc_mrr_at_3_diff1": 0.262323, + "nauc_mrr_at_5_max": 0.246397, + "nauc_mrr_at_5_std": 0.184786, + "nauc_mrr_at_5_diff1": 0.244108, + "nauc_mrr_at_10_max": 0.250233, + "nauc_mrr_at_10_std": 0.186355, + "nauc_mrr_at_10_diff1": 0.241486, + "nauc_mrr_at_20_max": 0.247394, + "nauc_mrr_at_20_std": 0.185755, + "nauc_mrr_at_20_diff1": 0.244462, + "nauc_mrr_at_100_max": 0.247205, + "nauc_mrr_at_100_std": 0.185575, + "nauc_mrr_at_100_diff1": 0.244272, + "nauc_mrr_at_1000_max": 0.247205, + "nauc_mrr_at_1000_std": 0.185575, + "nauc_mrr_at_1000_diff1": 0.244272, + "hit_rate_at_1": 0.39368, + "hit_rate_at_3": 0.61782, + "hit_rate_at_5": 0.70115, + "hit_rate_at_10": 0.86494, + "hit_rate_at_20": 0.94253, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.44572, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.4023, + "ndcg_at_3": 0.41049, + "ndcg_at_5": 0.43585, + "ndcg_at_10": 0.48645, + "ndcg_at_20": 0.53502, + "ndcg_at_100": 0.60052, + "ndcg_at_1000": 0.60052, + "map_at_1": 0.1733, + "map_at_3": 0.26983, + "map_at_5": 0.3129, + "map_at_10": 0.36005, + "map_at_20": 0.39118, + "map_at_100": 0.41863, + "map_at_1000": 0.41863, + "recall_at_1": 0.1733, + "recall_at_3": 0.32517, + "recall_at_5": 0.42406, + "recall_at_10": 0.58736, + "recall_at_20": 0.72943, + "recall_at_100": 0.93268, + "recall_at_1000": 0.93268, + "accuracy": 0.1733, + "precision_at_1": 0.42816, + "precision_at_3": 0.31897, + "precision_at_5": 0.26954, + "precision_at_10": 0.19655, + "precision_at_20": 0.13621, + "precision_at_100": 0.03937, + "precision_at_1000": 0.00394, + "mrr_at_1": 0.428161, + "mrr_at_3": 0.534004, + "mrr_at_5": 0.558429, + "mrr_at_10": 0.57413, + "mrr_at_20": 0.578635, + "mrr_at_100": 0.580644, + "mrr_at_1000": 0.580644, + "nauc_ndcg_at_1_max": 0.086633, + "nauc_ndcg_at_1_std": 0.256533, + "nauc_ndcg_at_1_diff1": 0.393343, + "nauc_ndcg_at_3_max": 0.149838, + "nauc_ndcg_at_3_std": 0.293796, + "nauc_ndcg_at_3_diff1": 0.352959, + "nauc_ndcg_at_5_max": 0.159989, + "nauc_ndcg_at_5_std": 0.327867, + "nauc_ndcg_at_5_diff1": 0.334095, + "nauc_ndcg_at_10_max": 0.17991, + "nauc_ndcg_at_10_std": 0.318299, + "nauc_ndcg_at_10_diff1": 0.342935, + "nauc_ndcg_at_20_max": 0.183582, + "nauc_ndcg_at_20_std": 0.33526, + "nauc_ndcg_at_20_diff1": 0.353534, + "nauc_ndcg_at_100_max": 0.178171, + "nauc_ndcg_at_100_std": 0.315126, + "nauc_ndcg_at_100_diff1": 0.361195, + "nauc_ndcg_at_1000_max": 0.178171, + "nauc_ndcg_at_1000_std": 0.315126, + "nauc_ndcg_at_1000_diff1": 0.361195, + "nauc_map_at_1_max": -0.0211, + "nauc_map_at_1_std": 0.17745, + "nauc_map_at_1_diff1": 0.336165, + "nauc_map_at_3_max": 0.069554, + "nauc_map_at_3_std": 0.263404, + "nauc_map_at_3_diff1": 0.322531, + "nauc_map_at_5_max": 0.096852, + "nauc_map_at_5_std": 0.288509, + "nauc_map_at_5_diff1": 0.311917, + "nauc_map_at_10_max": 0.133811, + "nauc_map_at_10_std": 0.303232, + "nauc_map_at_10_diff1": 0.31361, + "nauc_map_at_20_max": 0.147285, + "nauc_map_at_20_std": 0.320236, + "nauc_map_at_20_diff1": 0.324938, + "nauc_map_at_100_max": 0.14633, + "nauc_map_at_100_std": 0.317851, + "nauc_map_at_100_diff1": 0.330062, + "nauc_map_at_1000_max": 0.14633, + "nauc_map_at_1000_std": 0.317851, + "nauc_map_at_1000_diff1": 0.330062, + "nauc_recall_at_1_max": -0.0211, + "nauc_recall_at_1_std": 0.17745, + "nauc_recall_at_1_diff1": 0.336165, + "nauc_recall_at_3_max": 0.058306, + "nauc_recall_at_3_std": 0.267646, + "nauc_recall_at_3_diff1": 0.289184, + "nauc_recall_at_5_max": 0.092383, + "nauc_recall_at_5_std": 0.305579, + "nauc_recall_at_5_diff1": 0.235304, + "nauc_recall_at_10_max": 0.16637, + "nauc_recall_at_10_std": 0.261135, + "nauc_recall_at_10_diff1": 0.233026, + "nauc_recall_at_20_max": 0.186275, + "nauc_recall_at_20_std": 0.302425, + "nauc_recall_at_20_diff1": 0.261395, + "nauc_recall_at_100_max": 0.338031, + "nauc_recall_at_100_std": 0.237258, + "nauc_recall_at_100_diff1": 0.352803, + "nauc_recall_at_1000_max": 0.338031, + "nauc_recall_at_1000_std": 0.237258, + "nauc_recall_at_1000_diff1": 0.352803, + "nauc_precision_at_1_max": 0.077469, + "nauc_precision_at_1_std": 0.252236, + "nauc_precision_at_1_diff1": 0.407262, + "nauc_precision_at_3_max": 0.221823, + "nauc_precision_at_3_std": 0.257616, + "nauc_precision_at_3_diff1": 0.251228, + "nauc_precision_at_5_max": 0.238588, + "nauc_precision_at_5_std": 0.269312, + "nauc_precision_at_5_diff1": 0.155348, + "nauc_precision_at_10_max": 0.235631, + "nauc_precision_at_10_std": 0.202146, + "nauc_precision_at_10_diff1": 0.087246, + "nauc_precision_at_20_max": 0.165883, + "nauc_precision_at_20_std": 0.146261, + "nauc_precision_at_20_diff1": 0.063825, + "nauc_precision_at_100_max": 0.055194, + "nauc_precision_at_100_std": 0.004786, + "nauc_precision_at_100_diff1": -0.013545, + "nauc_precision_at_1000_max": 0.055194, + "nauc_precision_at_1000_std": 0.004786, + "nauc_precision_at_1000_diff1": -0.013545, + "nauc_mrr_at_1_max": 0.077469, + "nauc_mrr_at_1_std": 0.252236, + "nauc_mrr_at_1_diff1": 0.407262, + "nauc_mrr_at_3_max": 0.119552, + "nauc_mrr_at_3_std": 0.26289, + "nauc_mrr_at_3_diff1": 0.414004, + "nauc_mrr_at_5_max": 0.117453, + "nauc_mrr_at_5_std": 0.281151, + "nauc_mrr_at_5_diff1": 0.395338, + "nauc_mrr_at_10_max": 0.112887, + "nauc_mrr_at_10_std": 0.270251, + "nauc_mrr_at_10_diff1": 0.401156, + "nauc_mrr_at_20_max": 0.114283, + "nauc_mrr_at_20_std": 0.271819, + "nauc_mrr_at_20_diff1": 0.401766, + "nauc_mrr_at_100_max": 0.113648, + "nauc_mrr_at_100_std": 0.270261, + "nauc_mrr_at_100_diff1": 0.401498, + "nauc_mrr_at_1000_max": 0.113648, + "nauc_mrr_at_1000_std": 0.270261, + "nauc_mrr_at_1000_diff1": 0.401498, + "hit_rate_at_1": 0.42816, + "hit_rate_at_3": 0.66379, + "hit_rate_at_5": 0.76724, + "hit_rate_at_10": 0.88218, + "hit_rate_at_20": 0.93966, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.48645, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 640.5292287591146, + "kg_co2_emissions": null, + "date": 1792439457.147066, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 1.435109879821539e-05, + "end": 31.3412769860588, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 31.341369344037957, + "end": 31.341375233023427, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries", + "start": 31.36925027007237, + "end": 46.04694979602937, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding corpus", + "start": 46.06879396410659, + "end": 144.4086305850651, + "split": "test", + "subset": "french" + }, + { + "name": "Computing similarity", + "start": 144.40868822101038, + "end": 144.43128648702987, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 144.43272343708668, + "end": 144.48139108205214, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 144.51734182308428, + "end": 144.570451934007, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding corpus", + "start": 144.59106007299852, + "end": 242.78911903710105, + "split": "test", + "subset": "spanish" + }, + { + "name": "Computing similarity", + "start": 242.78916203207336, + "end": 242.81088824302424, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 242.81233692006208, + "end": 242.85398424300365, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 242.88688925805036, + "end": 242.93909032805823, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding corpus", + "start": 242.95992831105832, + "end": 341.0673024050193, + "split": "test", + "subset": "english" + }, + { + "name": "Computing similarity", + "start": 341.0673447990557, + "end": 341.08946010202635, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 341.09097756107803, + "end": 341.1329810670577, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 341.1656573941, + "end": 341.2247318990994, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding corpus", + "start": 341.24538090010174, + "end": 439.72524819907267, + "split": "test", + "subset": "german" + }, + { + "name": "Computing similarity", + "start": 439.72528317105025, + "end": 439.7464534790488, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 439.74792151502334, + "end": 439.7897779570194, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 439.8226359930122, + "end": 439.8763907790417, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding corpus", + "start": 439.897382941097, + "end": 538.4984574880218, + "split": "test", + "subset": "italian" + }, + { + "name": "Computing similarity", + "start": 538.4984950741054, + "end": 538.5204111170024, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 538.5222556150984, + "end": 538.5652613260318, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 538.5993162420345, + "end": 538.6527672220254, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Encoding corpus", + "start": 538.6736607761122, + "end": 640.4528925260529, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Computing similarity", + "start": 640.4529305630131, + "end": 640.4757160530426, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 640.4772848990979, + "end": 640.5197051180294, + "split": "test", + "subset": "portuguese" + } + ] +} diff --git a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/run_settings.jsonl b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/run_settings.jsonl index 65a247c067..058df377a5 100644 --- a/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/run_settings.jsonl +++ b/results/intfloat__multilingual-e5-small/fd1525a9fd15316a2d503bf26ab031a61d056e98/run_settings.jsonl @@ -223,3 +223,5 @@ {"task": "Vidore3PhysicsNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "SIB200Classification.v3", "splits": ["test"], "version": {"mteb": "2.20.12"}, "subsets": ["ita_Latn"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 64}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 64}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} diff --git a/results/mteb__baseline-bm25s/0_3_0/Vidore3NuclearNemotron35Embed8BReranking.v2.json b/results/mteb__baseline-bm25s/0_3_0/Vidore3NuclearNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..9dc34e70e6 --- /dev/null +++ b/results/mteb__baseline-bm25s/0_3_0/Vidore3NuclearNemotron35Embed8BReranking.v2.json @@ -0,0 +1,1090 @@ +{ + "dataset_revision": "97ff9e10e265e64abece589ff40c552215754f26", + "task_name": "Vidore3NuclearNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.38102, + "ndcg_at_3": 0.40406, + "ndcg_at_5": 0.43175, + "ndcg_at_10": 0.47182, + "ndcg_at_20": 0.51298, + "ndcg_at_100": 0.55627, + "ndcg_at_1000": 0.55627, + "map_at_1": 0.17193, + "map_at_3": 0.26585, + "map_at_5": 0.31016, + "map_at_10": 0.35308, + "map_at_20": 0.37997, + "map_at_100": 0.39725, + "map_at_1000": 0.39725, + "recall_at_1": 0.17193, + "recall_at_3": 0.33288, + "recall_at_5": 0.43583, + "recall_at_10": 0.56341, + "recall_at_20": 0.67396, + "recall_at_100": 0.80968, + "recall_at_1000": 0.80968, + "accuracy": 0.17193, + "precision_at_1": 0.39458, + "precision_at_3": 0.3002, + "precision_at_5": 0.25904, + "precision_at_10": 0.19187, + "precision_at_20": 0.12937, + "precision_at_100": 0.0337, + "precision_at_1000": 0.00337, + "mrr_at_1": 0.394578, + "mrr_at_3": 0.509538, + "mrr_at_5": 0.533333, + "mrr_at_10": 0.545732, + "mrr_at_20": 0.548857, + "mrr_at_100": 0.550296, + "mrr_at_1000": 0.550296, + "nauc_ndcg_at_1_max": 0.292972, + "nauc_ndcg_at_1_std": 0.228596, + "nauc_ndcg_at_1_diff1": 0.390814, + "nauc_ndcg_at_3_max": 0.319709, + "nauc_ndcg_at_3_std": 0.259473, + "nauc_ndcg_at_3_diff1": 0.359581, + "nauc_ndcg_at_5_max": 0.339611, + "nauc_ndcg_at_5_std": 0.28425, + "nauc_ndcg_at_5_diff1": 0.382469, + "nauc_ndcg_at_10_max": 0.368176, + "nauc_ndcg_at_10_std": 0.340023, + "nauc_ndcg_at_10_diff1": 0.365625, + "nauc_ndcg_at_20_max": 0.377033, + "nauc_ndcg_at_20_std": 0.35641, + "nauc_ndcg_at_20_diff1": 0.363528, + "nauc_ndcg_at_100_max": 0.388199, + "nauc_ndcg_at_100_std": 0.371196, + "nauc_ndcg_at_100_diff1": 0.360875, + "nauc_ndcg_at_1000_max": 0.388199, + "nauc_ndcg_at_1000_std": 0.371196, + "nauc_ndcg_at_1000_diff1": 0.360875, + "nauc_map_at_1_max": 0.240778, + "nauc_map_at_1_std": 0.08272, + "nauc_map_at_1_diff1": 0.3962, + "nauc_map_at_3_max": 0.288636, + "nauc_map_at_3_std": 0.166621, + "nauc_map_at_3_diff1": 0.370852, + "nauc_map_at_5_max": 0.320458, + "nauc_map_at_5_std": 0.223294, + "nauc_map_at_5_diff1": 0.381388, + "nauc_map_at_10_max": 0.342136, + "nauc_map_at_10_std": 0.276282, + "nauc_map_at_10_diff1": 0.374996, + "nauc_map_at_20_max": 0.347878, + "nauc_map_at_20_std": 0.292244, + "nauc_map_at_20_diff1": 0.369665, + "nauc_map_at_100_max": 0.355433, + "nauc_map_at_100_std": 0.302248, + "nauc_map_at_100_diff1": 0.36797, + "nauc_map_at_1000_max": 0.355433, + "nauc_map_at_1000_std": 0.302248, + "nauc_map_at_1000_diff1": 0.36797, + "nauc_recall_at_1_max": 0.240778, + "nauc_recall_at_1_std": 0.08272, + "nauc_recall_at_1_diff1": 0.3962, + "nauc_recall_at_3_max": 0.280574, + "nauc_recall_at_3_std": 0.177552, + "nauc_recall_at_3_diff1": 0.299018, + "nauc_recall_at_5_max": 0.306322, + "nauc_recall_at_5_std": 0.237538, + "nauc_recall_at_5_diff1": 0.332819, + "nauc_recall_at_10_max": 0.363, + "nauc_recall_at_10_std": 0.364399, + "nauc_recall_at_10_diff1": 0.297142, + "nauc_recall_at_20_max": 0.393132, + "nauc_recall_at_20_std": 0.424803, + "nauc_recall_at_20_diff1": 0.277673, + "nauc_recall_at_100_max": 0.494387, + "nauc_recall_at_100_std": 0.570112, + "nauc_recall_at_100_diff1": 0.244652, + "nauc_recall_at_1000_max": 0.494387, + "nauc_recall_at_1000_std": 0.570112, + "nauc_recall_at_1000_diff1": 0.244652, + "nauc_precision_at_1_max": 0.300187, + "nauc_precision_at_1_std": 0.238572, + "nauc_precision_at_1_diff1": 0.389503, + "nauc_precision_at_3_max": 0.302114, + "nauc_precision_at_3_std": 0.363248, + "nauc_precision_at_3_diff1": 0.253655, + "nauc_precision_at_5_max": 0.28564, + "nauc_precision_at_5_std": 0.395228, + "nauc_precision_at_5_diff1": 0.20332, + "nauc_precision_at_10_max": 0.221531, + "nauc_precision_at_10_std": 0.371237, + "nauc_precision_at_10_diff1": 0.09023, + "nauc_precision_at_20_max": 0.135881, + "nauc_precision_at_20_std": 0.272373, + "nauc_precision_at_20_diff1": 0.019232, + "nauc_precision_at_100_max": 0.068154, + "nauc_precision_at_100_std": 0.185171, + "nauc_precision_at_100_diff1": -0.045445, + "nauc_precision_at_1000_max": 0.068154, + "nauc_precision_at_1000_std": 0.185171, + "nauc_precision_at_1000_diff1": -0.045445, + "nauc_mrr_at_1_max": 0.300187, + "nauc_mrr_at_1_std": 0.238572, + "nauc_mrr_at_1_diff1": 0.389503, + "nauc_mrr_at_3_max": 0.328073, + "nauc_mrr_at_3_std": 0.298315, + "nauc_mrr_at_3_diff1": 0.3749, + "nauc_mrr_at_5_max": 0.331071, + "nauc_mrr_at_5_std": 0.302825, + "nauc_mrr_at_5_diff1": 0.382147, + "nauc_mrr_at_10_max": 0.335963, + "nauc_mrr_at_10_std": 0.308656, + "nauc_mrr_at_10_diff1": 0.378807, + "nauc_mrr_at_20_max": 0.333473, + "nauc_mrr_at_20_std": 0.305004, + "nauc_mrr_at_20_diff1": 0.378418, + "nauc_mrr_at_100_max": 0.333186, + "nauc_mrr_at_100_std": 0.304305, + "nauc_mrr_at_100_diff1": 0.378309, + "nauc_mrr_at_1000_max": 0.333186, + "nauc_mrr_at_1000_std": 0.304305, + "nauc_mrr_at_1000_diff1": 0.378309, + "hit_rate_at_1": 0.39458, + "hit_rate_at_3": 0.65663, + "hit_rate_at_5": 0.75904, + "hit_rate_at_10": 0.85241, + "hit_rate_at_20": 0.89759, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.47182, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.21687, + "ndcg_at_3": 0.24485, + "ndcg_at_5": 0.27, + "ndcg_at_10": 0.30894, + "ndcg_at_20": 0.3617, + "ndcg_at_100": 0.41599, + "ndcg_at_1000": 0.41599, + "map_at_1": 0.09692, + "map_at_3": 0.1512, + "map_at_5": 0.18169, + "map_at_10": 0.21595, + "map_at_20": 0.2409, + "map_at_100": 0.25863, + "map_at_1000": 0.25863, + "recall_at_1": 0.09692, + "recall_at_3": 0.19906, + "recall_at_5": 0.27671, + "recall_at_10": 0.39379, + "recall_at_20": 0.54428, + "recall_at_100": 0.72506, + "recall_at_1000": 0.72506, + "accuracy": 0.09692, + "precision_at_1": 0.23494, + "precision_at_3": 0.18976, + "precision_at_5": 0.1741, + "precision_at_10": 0.13946, + "precision_at_20": 0.10331, + "precision_at_100": 0.0287, + "precision_at_1000": 0.00287, + "mrr_at_1": 0.23494, + "mrr_at_3": 0.322289, + "mrr_at_5": 0.349548, + "mrr_at_10": 0.367365, + "mrr_at_20": 0.377194, + "mrr_at_100": 0.380421, + "mrr_at_1000": 0.380421, + "nauc_ndcg_at_1_max": 0.275242, + "nauc_ndcg_at_1_std": 0.186295, + "nauc_ndcg_at_1_diff1": 0.202959, + "nauc_ndcg_at_3_max": 0.363242, + "nauc_ndcg_at_3_std": 0.281595, + "nauc_ndcg_at_3_diff1": 0.216076, + "nauc_ndcg_at_5_max": 0.390465, + "nauc_ndcg_at_5_std": 0.31681, + "nauc_ndcg_at_5_diff1": 0.229679, + "nauc_ndcg_at_10_max": 0.407558, + "nauc_ndcg_at_10_std": 0.358086, + "nauc_ndcg_at_10_diff1": 0.244326, + "nauc_ndcg_at_20_max": 0.419387, + "nauc_ndcg_at_20_std": 0.381442, + "nauc_ndcg_at_20_diff1": 0.249855, + "nauc_ndcg_at_100_max": 0.434608, + "nauc_ndcg_at_100_std": 0.382296, + "nauc_ndcg_at_100_diff1": 0.272091, + "nauc_ndcg_at_1000_max": 0.434608, + "nauc_ndcg_at_1000_std": 0.382296, + "nauc_ndcg_at_1000_diff1": 0.272091, + "nauc_map_at_1_max": 0.333011, + "nauc_map_at_1_std": 0.190289, + "nauc_map_at_1_diff1": 0.267317, + "nauc_map_at_3_max": 0.389908, + "nauc_map_at_3_std": 0.272982, + "nauc_map_at_3_diff1": 0.231504, + "nauc_map_at_5_max": 0.408279, + "nauc_map_at_5_std": 0.308475, + "nauc_map_at_5_diff1": 0.232558, + "nauc_map_at_10_max": 0.419669, + "nauc_map_at_10_std": 0.34189, + "nauc_map_at_10_diff1": 0.251332, + "nauc_map_at_20_max": 0.426182, + "nauc_map_at_20_std": 0.353705, + "nauc_map_at_20_diff1": 0.252803, + "nauc_map_at_100_max": 0.432837, + "nauc_map_at_100_std": 0.355727, + "nauc_map_at_100_diff1": 0.264372, + "nauc_map_at_1000_max": 0.432837, + "nauc_map_at_1000_std": 0.355727, + "nauc_map_at_1000_diff1": 0.264372, + "nauc_recall_at_1_max": 0.333011, + "nauc_recall_at_1_std": 0.190289, + "nauc_recall_at_1_diff1": 0.267317, + "nauc_recall_at_3_max": 0.36641, + "nauc_recall_at_3_std": 0.272072, + "nauc_recall_at_3_diff1": 0.173681, + "nauc_recall_at_5_max": 0.370543, + "nauc_recall_at_5_std": 0.297911, + "nauc_recall_at_5_diff1": 0.164119, + "nauc_recall_at_10_max": 0.393381, + "nauc_recall_at_10_std": 0.376503, + "nauc_recall_at_10_diff1": 0.19818, + "nauc_recall_at_20_max": 0.412174, + "nauc_recall_at_20_std": 0.435288, + "nauc_recall_at_20_diff1": 0.193135, + "nauc_recall_at_100_max": 0.48469, + "nauc_recall_at_100_std": 0.485225, + "nauc_recall_at_100_diff1": 0.242223, + "nauc_recall_at_1000_max": 0.48469, + "nauc_recall_at_1000_std": 0.485225, + "nauc_recall_at_1000_diff1": 0.242223, + "nauc_precision_at_1_max": 0.278099, + "nauc_precision_at_1_std": 0.193945, + "nauc_precision_at_1_diff1": 0.196516, + "nauc_precision_at_3_max": 0.345921, + "nauc_precision_at_3_std": 0.329589, + "nauc_precision_at_3_diff1": 0.195894, + "nauc_precision_at_5_max": 0.318168, + "nauc_precision_at_5_std": 0.329859, + "nauc_precision_at_5_diff1": 0.202064, + "nauc_precision_at_10_max": 0.265094, + "nauc_precision_at_10_std": 0.316942, + "nauc_precision_at_10_diff1": 0.205298, + "nauc_precision_at_20_max": 0.172829, + "nauc_precision_at_20_std": 0.221179, + "nauc_precision_at_20_diff1": 0.179047, + "nauc_precision_at_100_max": 0.115745, + "nauc_precision_at_100_std": 0.124018, + "nauc_precision_at_100_diff1": 0.164556, + "nauc_precision_at_1000_max": 0.115745, + "nauc_precision_at_1000_std": 0.124018, + "nauc_precision_at_1000_diff1": 0.164556, + "nauc_mrr_at_1_max": 0.278099, + "nauc_mrr_at_1_std": 0.193945, + "nauc_mrr_at_1_diff1": 0.196516, + "nauc_mrr_at_3_max": 0.340512, + "nauc_mrr_at_3_std": 0.278713, + "nauc_mrr_at_3_diff1": 0.221391, + "nauc_mrr_at_5_max": 0.334411, + "nauc_mrr_at_5_std": 0.2732, + "nauc_mrr_at_5_diff1": 0.230598, + "nauc_mrr_at_10_max": 0.337187, + "nauc_mrr_at_10_std": 0.278815, + "nauc_mrr_at_10_diff1": 0.231991, + "nauc_mrr_at_20_max": 0.339639, + "nauc_mrr_at_20_std": 0.282878, + "nauc_mrr_at_20_diff1": 0.232085, + "nauc_mrr_at_100_max": 0.338944, + "nauc_mrr_at_100_std": 0.281272, + "nauc_mrr_at_100_diff1": 0.231312, + "nauc_mrr_at_1000_max": 0.338944, + "nauc_mrr_at_1000_std": 0.281272, + "nauc_mrr_at_1000_diff1": 0.231312, + "hit_rate_at_1": 0.23494, + "hit_rate_at_3": 0.43373, + "hit_rate_at_5": 0.5512, + "hit_rate_at_10": 0.68072, + "hit_rate_at_20": 0.81627, + "hit_rate_at_100": 0.91566, + "hit_rate_at_1000": 0.91566, + "main_score": 0.30894, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.15964, + "ndcg_at_3": 0.20265, + "ndcg_at_5": 0.22434, + "ndcg_at_10": 0.26628, + "ndcg_at_20": 0.31697, + "ndcg_at_100": 0.37379, + "ndcg_at_1000": 0.37379, + "map_at_1": 0.06978, + "map_at_3": 0.12548, + "map_at_5": 0.14982, + "map_at_10": 0.18055, + "map_at_20": 0.20407, + "map_at_100": 0.22218, + "map_at_1000": 0.22218, + "recall_at_1": 0.06978, + "recall_at_3": 0.1741, + "recall_at_5": 0.23796, + "recall_at_10": 0.36035, + "recall_at_20": 0.50406, + "recall_at_100": 0.69851, + "recall_at_1000": 0.69851, + "accuracy": 0.06978, + "precision_at_1": 0.1747, + "precision_at_3": 0.16365, + "precision_at_5": 0.1494, + "precision_at_10": 0.12108, + "precision_at_20": 0.09096, + "precision_at_100": 0.02687, + "precision_at_1000": 0.00269, + "mrr_at_1": 0.174699, + "mrr_at_3": 0.261546, + "mrr_at_5": 0.284588, + "mrr_at_10": 0.302903, + "mrr_at_20": 0.311559, + "mrr_at_100": 0.315786, + "mrr_at_1000": 0.315786, + "nauc_ndcg_at_1_max": 0.399603, + "nauc_ndcg_at_1_std": 0.288804, + "nauc_ndcg_at_1_diff1": 0.416176, + "nauc_ndcg_at_3_max": 0.455701, + "nauc_ndcg_at_3_std": 0.385207, + "nauc_ndcg_at_3_diff1": 0.381033, + "nauc_ndcg_at_5_max": 0.458757, + "nauc_ndcg_at_5_std": 0.396485, + "nauc_ndcg_at_5_diff1": 0.371418, + "nauc_ndcg_at_10_max": 0.48107, + "nauc_ndcg_at_10_std": 0.445403, + "nauc_ndcg_at_10_diff1": 0.383853, + "nauc_ndcg_at_20_max": 0.475915, + "nauc_ndcg_at_20_std": 0.456579, + "nauc_ndcg_at_20_diff1": 0.369463, + "nauc_ndcg_at_100_max": 0.496113, + "nauc_ndcg_at_100_std": 0.46503, + "nauc_ndcg_at_100_diff1": 0.37452, + "nauc_ndcg_at_1000_max": 0.496113, + "nauc_ndcg_at_1000_std": 0.46503, + "nauc_ndcg_at_1000_diff1": 0.37452, + "nauc_map_at_1_max": 0.459366, + "nauc_map_at_1_std": 0.284587, + "nauc_map_at_1_diff1": 0.565646, + "nauc_map_at_3_max": 0.493461, + "nauc_map_at_3_std": 0.382288, + "nauc_map_at_3_diff1": 0.451783, + "nauc_map_at_5_max": 0.496855, + "nauc_map_at_5_std": 0.401108, + "nauc_map_at_5_diff1": 0.429767, + "nauc_map_at_10_max": 0.496369, + "nauc_map_at_10_std": 0.42666, + "nauc_map_at_10_diff1": 0.419179, + "nauc_map_at_20_max": 0.495258, + "nauc_map_at_20_std": 0.434427, + "nauc_map_at_20_diff1": 0.406271, + "nauc_map_at_100_max": 0.496577, + "nauc_map_at_100_std": 0.433733, + "nauc_map_at_100_diff1": 0.401279, + "nauc_map_at_1000_max": 0.496577, + "nauc_map_at_1000_std": 0.433733, + "nauc_map_at_1000_diff1": 0.401279, + "nauc_recall_at_1_max": 0.459366, + "nauc_recall_at_1_std": 0.284587, + "nauc_recall_at_1_diff1": 0.565646, + "nauc_recall_at_3_max": 0.44342, + "nauc_recall_at_3_std": 0.380173, + "nauc_recall_at_3_diff1": 0.339782, + "nauc_recall_at_5_max": 0.435205, + "nauc_recall_at_5_std": 0.390119, + "nauc_recall_at_5_diff1": 0.291728, + "nauc_recall_at_10_max": 0.438568, + "nauc_recall_at_10_std": 0.451111, + "nauc_recall_at_10_diff1": 0.282992, + "nauc_recall_at_20_max": 0.402705, + "nauc_recall_at_20_std": 0.454633, + "nauc_recall_at_20_diff1": 0.238902, + "nauc_recall_at_100_max": 0.472748, + "nauc_recall_at_100_std": 0.500972, + "nauc_recall_at_100_diff1": 0.25017, + "nauc_recall_at_1000_max": 0.472748, + "nauc_recall_at_1000_std": 0.500972, + "nauc_recall_at_1000_diff1": 0.25017, + "nauc_precision_at_1_max": 0.407831, + "nauc_precision_at_1_std": 0.320889, + "nauc_precision_at_1_diff1": 0.395945, + "nauc_precision_at_3_max": 0.42796, + "nauc_precision_at_3_std": 0.425428, + "nauc_precision_at_3_diff1": 0.290317, + "nauc_precision_at_5_max": 0.377418, + "nauc_precision_at_5_std": 0.398979, + "nauc_precision_at_5_diff1": 0.246277, + "nauc_precision_at_10_max": 0.320946, + "nauc_precision_at_10_std": 0.374768, + "nauc_precision_at_10_diff1": 0.225837, + "nauc_precision_at_20_max": 0.216011, + "nauc_precision_at_20_std": 0.272688, + "nauc_precision_at_20_diff1": 0.127604, + "nauc_precision_at_100_max": 0.134132, + "nauc_precision_at_100_std": 0.16131, + "nauc_precision_at_100_diff1": 0.049801, + "nauc_precision_at_1000_max": 0.134132, + "nauc_precision_at_1000_std": 0.16131, + "nauc_precision_at_1000_diff1": 0.049801, + "nauc_mrr_at_1_max": 0.407831, + "nauc_mrr_at_1_std": 0.320889, + "nauc_mrr_at_1_diff1": 0.395945, + "nauc_mrr_at_3_max": 0.440373, + "nauc_mrr_at_3_std": 0.392784, + "nauc_mrr_at_3_diff1": 0.372202, + "nauc_mrr_at_5_max": 0.438065, + "nauc_mrr_at_5_std": 0.394658, + "nauc_mrr_at_5_diff1": 0.356092, + "nauc_mrr_at_10_max": 0.443733, + "nauc_mrr_at_10_std": 0.402901, + "nauc_mrr_at_10_diff1": 0.369997, + "nauc_mrr_at_20_max": 0.446182, + "nauc_mrr_at_20_std": 0.406112, + "nauc_mrr_at_20_diff1": 0.37167, + "nauc_mrr_at_100_max": 0.445776, + "nauc_mrr_at_100_std": 0.404309, + "nauc_mrr_at_100_diff1": 0.372141, + "nauc_mrr_at_1000_max": 0.445776, + "nauc_mrr_at_1000_std": 0.404309, + "nauc_mrr_at_1000_diff1": 0.372141, + "hit_rate_at_1": 0.1747, + "hit_rate_at_3": 0.37349, + "hit_rate_at_5": 0.4759, + "hit_rate_at_10": 0.61145, + "hit_rate_at_20": 0.73795, + "hit_rate_at_100": 0.86446, + "hit_rate_at_1000": 0.86446, + "main_score": 0.26628, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.19729, + "ndcg_at_3": 0.25171, + "ndcg_at_5": 0.27133, + "ndcg_at_10": 0.30725, + "ndcg_at_20": 0.35889, + "ndcg_at_100": 0.41233, + "ndcg_at_1000": 0.41233, + "map_at_1": 0.08461, + "map_at_3": 0.15096, + "map_at_5": 0.18054, + "map_at_10": 0.21063, + "map_at_20": 0.23708, + "map_at_100": 0.25457, + "map_at_1000": 0.25457, + "recall_at_1": 0.08461, + "recall_at_3": 0.20644, + "recall_at_5": 0.27955, + "recall_at_10": 0.39668, + "recall_at_20": 0.54321, + "recall_at_100": 0.71825, + "recall_at_1000": 0.71825, + "accuracy": 0.08461, + "precision_at_1": 0.21386, + "precision_at_3": 0.20382, + "precision_at_5": 0.17711, + "precision_at_10": 0.13584, + "precision_at_20": 0.10151, + "precision_at_100": 0.02877, + "precision_at_1000": 0.00288, + "mrr_at_1": 0.213855, + "mrr_at_3": 0.322289, + "mrr_at_5": 0.343976, + "mrr_at_10": 0.362016, + "mrr_at_20": 0.368956, + "mrr_at_100": 0.372545, + "mrr_at_1000": 0.372545, + "nauc_ndcg_at_1_max": 0.225169, + "nauc_ndcg_at_1_std": 0.183621, + "nauc_ndcg_at_1_diff1": 0.156809, + "nauc_ndcg_at_3_max": 0.362883, + "nauc_ndcg_at_3_std": 0.321908, + "nauc_ndcg_at_3_diff1": 0.257529, + "nauc_ndcg_at_5_max": 0.385001, + "nauc_ndcg_at_5_std": 0.337053, + "nauc_ndcg_at_5_diff1": 0.254419, + "nauc_ndcg_at_10_max": 0.395645, + "nauc_ndcg_at_10_std": 0.377538, + "nauc_ndcg_at_10_diff1": 0.244387, + "nauc_ndcg_at_20_max": 0.414259, + "nauc_ndcg_at_20_std": 0.417353, + "nauc_ndcg_at_20_diff1": 0.242858, + "nauc_ndcg_at_100_max": 0.418491, + "nauc_ndcg_at_100_std": 0.399927, + "nauc_ndcg_at_100_diff1": 0.241335, + "nauc_ndcg_at_1000_max": 0.418491, + "nauc_ndcg_at_1000_std": 0.399927, + "nauc_ndcg_at_1000_diff1": 0.241335, + "nauc_map_at_1_max": 0.299124, + "nauc_map_at_1_std": 0.201968, + "nauc_map_at_1_diff1": 0.258785, + "nauc_map_at_3_max": 0.36995, + "nauc_map_at_3_std": 0.303002, + "nauc_map_at_3_diff1": 0.294151, + "nauc_map_at_5_max": 0.399632, + "nauc_map_at_5_std": 0.337723, + "nauc_map_at_5_diff1": 0.2859, + "nauc_map_at_10_max": 0.406168, + "nauc_map_at_10_std": 0.369253, + "nauc_map_at_10_diff1": 0.270075, + "nauc_map_at_20_max": 0.416766, + "nauc_map_at_20_std": 0.388691, + "nauc_map_at_20_diff1": 0.271333, + "nauc_map_at_100_max": 0.419672, + "nauc_map_at_100_std": 0.385704, + "nauc_map_at_100_diff1": 0.27102, + "nauc_map_at_1000_max": 0.419672, + "nauc_map_at_1000_std": 0.385704, + "nauc_map_at_1000_diff1": 0.27102, + "nauc_recall_at_1_max": 0.299124, + "nauc_recall_at_1_std": 0.201968, + "nauc_recall_at_1_diff1": 0.258785, + "nauc_recall_at_3_max": 0.363625, + "nauc_recall_at_3_std": 0.311697, + "nauc_recall_at_3_diff1": 0.290404, + "nauc_recall_at_5_max": 0.393651, + "nauc_recall_at_5_std": 0.337125, + "nauc_recall_at_5_diff1": 0.253543, + "nauc_recall_at_10_max": 0.401159, + "nauc_recall_at_10_std": 0.402234, + "nauc_recall_at_10_diff1": 0.216783, + "nauc_recall_at_20_max": 0.397031, + "nauc_recall_at_20_std": 0.453948, + "nauc_recall_at_20_diff1": 0.177097, + "nauc_recall_at_100_max": 0.436533, + "nauc_recall_at_100_std": 0.443836, + "nauc_recall_at_100_diff1": 0.16286, + "nauc_recall_at_1000_max": 0.436533, + "nauc_recall_at_1000_std": 0.443836, + "nauc_recall_at_1000_diff1": 0.16286, + "nauc_precision_at_1_max": 0.242985, + "nauc_precision_at_1_std": 0.225057, + "nauc_precision_at_1_diff1": 0.159243, + "nauc_precision_at_3_max": 0.367895, + "nauc_precision_at_3_std": 0.387978, + "nauc_precision_at_3_diff1": 0.25193, + "nauc_precision_at_5_max": 0.354358, + "nauc_precision_at_5_std": 0.382199, + "nauc_precision_at_5_diff1": 0.21282, + "nauc_precision_at_10_max": 0.295549, + "nauc_precision_at_10_std": 0.37479, + "nauc_precision_at_10_diff1": 0.135908, + "nauc_precision_at_20_max": 0.238204, + "nauc_precision_at_20_std": 0.310269, + "nauc_precision_at_20_diff1": 0.120981, + "nauc_precision_at_100_max": 0.144408, + "nauc_precision_at_100_std": 0.157602, + "nauc_precision_at_100_diff1": 0.06675, + "nauc_precision_at_1000_max": 0.144408, + "nauc_precision_at_1000_std": 0.157602, + "nauc_precision_at_1000_diff1": 0.06675, + "nauc_mrr_at_1_max": 0.242985, + "nauc_mrr_at_1_std": 0.225057, + "nauc_mrr_at_1_diff1": 0.159243, + "nauc_mrr_at_3_max": 0.325074, + "nauc_mrr_at_3_std": 0.309073, + "nauc_mrr_at_3_diff1": 0.224183, + "nauc_mrr_at_5_max": 0.326162, + "nauc_mrr_at_5_std": 0.311222, + "nauc_mrr_at_5_diff1": 0.214262, + "nauc_mrr_at_10_max": 0.333542, + "nauc_mrr_at_10_std": 0.325225, + "nauc_mrr_at_10_diff1": 0.210712, + "nauc_mrr_at_20_max": 0.333894, + "nauc_mrr_at_20_std": 0.326156, + "nauc_mrr_at_20_diff1": 0.20943, + "nauc_mrr_at_100_max": 0.332456, + "nauc_mrr_at_100_std": 0.32309, + "nauc_mrr_at_100_diff1": 0.209056, + "nauc_mrr_at_1000_max": 0.332456, + "nauc_mrr_at_1000_std": 0.32309, + "nauc_mrr_at_1000_diff1": 0.209056, + "hit_rate_at_1": 0.21386, + "hit_rate_at_3": 0.45783, + "hit_rate_at_5": 0.5512, + "hit_rate_at_10": 0.68675, + "hit_rate_at_20": 0.78313, + "hit_rate_at_100": 0.88855, + "hit_rate_at_1000": 0.88855, + "main_score": 0.30725, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.21386, + "ndcg_at_3": 0.2468, + "ndcg_at_5": 0.27206, + "ndcg_at_10": 0.30894, + "ndcg_at_20": 0.3619, + "ndcg_at_100": 0.41225, + "ndcg_at_1000": 0.41225, + "map_at_1": 0.09533, + "map_at_3": 0.15537, + "map_at_5": 0.18562, + "map_at_10": 0.21656, + "map_at_20": 0.24472, + "map_at_100": 0.26151, + "map_at_1000": 0.26151, + "recall_at_1": 0.09533, + "recall_at_3": 0.2016, + "recall_at_5": 0.27976, + "recall_at_10": 0.38798, + "recall_at_20": 0.54171, + "recall_at_100": 0.71041, + "recall_at_1000": 0.71041, + "accuracy": 0.09533, + "precision_at_1": 0.23193, + "precision_at_3": 0.19378, + "precision_at_5": 0.17349, + "precision_at_10": 0.13795, + "precision_at_20": 0.10346, + "precision_at_100": 0.02852, + "precision_at_1000": 0.00285, + "mrr_at_1": 0.231928, + "mrr_at_3": 0.320281, + "mrr_at_5": 0.347691, + "mrr_at_10": 0.362728, + "mrr_at_20": 0.371183, + "mrr_at_100": 0.374949, + "mrr_at_1000": 0.374949, + "nauc_ndcg_at_1_max": 0.235463, + "nauc_ndcg_at_1_std": 0.157271, + "nauc_ndcg_at_1_diff1": 0.238124, + "nauc_ndcg_at_3_max": 0.359666, + "nauc_ndcg_at_3_std": 0.284561, + "nauc_ndcg_at_3_diff1": 0.266902, + "nauc_ndcg_at_5_max": 0.413631, + "nauc_ndcg_at_5_std": 0.337521, + "nauc_ndcg_at_5_diff1": 0.251015, + "nauc_ndcg_at_10_max": 0.454127, + "nauc_ndcg_at_10_std": 0.397208, + "nauc_ndcg_at_10_diff1": 0.264021, + "nauc_ndcg_at_20_max": 0.46455, + "nauc_ndcg_at_20_std": 0.420049, + "nauc_ndcg_at_20_diff1": 0.273411, + "nauc_ndcg_at_100_max": 0.466724, + "nauc_ndcg_at_100_std": 0.408516, + "nauc_ndcg_at_100_diff1": 0.272228, + "nauc_ndcg_at_1000_max": 0.466724, + "nauc_ndcg_at_1000_std": 0.408516, + "nauc_ndcg_at_1000_diff1": 0.272228, + "nauc_map_at_1_max": 0.276373, + "nauc_map_at_1_std": 0.088885, + "nauc_map_at_1_diff1": 0.346812, + "nauc_map_at_3_max": 0.356423, + "nauc_map_at_3_std": 0.220222, + "nauc_map_at_3_diff1": 0.299841, + "nauc_map_at_5_max": 0.404956, + "nauc_map_at_5_std": 0.28505, + "nauc_map_at_5_diff1": 0.288002, + "nauc_map_at_10_max": 0.438345, + "nauc_map_at_10_std": 0.344349, + "nauc_map_at_10_diff1": 0.297597, + "nauc_map_at_20_max": 0.446532, + "nauc_map_at_20_std": 0.365785, + "nauc_map_at_20_diff1": 0.30234, + "nauc_map_at_100_max": 0.447607, + "nauc_map_at_100_std": 0.364747, + "nauc_map_at_100_diff1": 0.302049, + "nauc_map_at_1000_max": 0.447607, + "nauc_map_at_1000_std": 0.364747, + "nauc_map_at_1000_diff1": 0.302049, + "nauc_recall_at_1_max": 0.276373, + "nauc_recall_at_1_std": 0.088885, + "nauc_recall_at_1_diff1": 0.346812, + "nauc_recall_at_3_max": 0.35902, + "nauc_recall_at_3_std": 0.253152, + "nauc_recall_at_3_diff1": 0.245688, + "nauc_recall_at_5_max": 0.43532, + "nauc_recall_at_5_std": 0.354621, + "nauc_recall_at_5_diff1": 0.194923, + "nauc_recall_at_10_max": 0.492415, + "nauc_recall_at_10_std": 0.466521, + "nauc_recall_at_10_diff1": 0.204462, + "nauc_recall_at_20_max": 0.488369, + "nauc_recall_at_20_std": 0.499939, + "nauc_recall_at_20_diff1": 0.195363, + "nauc_recall_at_100_max": 0.522227, + "nauc_recall_at_100_std": 0.503489, + "nauc_recall_at_100_diff1": 0.174522, + "nauc_recall_at_1000_max": 0.522227, + "nauc_recall_at_1000_std": 0.503489, + "nauc_recall_at_1000_diff1": 0.174522, + "nauc_precision_at_1_max": 0.235293, + "nauc_precision_at_1_std": 0.175941, + "nauc_precision_at_1_diff1": 0.233401, + "nauc_precision_at_3_max": 0.369002, + "nauc_precision_at_3_std": 0.369278, + "nauc_precision_at_3_diff1": 0.227778, + "nauc_precision_at_5_max": 0.40476, + "nauc_precision_at_5_std": 0.436743, + "nauc_precision_at_5_diff1": 0.167968, + "nauc_precision_at_10_max": 0.362002, + "nauc_precision_at_10_std": 0.440062, + "nauc_precision_at_10_diff1": 0.163632, + "nauc_precision_at_20_max": 0.281788, + "nauc_precision_at_20_std": 0.360831, + "nauc_precision_at_20_diff1": 0.141182, + "nauc_precision_at_100_max": 0.198221, + "nauc_precision_at_100_std": 0.240634, + "nauc_precision_at_100_diff1": 0.090999, + "nauc_precision_at_1000_max": 0.198221, + "nauc_precision_at_1000_std": 0.240634, + "nauc_precision_at_1000_diff1": 0.090999, + "nauc_mrr_at_1_max": 0.235293, + "nauc_mrr_at_1_std": 0.175941, + "nauc_mrr_at_1_diff1": 0.233401, + "nauc_mrr_at_3_max": 0.306191, + "nauc_mrr_at_3_std": 0.258367, + "nauc_mrr_at_3_diff1": 0.26056, + "nauc_mrr_at_5_max": 0.333445, + "nauc_mrr_at_5_std": 0.292087, + "nauc_mrr_at_5_diff1": 0.242862, + "nauc_mrr_at_10_max": 0.333591, + "nauc_mrr_at_10_std": 0.294168, + "nauc_mrr_at_10_diff1": 0.247533, + "nauc_mrr_at_20_max": 0.333528, + "nauc_mrr_at_20_std": 0.294156, + "nauc_mrr_at_20_diff1": 0.247616, + "nauc_mrr_at_100_max": 0.331806, + "nauc_mrr_at_100_std": 0.291683, + "nauc_mrr_at_100_diff1": 0.247188, + "nauc_mrr_at_1000_max": 0.331806, + "nauc_mrr_at_1000_std": 0.291683, + "nauc_mrr_at_1000_diff1": 0.247188, + "hit_rate_at_1": 0.23193, + "hit_rate_at_3": 0.42771, + "hit_rate_at_5": 0.54518, + "hit_rate_at_10": 0.65964, + "hit_rate_at_20": 0.77711, + "hit_rate_at_100": 0.88855, + "hit_rate_at_1000": 0.88855, + "main_score": 0.30894, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.21988, + "ndcg_at_3": 0.2417, + "ndcg_at_5": 0.26968, + "ndcg_at_10": 0.30621, + "ndcg_at_20": 0.35972, + "ndcg_at_100": 0.42252, + "ndcg_at_1000": 0.42252, + "map_at_1": 0.09622, + "map_at_3": 0.14941, + "map_at_5": 0.18035, + "map_at_10": 0.2118, + "map_at_20": 0.23885, + "map_at_100": 0.25966, + "map_at_1000": 0.25966, + "recall_at_1": 0.09622, + "recall_at_3": 0.19742, + "recall_at_5": 0.2749, + "recall_at_10": 0.39266, + "recall_at_20": 0.54183, + "recall_at_100": 0.74633, + "recall_at_1000": 0.74633, + "accuracy": 0.09622, + "precision_at_1": 0.23795, + "precision_at_3": 0.18373, + "precision_at_5": 0.1741, + "precision_at_10": 0.13464, + "precision_at_20": 0.10256, + "precision_at_100": 0.03045, + "precision_at_1000": 0.00305, + "mrr_at_1": 0.237952, + "mrr_at_3": 0.326305, + "mrr_at_5": 0.353564, + "mrr_at_10": 0.36916, + "mrr_at_20": 0.378523, + "mrr_at_100": 0.382279, + "mrr_at_1000": 0.382279, + "nauc_ndcg_at_1_max": 0.235258, + "nauc_ndcg_at_1_std": 0.189115, + "nauc_ndcg_at_1_diff1": 0.217353, + "nauc_ndcg_at_3_max": 0.329561, + "nauc_ndcg_at_3_std": 0.270361, + "nauc_ndcg_at_3_diff1": 0.218302, + "nauc_ndcg_at_5_max": 0.353794, + "nauc_ndcg_at_5_std": 0.313478, + "nauc_ndcg_at_5_diff1": 0.187465, + "nauc_ndcg_at_10_max": 0.407094, + "nauc_ndcg_at_10_std": 0.387234, + "nauc_ndcg_at_10_diff1": 0.177509, + "nauc_ndcg_at_20_max": 0.403432, + "nauc_ndcg_at_20_std": 0.397907, + "nauc_ndcg_at_20_diff1": 0.191066, + "nauc_ndcg_at_100_max": 0.3857, + "nauc_ndcg_at_100_std": 0.366511, + "nauc_ndcg_at_100_diff1": 0.190742, + "nauc_ndcg_at_1000_max": 0.3857, + "nauc_ndcg_at_1000_std": 0.366511, + "nauc_ndcg_at_1000_diff1": 0.190742, + "nauc_map_at_1_max": 0.27822, + "nauc_map_at_1_std": 0.139153, + "nauc_map_at_1_diff1": 0.245249, + "nauc_map_at_3_max": 0.349706, + "nauc_map_at_3_std": 0.246997, + "nauc_map_at_3_diff1": 0.186711, + "nauc_map_at_5_max": 0.379886, + "nauc_map_at_5_std": 0.298031, + "nauc_map_at_5_diff1": 0.18194, + "nauc_map_at_10_max": 0.400086, + "nauc_map_at_10_std": 0.337249, + "nauc_map_at_10_diff1": 0.18957, + "nauc_map_at_20_max": 0.401384, + "nauc_map_at_20_std": 0.351192, + "nauc_map_at_20_diff1": 0.197449, + "nauc_map_at_100_max": 0.397041, + "nauc_map_at_100_std": 0.343696, + "nauc_map_at_100_diff1": 0.201071, + "nauc_map_at_1000_max": 0.397041, + "nauc_map_at_1000_std": 0.343696, + "nauc_map_at_1000_diff1": 0.201071, + "nauc_recall_at_1_max": 0.27822, + "nauc_recall_at_1_std": 0.139153, + "nauc_recall_at_1_diff1": 0.245249, + "nauc_recall_at_3_max": 0.363627, + "nauc_recall_at_3_std": 0.290214, + "nauc_recall_at_3_diff1": 0.142982, + "nauc_recall_at_5_max": 0.384388, + "nauc_recall_at_5_std": 0.349085, + "nauc_recall_at_5_diff1": 0.120317, + "nauc_recall_at_10_max": 0.451278, + "nauc_recall_at_10_std": 0.462244, + "nauc_recall_at_10_diff1": 0.113458, + "nauc_recall_at_20_max": 0.402015, + "nauc_recall_at_20_std": 0.452289, + "nauc_recall_at_20_diff1": 0.119207, + "nauc_recall_at_100_max": 0.375163, + "nauc_recall_at_100_std": 0.391481, + "nauc_recall_at_100_diff1": 0.117196, + "nauc_recall_at_1000_max": 0.375163, + "nauc_recall_at_1000_std": 0.391481, + "nauc_recall_at_1000_diff1": 0.117196, + "nauc_precision_at_1_max": 0.248175, + "nauc_precision_at_1_std": 0.212392, + "nauc_precision_at_1_diff1": 0.216917, + "nauc_precision_at_3_max": 0.357452, + "nauc_precision_at_3_std": 0.352185, + "nauc_precision_at_3_diff1": 0.244819, + "nauc_precision_at_5_max": 0.323828, + "nauc_precision_at_5_std": 0.366477, + "nauc_precision_at_5_diff1": 0.174787, + "nauc_precision_at_10_max": 0.297014, + "nauc_precision_at_10_std": 0.374907, + "nauc_precision_at_10_diff1": 0.125591, + "nauc_precision_at_20_max": 0.198945, + "nauc_precision_at_20_std": 0.282473, + "nauc_precision_at_20_diff1": 0.122494, + "nauc_precision_at_100_max": 0.056685, + "nauc_precision_at_100_std": 0.098126, + "nauc_precision_at_100_diff1": 0.059712, + "nauc_precision_at_1000_max": 0.056685, + "nauc_precision_at_1000_std": 0.098126, + "nauc_precision_at_1000_diff1": 0.059712, + "nauc_mrr_at_1_max": 0.248175, + "nauc_mrr_at_1_std": 0.212392, + "nauc_mrr_at_1_diff1": 0.216917, + "nauc_mrr_at_3_max": 0.313458, + "nauc_mrr_at_3_std": 0.285492, + "nauc_mrr_at_3_diff1": 0.253391, + "nauc_mrr_at_5_max": 0.3046, + "nauc_mrr_at_5_std": 0.289822, + "nauc_mrr_at_5_diff1": 0.228705, + "nauc_mrr_at_10_max": 0.316709, + "nauc_mrr_at_10_std": 0.30574, + "nauc_mrr_at_10_diff1": 0.221669, + "nauc_mrr_at_20_max": 0.310848, + "nauc_mrr_at_20_std": 0.299861, + "nauc_mrr_at_20_diff1": 0.223267, + "nauc_mrr_at_100_max": 0.309524, + "nauc_mrr_at_100_std": 0.297381, + "nauc_mrr_at_100_diff1": 0.222748, + "nauc_mrr_at_1000_max": 0.309524, + "nauc_mrr_at_1000_std": 0.297381, + "nauc_mrr_at_1000_diff1": 0.222748, + "hit_rate_at_1": 0.23795, + "hit_rate_at_3": 0.44277, + "hit_rate_at_5": 0.56024, + "hit_rate_at_10": 0.68072, + "hit_rate_at_20": 0.80723, + "hit_rate_at_100": 0.92169, + "hit_rate_at_1000": 0.92169, + "main_score": 0.30621, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 723.492144736927, + "kg_co2_emissions": null, + "date": 1792440666.969728, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 4.6869972720742226e-06, + "end": 32.35965269699227, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 32.35966285201721, + "end": 32.359669552068226, + "split": "", + "subset": "" + }, + { + "name": "Encoding corpus", + "start": 32.363692008075304, + "end": 146.76688031305093, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 146.76695311209187, + "end": 146.95391105709132, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 146.9539287830703, + "end": 147.00007029308472, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding corpus", + "start": 147.0117595520569, + "end": 260.2620257040253, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 260.262103300076, + "end": 260.53642710403074, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 260.53644707310013, + "end": 260.596622192068, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding corpus", + "start": 260.61430883104913, + "end": 377.6446298430674, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 377.64470844110474, + "end": 377.84724931500386, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 377.8472698860569, + "end": 378.36741638404783, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding corpus", + "start": 378.38339400908444, + "end": 494.12879415706266, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 494.12887152307667, + "end": 494.38251450506505, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 494.3825366080273, + "end": 494.4267415840877, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding corpus", + "start": 494.44057096506003, + "end": 608.3111805970548, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 608.311236210051, + "end": 608.4982320159907, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 608.4982574840542, + "end": 608.5424416220048, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding corpus", + "start": 608.5555219430244, + "end": 723.249316239031, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Encoding queries", + "start": 723.2493759189965, + "end": 723.4387775650248, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 723.4387941299938, + "end": 723.4824842400849, + "split": "test", + "subset": "portuguese" + } + ] +} diff --git a/results/mteb__baseline-bm25s/0_3_0/Vidore3TelecomNemotron35Embed8BReranking.v2.json b/results/mteb__baseline-bm25s/0_3_0/Vidore3TelecomNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..d1415277b4 --- /dev/null +++ b/results/mteb__baseline-bm25s/0_3_0/Vidore3TelecomNemotron35Embed8BReranking.v2.json @@ -0,0 +1,1090 @@ +{ + "dataset_revision": "47c9b703527792f3fbf0a082f4589ff40c1aa8cc", + "task_name": "Vidore3TelecomNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.56897, + "ndcg_at_3": 0.55848, + "ndcg_at_5": 0.57086, + "ndcg_at_10": 0.6109, + "ndcg_at_20": 0.6509, + "ndcg_at_100": 0.6923, + "ndcg_at_1000": 0.6923, + "map_at_1": 0.25184, + "map_at_3": 0.39074, + "map_at_5": 0.43769, + "map_at_10": 0.48607, + "map_at_20": 0.51567, + "map_at_100": 0.53573, + "map_at_1000": 0.53573, + "recall_at_1": 0.25184, + "recall_at_3": 0.45673, + "recall_at_5": 0.54784, + "recall_at_10": 0.68257, + "recall_at_20": 0.80338, + "recall_at_100": 0.92358, + "recall_at_1000": 0.92358, + "accuracy": 0.25184, + "precision_at_1": 0.59483, + "precision_at_3": 0.41571, + "precision_at_5": 0.32701, + "precision_at_10": 0.23075, + "precision_at_20": 0.15, + "precision_at_100": 0.03874, + "precision_at_1000": 0.00387, + "mrr_at_1": 0.594828, + "mrr_at_3": 0.683429, + "mrr_at_5": 0.69636, + "mrr_at_10": 0.707361, + "mrr_at_20": 0.710001, + "mrr_at_100": 0.710475, + "mrr_at_1000": 0.710475, + "nauc_ndcg_at_1_max": 0.329083, + "nauc_ndcg_at_1_std": 0.335254, + "nauc_ndcg_at_1_diff1": 0.407617, + "nauc_ndcg_at_3_max": 0.326869, + "nauc_ndcg_at_3_std": 0.315337, + "nauc_ndcg_at_3_diff1": 0.359377, + "nauc_ndcg_at_5_max": 0.34251, + "nauc_ndcg_at_5_std": 0.332473, + "nauc_ndcg_at_5_diff1": 0.379056, + "nauc_ndcg_at_10_max": 0.341278, + "nauc_ndcg_at_10_std": 0.337967, + "nauc_ndcg_at_10_diff1": 0.355842, + "nauc_ndcg_at_20_max": 0.373705, + "nauc_ndcg_at_20_std": 0.374451, + "nauc_ndcg_at_20_diff1": 0.373233, + "nauc_ndcg_at_100_max": 0.387524, + "nauc_ndcg_at_100_std": 0.381508, + "nauc_ndcg_at_100_diff1": 0.377016, + "nauc_ndcg_at_1000_max": 0.387524, + "nauc_ndcg_at_1000_std": 0.381508, + "nauc_ndcg_at_1000_diff1": 0.377016, + "nauc_map_at_1_max": 0.226585, + "nauc_map_at_1_std": 0.161802, + "nauc_map_at_1_diff1": 0.434097, + "nauc_map_at_3_max": 0.283309, + "nauc_map_at_3_std": 0.245793, + "nauc_map_at_3_diff1": 0.380692, + "nauc_map_at_5_max": 0.303928, + "nauc_map_at_5_std": 0.286345, + "nauc_map_at_5_diff1": 0.365071, + "nauc_map_at_10_max": 0.303706, + "nauc_map_at_10_std": 0.299508, + "nauc_map_at_10_diff1": 0.339548, + "nauc_map_at_20_max": 0.32343, + "nauc_map_at_20_std": 0.323007, + "nauc_map_at_20_diff1": 0.345565, + "nauc_map_at_100_max": 0.329048, + "nauc_map_at_100_std": 0.326, + "nauc_map_at_100_diff1": 0.344386, + "nauc_map_at_1000_max": 0.329048, + "nauc_map_at_1000_std": 0.326, + "nauc_map_at_1000_diff1": 0.344386, + "nauc_recall_at_1_max": 0.226585, + "nauc_recall_at_1_std": 0.161802, + "nauc_recall_at_1_diff1": 0.434097, + "nauc_recall_at_3_max": 0.26836, + "nauc_recall_at_3_std": 0.223992, + "nauc_recall_at_3_diff1": 0.341086, + "nauc_recall_at_5_max": 0.295092, + "nauc_recall_at_5_std": 0.27448, + "nauc_recall_at_5_diff1": 0.316179, + "nauc_recall_at_10_max": 0.268918, + "nauc_recall_at_10_std": 0.291042, + "nauc_recall_at_10_diff1": 0.224131, + "nauc_recall_at_20_max": 0.375883, + "nauc_recall_at_20_std": 0.413587, + "nauc_recall_at_20_diff1": 0.266349, + "nauc_recall_at_100_max": 0.598257, + "nauc_recall_at_100_std": 0.629503, + "nauc_recall_at_100_diff1": 0.261494, + "nauc_recall_at_1000_max": 0.598257, + "nauc_recall_at_1000_std": 0.629503, + "nauc_recall_at_1000_diff1": 0.261494, + "nauc_precision_at_1_max": 0.316188, + "nauc_precision_at_1_std": 0.32664, + "nauc_precision_at_1_diff1": 0.419474, + "nauc_precision_at_3_max": 0.260663, + "nauc_precision_at_3_std": 0.295712, + "nauc_precision_at_3_diff1": 0.132371, + "nauc_precision_at_5_max": 0.200423, + "nauc_precision_at_5_std": 0.266973, + "nauc_precision_at_5_diff1": 0.022021, + "nauc_precision_at_10_max": 0.072609, + "nauc_precision_at_10_std": 0.150865, + "nauc_precision_at_10_diff1": -0.121002, + "nauc_precision_at_20_max": 0.039908, + "nauc_precision_at_20_std": 0.101843, + "nauc_precision_at_20_diff1": -0.148009, + "nauc_precision_at_100_max": -0.039189, + "nauc_precision_at_100_std": -0.009765, + "nauc_precision_at_100_diff1": -0.178882, + "nauc_precision_at_1000_max": -0.039189, + "nauc_precision_at_1000_std": -0.009765, + "nauc_precision_at_1000_diff1": -0.178882, + "nauc_mrr_at_1_max": 0.316188, + "nauc_mrr_at_1_std": 0.32664, + "nauc_mrr_at_1_diff1": 0.419474, + "nauc_mrr_at_3_max": 0.335719, + "nauc_mrr_at_3_std": 0.31938, + "nauc_mrr_at_3_diff1": 0.425034, + "nauc_mrr_at_5_max": 0.346632, + "nauc_mrr_at_5_std": 0.32614, + "nauc_mrr_at_5_diff1": 0.437954, + "nauc_mrr_at_10_max": 0.341714, + "nauc_mrr_at_10_std": 0.328468, + "nauc_mrr_at_10_diff1": 0.433486, + "nauc_mrr_at_20_max": 0.33943, + "nauc_mrr_at_20_std": 0.326035, + "nauc_mrr_at_20_diff1": 0.432322, + "nauc_mrr_at_100_max": 0.33946, + "nauc_mrr_at_100_std": 0.326083, + "nauc_mrr_at_100_diff1": 0.432131, + "nauc_mrr_at_1000_max": 0.33946, + "nauc_mrr_at_1000_std": 0.326083, + "nauc_mrr_at_1000_diff1": 0.432131, + "hit_rate_at_1": 0.59483, + "hit_rate_at_3": 0.8046, + "hit_rate_at_5": 0.86207, + "hit_rate_at_10": 0.93966, + "hit_rate_at_20": 0.97989, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.6109, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.38218, + "ndcg_at_3": 0.3983, + "ndcg_at_5": 0.41867, + "ndcg_at_10": 0.46374, + "ndcg_at_20": 0.5112, + "ndcg_at_100": 0.57126, + "ndcg_at_1000": 0.57126, + "map_at_1": 0.16445, + "map_at_3": 0.27688, + "map_at_5": 0.31336, + "map_at_10": 0.35505, + "map_at_20": 0.3832, + "map_at_100": 0.40852, + "map_at_1000": 0.40852, + "recall_at_1": 0.16445, + "recall_at_3": 0.33444, + "recall_at_5": 0.42197, + "recall_at_10": 0.55039, + "recall_at_20": 0.68928, + "recall_at_100": 0.87637, + "recall_at_1000": 0.87637, + "accuracy": 0.16445, + "precision_at_1": 0.40805, + "precision_at_3": 0.30747, + "precision_at_5": 0.24713, + "precision_at_10": 0.1842, + "precision_at_20": 0.1283, + "precision_at_100": 0.03647, + "precision_at_1000": 0.00365, + "mrr_at_1": 0.408046, + "mrr_at_3": 0.508142, + "mrr_at_5": 0.529981, + "mrr_at_10": 0.544444, + "mrr_at_20": 0.549934, + "mrr_at_100": 0.551903, + "mrr_at_1000": 0.551903, + "nauc_ndcg_at_1_max": 0.313952, + "nauc_ndcg_at_1_std": 0.268903, + "nauc_ndcg_at_1_diff1": 0.352322, + "nauc_ndcg_at_3_max": 0.33201, + "nauc_ndcg_at_3_std": 0.330887, + "nauc_ndcg_at_3_diff1": 0.294792, + "nauc_ndcg_at_5_max": 0.404812, + "nauc_ndcg_at_5_std": 0.390674, + "nauc_ndcg_at_5_diff1": 0.319391, + "nauc_ndcg_at_10_max": 0.434343, + "nauc_ndcg_at_10_std": 0.417762, + "nauc_ndcg_at_10_diff1": 0.320572, + "nauc_ndcg_at_20_max": 0.447014, + "nauc_ndcg_at_20_std": 0.433074, + "nauc_ndcg_at_20_diff1": 0.31488, + "nauc_ndcg_at_100_max": 0.434798, + "nauc_ndcg_at_100_std": 0.405015, + "nauc_ndcg_at_100_diff1": 0.313294, + "nauc_ndcg_at_1000_max": 0.434798, + "nauc_ndcg_at_1000_std": 0.405015, + "nauc_ndcg_at_1000_diff1": 0.313294, + "nauc_map_at_1_max": 0.265117, + "nauc_map_at_1_std": 0.204638, + "nauc_map_at_1_diff1": 0.381324, + "nauc_map_at_3_max": 0.304681, + "nauc_map_at_3_std": 0.296682, + "nauc_map_at_3_diff1": 0.293374, + "nauc_map_at_5_max": 0.363593, + "nauc_map_at_5_std": 0.3522, + "nauc_map_at_5_diff1": 0.304389, + "nauc_map_at_10_max": 0.397092, + "nauc_map_at_10_std": 0.387971, + "nauc_map_at_10_diff1": 0.306103, + "nauc_map_at_20_max": 0.40526, + "nauc_map_at_20_std": 0.399302, + "nauc_map_at_20_diff1": 0.301576, + "nauc_map_at_100_max": 0.398412, + "nauc_map_at_100_std": 0.38966, + "nauc_map_at_100_diff1": 0.30067, + "nauc_map_at_1000_max": 0.398412, + "nauc_map_at_1000_std": 0.38966, + "nauc_map_at_1000_diff1": 0.30067, + "nauc_recall_at_1_max": 0.265117, + "nauc_recall_at_1_std": 0.204638, + "nauc_recall_at_1_diff1": 0.381324, + "nauc_recall_at_3_max": 0.282873, + "nauc_recall_at_3_std": 0.28776, + "nauc_recall_at_3_diff1": 0.237812, + "nauc_recall_at_5_max": 0.406508, + "nauc_recall_at_5_std": 0.393177, + "nauc_recall_at_5_diff1": 0.256114, + "nauc_recall_at_10_max": 0.459864, + "nauc_recall_at_10_std": 0.448736, + "nauc_recall_at_10_diff1": 0.256884, + "nauc_recall_at_20_max": 0.491327, + "nauc_recall_at_20_std": 0.505568, + "nauc_recall_at_20_diff1": 0.21618, + "nauc_recall_at_100_max": 0.564775, + "nauc_recall_at_100_std": 0.495792, + "nauc_recall_at_100_diff1": 0.142047, + "nauc_recall_at_1000_max": 0.564775, + "nauc_recall_at_1000_std": 0.495792, + "nauc_recall_at_1000_diff1": 0.142047, + "nauc_precision_at_1_max": 0.310212, + "nauc_precision_at_1_std": 0.270494, + "nauc_precision_at_1_diff1": 0.364802, + "nauc_precision_at_3_max": 0.331516, + "nauc_precision_at_3_std": 0.367482, + "nauc_precision_at_3_diff1": 0.188074, + "nauc_precision_at_5_max": 0.389574, + "nauc_precision_at_5_std": 0.4103, + "nauc_precision_at_5_diff1": 0.169907, + "nauc_precision_at_10_max": 0.292125, + "nauc_precision_at_10_std": 0.313602, + "nauc_precision_at_10_diff1": 0.11017, + "nauc_precision_at_20_max": 0.160006, + "nauc_precision_at_20_std": 0.176964, + "nauc_precision_at_20_diff1": 0.028025, + "nauc_precision_at_100_max": -0.009361, + "nauc_precision_at_100_std": -0.016079, + "nauc_precision_at_100_diff1": -0.042468, + "nauc_precision_at_1000_max": -0.009361, + "nauc_precision_at_1000_std": -0.016079, + "nauc_precision_at_1000_diff1": -0.042468, + "nauc_mrr_at_1_max": 0.310212, + "nauc_mrr_at_1_std": 0.270494, + "nauc_mrr_at_1_diff1": 0.364802, + "nauc_mrr_at_3_max": 0.323549, + "nauc_mrr_at_3_std": 0.298098, + "nauc_mrr_at_3_diff1": 0.33298, + "nauc_mrr_at_5_max": 0.356113, + "nauc_mrr_at_5_std": 0.32487, + "nauc_mrr_at_5_diff1": 0.339526, + "nauc_mrr_at_10_max": 0.346812, + "nauc_mrr_at_10_std": 0.313255, + "nauc_mrr_at_10_diff1": 0.343882, + "nauc_mrr_at_20_max": 0.343765, + "nauc_mrr_at_20_std": 0.311339, + "nauc_mrr_at_20_diff1": 0.341841, + "nauc_mrr_at_100_max": 0.344135, + "nauc_mrr_at_100_std": 0.311153, + "nauc_mrr_at_100_diff1": 0.341718, + "nauc_mrr_at_1000_max": 0.344135, + "nauc_mrr_at_1000_std": 0.311153, + "nauc_mrr_at_1000_diff1": 0.341718, + "hit_rate_at_1": 0.40805, + "hit_rate_at_3": 0.62644, + "hit_rate_at_5": 0.71839, + "hit_rate_at_10": 0.82759, + "hit_rate_at_20": 0.91092, + "hit_rate_at_100": 0.95977, + "hit_rate_at_1000": 0.95977, + "main_score": 0.46374, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.3592, + "ndcg_at_3": 0.38391, + "ndcg_at_5": 0.41009, + "ndcg_at_10": 0.45447, + "ndcg_at_20": 0.50065, + "ndcg_at_100": 0.55846, + "ndcg_at_1000": 0.55846, + "map_at_1": 0.16262, + "map_at_3": 0.26431, + "map_at_5": 0.30454, + "map_at_10": 0.34777, + "map_at_20": 0.37569, + "map_at_100": 0.39956, + "map_at_1000": 0.39956, + "recall_at_1": 0.16262, + "recall_at_3": 0.32559, + "recall_at_5": 0.41432, + "recall_at_10": 0.54689, + "recall_at_20": 0.68274, + "recall_at_100": 0.86338, + "recall_at_1000": 0.86338, + "accuracy": 0.16262, + "precision_at_1": 0.38506, + "precision_at_3": 0.29789, + "precision_at_5": 0.24713, + "precision_at_10": 0.18391, + "precision_at_20": 0.12716, + "precision_at_100": 0.0358, + "precision_at_1000": 0.00358, + "mrr_at_1": 0.385057, + "mrr_at_3": 0.484674, + "mrr_at_5": 0.504646, + "mrr_at_10": 0.520007, + "mrr_at_20": 0.525615, + "mrr_at_100": 0.527603, + "mrr_at_1000": 0.527603, + "nauc_ndcg_at_1_max": 0.424635, + "nauc_ndcg_at_1_std": 0.375546, + "nauc_ndcg_at_1_diff1": 0.422637, + "nauc_ndcg_at_3_max": 0.440863, + "nauc_ndcg_at_3_std": 0.417609, + "nauc_ndcg_at_3_diff1": 0.431622, + "nauc_ndcg_at_5_max": 0.472697, + "nauc_ndcg_at_5_std": 0.449368, + "nauc_ndcg_at_5_diff1": 0.439671, + "nauc_ndcg_at_10_max": 0.491128, + "nauc_ndcg_at_10_std": 0.47028, + "nauc_ndcg_at_10_diff1": 0.438332, + "nauc_ndcg_at_20_max": 0.523062, + "nauc_ndcg_at_20_std": 0.50457, + "nauc_ndcg_at_20_diff1": 0.459694, + "nauc_ndcg_at_100_max": 0.522504, + "nauc_ndcg_at_100_std": 0.483988, + "nauc_ndcg_at_100_diff1": 0.458709, + "nauc_ndcg_at_1000_max": 0.522504, + "nauc_ndcg_at_1000_std": 0.483988, + "nauc_ndcg_at_1000_diff1": 0.458709, + "nauc_map_at_1_max": 0.334826, + "nauc_map_at_1_std": 0.298149, + "nauc_map_at_1_diff1": 0.470311, + "nauc_map_at_3_max": 0.398254, + "nauc_map_at_3_std": 0.390492, + "nauc_map_at_3_diff1": 0.445722, + "nauc_map_at_5_max": 0.431751, + "nauc_map_at_5_std": 0.425568, + "nauc_map_at_5_diff1": 0.439258, + "nauc_map_at_10_max": 0.449022, + "nauc_map_at_10_std": 0.447694, + "nauc_map_at_10_diff1": 0.427049, + "nauc_map_at_20_max": 0.466929, + "nauc_map_at_20_std": 0.465148, + "nauc_map_at_20_diff1": 0.435517, + "nauc_map_at_100_max": 0.463154, + "nauc_map_at_100_std": 0.45435, + "nauc_map_at_100_diff1": 0.434647, + "nauc_map_at_1000_max": 0.463154, + "nauc_map_at_1000_std": 0.45435, + "nauc_map_at_1000_diff1": 0.434647, + "nauc_recall_at_1_max": 0.334826, + "nauc_recall_at_1_std": 0.298149, + "nauc_recall_at_1_diff1": 0.470311, + "nauc_recall_at_3_max": 0.372889, + "nauc_recall_at_3_std": 0.372819, + "nauc_recall_at_3_diff1": 0.396232, + "nauc_recall_at_5_max": 0.43224, + "nauc_recall_at_5_std": 0.424952, + "nauc_recall_at_5_diff1": 0.374877, + "nauc_recall_at_10_max": 0.464728, + "nauc_recall_at_10_std": 0.463126, + "nauc_recall_at_10_diff1": 0.363046, + "nauc_recall_at_20_max": 0.561057, + "nauc_recall_at_20_std": 0.575575, + "nauc_recall_at_20_diff1": 0.423496, + "nauc_recall_at_100_max": 0.683247, + "nauc_recall_at_100_std": 0.631807, + "nauc_recall_at_100_diff1": 0.39685, + "nauc_recall_at_1000_max": 0.683247, + "nauc_recall_at_1000_std": 0.631807, + "nauc_recall_at_1000_diff1": 0.39685, + "nauc_precision_at_1_max": 0.413941, + "nauc_precision_at_1_std": 0.36767, + "nauc_precision_at_1_diff1": 0.415544, + "nauc_precision_at_3_max": 0.405375, + "nauc_precision_at_3_std": 0.407317, + "nauc_precision_at_3_diff1": 0.297283, + "nauc_precision_at_5_max": 0.39495, + "nauc_precision_at_5_std": 0.394466, + "nauc_precision_at_5_diff1": 0.23445, + "nauc_precision_at_10_max": 0.283081, + "nauc_precision_at_10_std": 0.280642, + "nauc_precision_at_10_diff1": 0.126115, + "nauc_precision_at_20_max": 0.186877, + "nauc_precision_at_20_std": 0.165627, + "nauc_precision_at_20_diff1": 0.067059, + "nauc_precision_at_100_max": 0.011607, + "nauc_precision_at_100_std": -0.04466, + "nauc_precision_at_100_diff1": -0.028366, + "nauc_precision_at_1000_max": 0.011607, + "nauc_precision_at_1000_std": -0.04466, + "nauc_precision_at_1000_diff1": -0.028366, + "nauc_mrr_at_1_max": 0.413941, + "nauc_mrr_at_1_std": 0.36767, + "nauc_mrr_at_1_diff1": 0.415544, + "nauc_mrr_at_3_max": 0.446575, + "nauc_mrr_at_3_std": 0.398833, + "nauc_mrr_at_3_diff1": 0.422602, + "nauc_mrr_at_5_max": 0.461634, + "nauc_mrr_at_5_std": 0.40927, + "nauc_mrr_at_5_diff1": 0.426145, + "nauc_mrr_at_10_max": 0.458607, + "nauc_mrr_at_10_std": 0.404504, + "nauc_mrr_at_10_diff1": 0.431699, + "nauc_mrr_at_20_max": 0.459778, + "nauc_mrr_at_20_std": 0.407652, + "nauc_mrr_at_20_diff1": 0.433058, + "nauc_mrr_at_100_max": 0.458966, + "nauc_mrr_at_100_std": 0.406831, + "nauc_mrr_at_100_diff1": 0.431509, + "nauc_mrr_at_1000_max": 0.458966, + "nauc_mrr_at_1000_std": 0.406831, + "nauc_mrr_at_1000_diff1": 0.431509, + "hit_rate_at_1": 0.38506, + "hit_rate_at_3": 0.60632, + "hit_rate_at_5": 0.69253, + "hit_rate_at_10": 0.81034, + "hit_rate_at_20": 0.88793, + "hit_rate_at_100": 0.94253, + "hit_rate_at_1000": 0.94253, + "main_score": 0.45447, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.40086, + "ndcg_at_3": 0.40026, + "ndcg_at_5": 0.4198, + "ndcg_at_10": 0.46577, + "ndcg_at_20": 0.51346, + "ndcg_at_100": 0.56997, + "ndcg_at_1000": 0.56997, + "map_at_1": 0.17666, + "map_at_3": 0.27864, + "map_at_5": 0.31522, + "map_at_10": 0.35856, + "map_at_20": 0.38584, + "map_at_100": 0.40903, + "map_at_1000": 0.40903, + "recall_at_1": 0.17666, + "recall_at_3": 0.33185, + "recall_at_5": 0.41913, + "recall_at_10": 0.55404, + "recall_at_20": 0.69626, + "recall_at_100": 0.87051, + "recall_at_1000": 0.87051, + "accuracy": 0.17666, + "precision_at_1": 0.43103, + "precision_at_3": 0.30172, + "precision_at_5": 0.24253, + "precision_at_10": 0.18161, + "precision_at_20": 0.125, + "precision_at_100": 0.03557, + "precision_at_1000": 0.00356, + "mrr_at_1": 0.431034, + "mrr_at_3": 0.514847, + "mrr_at_5": 0.534818, + "mrr_at_10": 0.549401, + "mrr_at_20": 0.556597, + "mrr_at_100": 0.558391, + "mrr_at_1000": 0.558391, + "nauc_ndcg_at_1_max": 0.396494, + "nauc_ndcg_at_1_std": 0.362747, + "nauc_ndcg_at_1_diff1": 0.390398, + "nauc_ndcg_at_3_max": 0.410333, + "nauc_ndcg_at_3_std": 0.41325, + "nauc_ndcg_at_3_diff1": 0.325021, + "nauc_ndcg_at_5_max": 0.4347, + "nauc_ndcg_at_5_std": 0.447567, + "nauc_ndcg_at_5_diff1": 0.336086, + "nauc_ndcg_at_10_max": 0.47534, + "nauc_ndcg_at_10_std": 0.487965, + "nauc_ndcg_at_10_diff1": 0.32151, + "nauc_ndcg_at_20_max": 0.490921, + "nauc_ndcg_at_20_std": 0.496673, + "nauc_ndcg_at_20_diff1": 0.31959, + "nauc_ndcg_at_100_max": 0.496486, + "nauc_ndcg_at_100_std": 0.479536, + "nauc_ndcg_at_100_diff1": 0.32529, + "nauc_ndcg_at_1000_max": 0.496486, + "nauc_ndcg_at_1000_std": 0.479536, + "nauc_ndcg_at_1000_diff1": 0.32529, + "nauc_map_at_1_max": 0.347414, + "nauc_map_at_1_std": 0.329055, + "nauc_map_at_1_diff1": 0.378094, + "nauc_map_at_3_max": 0.390693, + "nauc_map_at_3_std": 0.403736, + "nauc_map_at_3_diff1": 0.308429, + "nauc_map_at_5_max": 0.406451, + "nauc_map_at_5_std": 0.434406, + "nauc_map_at_5_diff1": 0.306351, + "nauc_map_at_10_max": 0.432913, + "nauc_map_at_10_std": 0.462766, + "nauc_map_at_10_diff1": 0.296621, + "nauc_map_at_20_max": 0.445963, + "nauc_map_at_20_std": 0.469918, + "nauc_map_at_20_diff1": 0.30002, + "nauc_map_at_100_max": 0.444166, + "nauc_map_at_100_std": 0.460187, + "nauc_map_at_100_diff1": 0.299932, + "nauc_map_at_1000_max": 0.444166, + "nauc_map_at_1000_std": 0.460187, + "nauc_map_at_1000_diff1": 0.299932, + "nauc_recall_at_1_max": 0.347414, + "nauc_recall_at_1_std": 0.329055, + "nauc_recall_at_1_diff1": 0.378094, + "nauc_recall_at_3_max": 0.349374, + "nauc_recall_at_3_std": 0.375122, + "nauc_recall_at_3_diff1": 0.252581, + "nauc_recall_at_5_max": 0.370425, + "nauc_recall_at_5_std": 0.418026, + "nauc_recall_at_5_diff1": 0.235803, + "nauc_recall_at_10_max": 0.46516, + "nauc_recall_at_10_std": 0.508221, + "nauc_recall_at_10_diff1": 0.217371, + "nauc_recall_at_20_max": 0.499709, + "nauc_recall_at_20_std": 0.534613, + "nauc_recall_at_20_diff1": 0.201017, + "nauc_recall_at_100_max": 0.666504, + "nauc_recall_at_100_std": 0.591998, + "nauc_recall_at_100_diff1": 0.182256, + "nauc_recall_at_1000_max": 0.666504, + "nauc_recall_at_1000_std": 0.591998, + "nauc_recall_at_1000_diff1": 0.182256, + "nauc_precision_at_1_max": 0.397291, + "nauc_precision_at_1_std": 0.379765, + "nauc_precision_at_1_diff1": 0.393482, + "nauc_precision_at_3_max": 0.352637, + "nauc_precision_at_3_std": 0.382981, + "nauc_precision_at_3_diff1": 0.202089, + "nauc_precision_at_5_max": 0.320468, + "nauc_precision_at_5_std": 0.368466, + "nauc_precision_at_5_diff1": 0.159449, + "nauc_precision_at_10_max": 0.26091, + "nauc_precision_at_10_std": 0.278112, + "nauc_precision_at_10_diff1": 0.089539, + "nauc_precision_at_20_max": 0.156593, + "nauc_precision_at_20_std": 0.134073, + "nauc_precision_at_20_diff1": 0.025881, + "nauc_precision_at_100_max": -0.01851, + "nauc_precision_at_100_std": -0.078861, + "nauc_precision_at_100_diff1": -0.039784, + "nauc_precision_at_1000_max": -0.01851, + "nauc_precision_at_1000_std": -0.078861, + "nauc_precision_at_1000_diff1": -0.039784, + "nauc_mrr_at_1_max": 0.397291, + "nauc_mrr_at_1_std": 0.379765, + "nauc_mrr_at_1_diff1": 0.393482, + "nauc_mrr_at_3_max": 0.414048, + "nauc_mrr_at_3_std": 0.408822, + "nauc_mrr_at_3_diff1": 0.369062, + "nauc_mrr_at_5_max": 0.429844, + "nauc_mrr_at_5_std": 0.422282, + "nauc_mrr_at_5_diff1": 0.376547, + "nauc_mrr_at_10_max": 0.43193, + "nauc_mrr_at_10_std": 0.422056, + "nauc_mrr_at_10_diff1": 0.377432, + "nauc_mrr_at_20_max": 0.42844, + "nauc_mrr_at_20_std": 0.419436, + "nauc_mrr_at_20_diff1": 0.377925, + "nauc_mrr_at_100_max": 0.429064, + "nauc_mrr_at_100_std": 0.419191, + "nauc_mrr_at_100_diff1": 0.378931, + "nauc_mrr_at_1000_max": 0.429064, + "nauc_mrr_at_1000_std": 0.419191, + "nauc_mrr_at_1000_diff1": 0.378931, + "hit_rate_at_1": 0.43103, + "hit_rate_at_3": 0.61494, + "hit_rate_at_5": 0.70115, + "hit_rate_at_10": 0.81034, + "hit_rate_at_20": 0.91092, + "hit_rate_at_100": 0.9569, + "hit_rate_at_1000": 0.9569, + "main_score": 0.46577, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.31609, + "ndcg_at_3": 0.33757, + "ndcg_at_5": 0.35849, + "ndcg_at_10": 0.41187, + "ndcg_at_20": 0.46008, + "ndcg_at_100": 0.51696, + "ndcg_at_1000": 0.51696, + "map_at_1": 0.13633, + "map_at_3": 0.22579, + "map_at_5": 0.25916, + "map_at_10": 0.3051, + "map_at_20": 0.3333, + "map_at_100": 0.3559, + "map_at_1000": 0.3559, + "recall_at_1": 0.13633, + "recall_at_3": 0.28441, + "recall_at_5": 0.366, + "recall_at_10": 0.52088, + "recall_at_20": 0.65819, + "recall_at_100": 0.83893, + "recall_at_1000": 0.83893, + "accuracy": 0.13633, + "precision_at_1": 0.33908, + "precision_at_3": 0.26628, + "precision_at_5": 0.22069, + "precision_at_10": 0.17385, + "precision_at_20": 0.1217, + "precision_at_100": 0.0344, + "precision_at_1000": 0.00344, + "mrr_at_1": 0.33908, + "mrr_at_3": 0.432471, + "mrr_at_5": 0.454598, + "mrr_at_10": 0.471276, + "mrr_at_20": 0.477477, + "mrr_at_100": 0.479957, + "mrr_at_1000": 0.479957, + "nauc_ndcg_at_1_max": 0.432141, + "nauc_ndcg_at_1_std": 0.355358, + "nauc_ndcg_at_1_diff1": 0.342304, + "nauc_ndcg_at_3_max": 0.452394, + "nauc_ndcg_at_3_std": 0.39512, + "nauc_ndcg_at_3_diff1": 0.329978, + "nauc_ndcg_at_5_max": 0.469816, + "nauc_ndcg_at_5_std": 0.430543, + "nauc_ndcg_at_5_diff1": 0.323399, + "nauc_ndcg_at_10_max": 0.512753, + "nauc_ndcg_at_10_std": 0.475737, + "nauc_ndcg_at_10_diff1": 0.320065, + "nauc_ndcg_at_20_max": 0.527302, + "nauc_ndcg_at_20_std": 0.493928, + "nauc_ndcg_at_20_diff1": 0.331834, + "nauc_ndcg_at_100_max": 0.532889, + "nauc_ndcg_at_100_std": 0.474581, + "nauc_ndcg_at_100_diff1": 0.3334, + "nauc_ndcg_at_1000_max": 0.532889, + "nauc_ndcg_at_1000_std": 0.474581, + "nauc_ndcg_at_1000_diff1": 0.3334, + "nauc_map_at_1_max": 0.310444, + "nauc_map_at_1_std": 0.252258, + "nauc_map_at_1_diff1": 0.354613, + "nauc_map_at_3_max": 0.398833, + "nauc_map_at_3_std": 0.354486, + "nauc_map_at_3_diff1": 0.317401, + "nauc_map_at_5_max": 0.428291, + "nauc_map_at_5_std": 0.397775, + "nauc_map_at_5_diff1": 0.302555, + "nauc_map_at_10_max": 0.460755, + "nauc_map_at_10_std": 0.435448, + "nauc_map_at_10_diff1": 0.296541, + "nauc_map_at_20_max": 0.470239, + "nauc_map_at_20_std": 0.445945, + "nauc_map_at_20_diff1": 0.300556, + "nauc_map_at_100_max": 0.469874, + "nauc_map_at_100_std": 0.436597, + "nauc_map_at_100_diff1": 0.303445, + "nauc_map_at_1000_max": 0.469874, + "nauc_map_at_1000_std": 0.436597, + "nauc_map_at_1000_diff1": 0.303445, + "nauc_recall_at_1_max": 0.310444, + "nauc_recall_at_1_std": 0.252258, + "nauc_recall_at_1_diff1": 0.354613, + "nauc_recall_at_3_max": 0.380098, + "nauc_recall_at_3_std": 0.344194, + "nauc_recall_at_3_diff1": 0.290253, + "nauc_recall_at_5_max": 0.42032, + "nauc_recall_at_5_std": 0.410533, + "nauc_recall_at_5_diff1": 0.262027, + "nauc_recall_at_10_max": 0.487862, + "nauc_recall_at_10_std": 0.489403, + "nauc_recall_at_10_diff1": 0.23501, + "nauc_recall_at_20_max": 0.531424, + "nauc_recall_at_20_std": 0.54867, + "nauc_recall_at_20_diff1": 0.264364, + "nauc_recall_at_100_max": 0.672328, + "nauc_recall_at_100_std": 0.607858, + "nauc_recall_at_100_diff1": 0.243874, + "nauc_recall_at_1000_max": 0.672328, + "nauc_recall_at_1000_std": 0.607858, + "nauc_recall_at_1000_diff1": 0.243874, + "nauc_precision_at_1_max": 0.424603, + "nauc_precision_at_1_std": 0.357866, + "nauc_precision_at_1_diff1": 0.339672, + "nauc_precision_at_3_max": 0.432569, + "nauc_precision_at_3_std": 0.404204, + "nauc_precision_at_3_diff1": 0.235257, + "nauc_precision_at_5_max": 0.398145, + "nauc_precision_at_5_std": 0.404105, + "nauc_precision_at_5_diff1": 0.17289, + "nauc_precision_at_10_max": 0.335177, + "nauc_precision_at_10_std": 0.338169, + "nauc_precision_at_10_diff1": 0.131721, + "nauc_precision_at_20_max": 0.214357, + "nauc_precision_at_20_std": 0.201598, + "nauc_precision_at_20_diff1": 0.094836, + "nauc_precision_at_100_max": 0.065085, + "nauc_precision_at_100_std": 0.013135, + "nauc_precision_at_100_diff1": 0.0238, + "nauc_precision_at_1000_max": 0.065085, + "nauc_precision_at_1000_std": 0.013135, + "nauc_precision_at_1000_diff1": 0.0238, + "nauc_mrr_at_1_max": 0.424603, + "nauc_mrr_at_1_std": 0.357866, + "nauc_mrr_at_1_diff1": 0.339672, + "nauc_mrr_at_3_max": 0.463209, + "nauc_mrr_at_3_std": 0.393739, + "nauc_mrr_at_3_diff1": 0.338991, + "nauc_mrr_at_5_max": 0.476554, + "nauc_mrr_at_5_std": 0.408753, + "nauc_mrr_at_5_diff1": 0.346501, + "nauc_mrr_at_10_max": 0.482518, + "nauc_mrr_at_10_std": 0.415032, + "nauc_mrr_at_10_diff1": 0.349466, + "nauc_mrr_at_20_max": 0.479773, + "nauc_mrr_at_20_std": 0.412234, + "nauc_mrr_at_20_diff1": 0.350542, + "nauc_mrr_at_100_max": 0.479665, + "nauc_mrr_at_100_std": 0.411619, + "nauc_mrr_at_100_diff1": 0.348323, + "nauc_mrr_at_1000_max": 0.479665, + "nauc_mrr_at_1000_std": 0.411619, + "nauc_mrr_at_1000_diff1": 0.348323, + "hit_rate_at_1": 0.33908, + "hit_rate_at_3": 0.5546, + "hit_rate_at_5": 0.6523, + "hit_rate_at_10": 0.77586, + "hit_rate_at_20": 0.8592, + "hit_rate_at_100": 0.92816, + "hit_rate_at_1000": 0.92816, + "main_score": 0.41187, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.34339, + "ndcg_at_3": 0.35119, + "ndcg_at_5": 0.37426, + "ndcg_at_10": 0.42686, + "ndcg_at_20": 0.47177, + "ndcg_at_100": 0.52936, + "ndcg_at_1000": 0.52936, + "map_at_1": 0.14637, + "map_at_3": 0.23046, + "map_at_5": 0.26685, + "map_at_10": 0.31614, + "map_at_20": 0.34254, + "map_at_100": 0.36621, + "map_at_1000": 0.36621, + "recall_at_1": 0.14637, + "recall_at_3": 0.28739, + "recall_at_5": 0.37172, + "recall_at_10": 0.53572, + "recall_at_20": 0.66677, + "recall_at_100": 0.84722, + "recall_at_1000": 0.84722, + "accuracy": 0.14637, + "precision_at_1": 0.37069, + "precision_at_3": 0.2749, + "precision_at_5": 0.23218, + "precision_at_10": 0.18161, + "precision_at_20": 0.12371, + "precision_at_100": 0.03543, + "precision_at_1000": 0.00354, + "mrr_at_1": 0.37069, + "mrr_at_3": 0.462165, + "mrr_at_5": 0.478544, + "mrr_at_10": 0.496855, + "mrr_at_20": 0.502765, + "mrr_at_100": 0.505151, + "mrr_at_1000": 0.505151, + "nauc_ndcg_at_1_max": 0.369889, + "nauc_ndcg_at_1_std": 0.336147, + "nauc_ndcg_at_1_diff1": 0.27347, + "nauc_ndcg_at_3_max": 0.384961, + "nauc_ndcg_at_3_std": 0.400259, + "nauc_ndcg_at_3_diff1": 0.207153, + "nauc_ndcg_at_5_max": 0.419021, + "nauc_ndcg_at_5_std": 0.43953, + "nauc_ndcg_at_5_diff1": 0.207409, + "nauc_ndcg_at_10_max": 0.461943, + "nauc_ndcg_at_10_std": 0.485295, + "nauc_ndcg_at_10_diff1": 0.197124, + "nauc_ndcg_at_20_max": 0.472, + "nauc_ndcg_at_20_std": 0.505448, + "nauc_ndcg_at_20_diff1": 0.203782, + "nauc_ndcg_at_100_max": 0.483179, + "nauc_ndcg_at_100_std": 0.489007, + "nauc_ndcg_at_100_diff1": 0.218554, + "nauc_ndcg_at_1000_max": 0.483179, + "nauc_ndcg_at_1000_std": 0.489007, + "nauc_ndcg_at_1000_diff1": 0.218554, + "nauc_map_at_1_max": 0.278861, + "nauc_map_at_1_std": 0.286013, + "nauc_map_at_1_diff1": 0.310313, + "nauc_map_at_3_max": 0.331994, + "nauc_map_at_3_std": 0.371917, + "nauc_map_at_3_diff1": 0.222464, + "nauc_map_at_5_max": 0.373244, + "nauc_map_at_5_std": 0.414333, + "nauc_map_at_5_diff1": 0.205313, + "nauc_map_at_10_max": 0.405431, + "nauc_map_at_10_std": 0.449683, + "nauc_map_at_10_diff1": 0.189531, + "nauc_map_at_20_max": 0.413706, + "nauc_map_at_20_std": 0.463236, + "nauc_map_at_20_diff1": 0.193865, + "nauc_map_at_100_max": 0.415494, + "nauc_map_at_100_std": 0.455879, + "nauc_map_at_100_diff1": 0.198465, + "nauc_map_at_1000_max": 0.415494, + "nauc_map_at_1000_std": 0.455879, + "nauc_map_at_1000_diff1": 0.198465, + "nauc_recall_at_1_max": 0.278861, + "nauc_recall_at_1_std": 0.286013, + "nauc_recall_at_1_diff1": 0.310313, + "nauc_recall_at_3_max": 0.332247, + "nauc_recall_at_3_std": 0.360803, + "nauc_recall_at_3_diff1": 0.162967, + "nauc_recall_at_5_max": 0.383931, + "nauc_recall_at_5_std": 0.423738, + "nauc_recall_at_5_diff1": 0.130804, + "nauc_recall_at_10_max": 0.451903, + "nauc_recall_at_10_std": 0.494512, + "nauc_recall_at_10_diff1": 0.111199, + "nauc_recall_at_20_max": 0.472687, + "nauc_recall_at_20_std": 0.5433, + "nauc_recall_at_20_diff1": 0.096204, + "nauc_recall_at_100_max": 0.660759, + "nauc_recall_at_100_std": 0.612872, + "nauc_recall_at_100_diff1": 0.138343, + "nauc_recall_at_1000_max": 0.660759, + "nauc_recall_at_1000_std": 0.612872, + "nauc_recall_at_1000_diff1": 0.138343, + "nauc_precision_at_1_max": 0.361432, + "nauc_precision_at_1_std": 0.335911, + "nauc_precision_at_1_diff1": 0.279617, + "nauc_precision_at_3_max": 0.368763, + "nauc_precision_at_3_std": 0.403266, + "nauc_precision_at_3_diff1": 0.096392, + "nauc_precision_at_5_max": 0.36697, + "nauc_precision_at_5_std": 0.395693, + "nauc_precision_at_5_diff1": 0.062001, + "nauc_precision_at_10_max": 0.306292, + "nauc_precision_at_10_std": 0.325951, + "nauc_precision_at_10_diff1": 0.030145, + "nauc_precision_at_20_max": 0.21087, + "nauc_precision_at_20_std": 0.22098, + "nauc_precision_at_20_diff1": 0.029413, + "nauc_precision_at_100_max": 0.065753, + "nauc_precision_at_100_std": 0.0254, + "nauc_precision_at_100_diff1": -0.001772, + "nauc_precision_at_1000_max": 0.065753, + "nauc_precision_at_1000_std": 0.0254, + "nauc_precision_at_1000_diff1": -0.001772, + "nauc_mrr_at_1_max": 0.361432, + "nauc_mrr_at_1_std": 0.335911, + "nauc_mrr_at_1_diff1": 0.279617, + "nauc_mrr_at_3_max": 0.406887, + "nauc_mrr_at_3_std": 0.38973, + "nauc_mrr_at_3_diff1": 0.238221, + "nauc_mrr_at_5_max": 0.412471, + "nauc_mrr_at_5_std": 0.400663, + "nauc_mrr_at_5_diff1": 0.240941, + "nauc_mrr_at_10_max": 0.416099, + "nauc_mrr_at_10_std": 0.402839, + "nauc_mrr_at_10_diff1": 0.250432, + "nauc_mrr_at_20_max": 0.411808, + "nauc_mrr_at_20_std": 0.399471, + "nauc_mrr_at_20_diff1": 0.247368, + "nauc_mrr_at_100_max": 0.411668, + "nauc_mrr_at_100_std": 0.398569, + "nauc_mrr_at_100_diff1": 0.246847, + "nauc_mrr_at_1000_max": 0.411668, + "nauc_mrr_at_1000_std": 0.398569, + "nauc_mrr_at_1000_diff1": 0.246847, + "hit_rate_at_1": 0.37069, + "hit_rate_at_3": 0.58333, + "hit_rate_at_5": 0.65517, + "hit_rate_at_10": 0.7931, + "hit_rate_at_20": 0.87644, + "hit_rate_at_100": 0.94253, + "hit_rate_at_1000": 0.94253, + "main_score": 0.42686, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 1140.6146848539356, + "kg_co2_emissions": null, + "date": 1792439943.352994, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 7.790978997945786e-06, + "end": 36.25689580896869, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 36.25691629003268, + "end": 36.25691849295981, + "split": "", + "subset": "" + }, + { + "name": "Encoding corpus", + "start": 36.26146004500333, + "end": 222.25243264599703, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 222.25252998096403, + "end": 222.45906225498766, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 222.45907827897463, + "end": 222.5082021469716, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding corpus", + "start": 222.52043089002836, + "end": 404.83525753603317, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 404.8353324380005, + "end": 405.0434448720189, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 405.0434622679604, + "end": 405.08945518604014, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding corpus", + "start": 405.10416302096564, + "end": 587.9545163169969, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 587.9545798520558, + "end": 588.1465461179614, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 588.1465663580457, + "end": 588.1906432809774, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding corpus", + "start": 588.2039901489625, + "end": 771.1097436350537, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 771.1098140199902, + "end": 771.3133630190277, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 771.3133784929523, + "end": 771.3578229290433, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding corpus", + "start": 771.3711425450165, + "end": 955.8228917970555, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 955.8229498430155, + "end": 956.0311660190346, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 956.0311795689631, + "end": 956.0746795460582, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding corpus", + "start": 956.0877137070056, + "end": 1140.3434043410234, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Encoding queries", + "start": 1140.3435280660633, + "end": 1140.5532257490559, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 1140.5532518279506, + "end": 1140.602170758997, + "split": "test", + "subset": "portuguese" + } + ] +} diff --git a/results/mteb__baseline-bm25s/0_3_0/run_settings.jsonl b/results/mteb__baseline-bm25s/0_3_0/run_settings.jsonl index ec4658eff5..d143c99d00 100644 --- a/results/mteb__baseline-bm25s/0_3_0/run_settings.jsonl +++ b/results/mteb__baseline-bm25s/0_3_0/run_settings.jsonl @@ -390,3 +390,5 @@ {"task": "Vidore3PharmaceuticalsNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3PhysicsNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} diff --git a/results/mteb__baseline-no-reranking/2/Vidore3NuclearNemotron35Embed8BReranking.v2.json b/results/mteb__baseline-no-reranking/2/Vidore3NuclearNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..af18f4cec6 --- /dev/null +++ b/results/mteb__baseline-no-reranking/2/Vidore3NuclearNemotron35Embed8BReranking.v2.json @@ -0,0 +1,1048 @@ +{ + "dataset_revision": "97ff9e10e265e64abece589ff40c552215754f26", + "task_name": "Vidore3NuclearNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.44127, + "ndcg_at_3": 0.46332, + "ndcg_at_5": 0.47989, + "ndcg_at_10": 0.5116, + "ndcg_at_20": 0.5613, + "ndcg_at_100": 0.59399, + "ndcg_at_1000": 0.59399, + "map_at_1": 0.2118, + "map_at_3": 0.31174, + "map_at_5": 0.35297, + "map_at_10": 0.39231, + "map_at_20": 0.42486, + "map_at_100": 0.43978, + "map_at_1000": 0.43978, + "recall_at_1": 0.2118, + "recall_at_3": 0.37776, + "recall_at_5": 0.47168, + "recall_at_10": 0.58475, + "recall_at_20": 0.72171, + "recall_at_100": 0.81132, + "recall_at_1000": 0.81132, + "accuracy": 0.2118, + "precision_at_1": 0.45482, + "precision_at_3": 0.33735, + "precision_at_5": 0.27289, + "precision_at_10": 0.19669, + "precision_at_20": 0.13614, + "precision_at_100": 0.0338, + "precision_at_1000": 0.00338, + "mrr_at_1": 0.454819, + "mrr_at_3": 0.572289, + "mrr_at_5": 0.58991, + "mrr_at_10": 0.600785, + "mrr_at_20": 0.603555, + "mrr_at_100": 0.604153, + "mrr_at_1000": 0.604153, + "nauc_ndcg_at_1_max": 0.0, + "nauc_ndcg_at_1_std": 0.0, + "nauc_ndcg_at_1_diff1": 0.0, + "nauc_ndcg_at_3_max": 0.0, + "nauc_ndcg_at_3_std": 0.0, + "nauc_ndcg_at_3_diff1": 0.0, + "nauc_ndcg_at_5_max": 0.0, + "nauc_ndcg_at_5_std": 0.0, + "nauc_ndcg_at_5_diff1": 0.0, + "nauc_ndcg_at_10_max": 0.0, + "nauc_ndcg_at_10_std": 0.0, + "nauc_ndcg_at_10_diff1": 0.0, + "nauc_ndcg_at_20_max": -0.0, + "nauc_ndcg_at_20_std": -0.0, + "nauc_ndcg_at_20_diff1": -0.0, + "nauc_ndcg_at_100_max": 0.0, + "nauc_ndcg_at_100_std": 0.0, + "nauc_ndcg_at_100_diff1": 0.0, + "nauc_ndcg_at_1000_max": 0.0, + "nauc_ndcg_at_1000_std": 0.0, + "nauc_ndcg_at_1000_diff1": 0.0, + "nauc_map_at_1_max": -0.0, + "nauc_map_at_1_std": -0.0, + "nauc_map_at_1_diff1": -0.0, + "nauc_map_at_3_max": 0.0, + "nauc_map_at_3_std": 0.0, + "nauc_map_at_3_diff1": 0.0, + "nauc_map_at_5_max": 0.0, + "nauc_map_at_5_std": 0.0, + "nauc_map_at_5_diff1": 0.0, + "nauc_map_at_10_max": 0.0, + "nauc_map_at_10_std": 0.0, + "nauc_map_at_10_diff1": 0.0, + "nauc_map_at_20_max": 0.0, + "nauc_map_at_20_std": 0.0, + "nauc_map_at_20_diff1": 0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": -0.0, + "nauc_recall_at_1_std": -0.0, + "nauc_recall_at_1_diff1": -0.0, + "nauc_recall_at_3_max": 0.0, + "nauc_recall_at_3_std": 0.0, + "nauc_recall_at_3_diff1": 0.0, + "nauc_recall_at_5_max": 0.0, + "nauc_recall_at_5_std": 0.0, + "nauc_recall_at_5_diff1": 0.0, + "nauc_recall_at_10_max": 0.0, + "nauc_recall_at_10_std": 0.0, + "nauc_recall_at_10_diff1": 0.0, + "nauc_recall_at_20_max": -0.0, + "nauc_recall_at_20_std": -0.0, + "nauc_recall_at_20_diff1": -0.0, + "nauc_recall_at_100_max": -0.0, + "nauc_recall_at_100_std": -0.0, + "nauc_recall_at_100_diff1": -0.0, + "nauc_recall_at_1000_max": -0.0, + "nauc_recall_at_1000_std": -0.0, + "nauc_recall_at_1000_diff1": -0.0, + "nauc_precision_at_1_max": 0.0, + "nauc_precision_at_1_std": 0.0, + "nauc_precision_at_1_diff1": 0.0, + "nauc_precision_at_3_max": -0.0, + "nauc_precision_at_3_std": -0.0, + "nauc_precision_at_3_diff1": -0.0, + "nauc_precision_at_5_max": 0.0, + "nauc_precision_at_5_std": 0.0, + "nauc_precision_at_5_diff1": 0.0, + "nauc_precision_at_10_max": 0.0, + "nauc_precision_at_10_std": 0.0, + "nauc_precision_at_10_diff1": 0.0, + "nauc_precision_at_20_max": -0.0, + "nauc_precision_at_20_std": -0.0, + "nauc_precision_at_20_diff1": -0.0, + "nauc_precision_at_100_max": -0.0, + "nauc_precision_at_100_std": -0.0, + "nauc_precision_at_100_diff1": -0.0, + "nauc_precision_at_1000_max": 0.0, + "nauc_precision_at_1000_std": 0.0, + "nauc_precision_at_1000_diff1": 0.0, + "nauc_mrr_at_1_max": 0.0, + "nauc_mrr_at_1_std": 0.0, + "nauc_mrr_at_1_diff1": 0.0, + "nauc_mrr_at_3_max": -0.0, + "nauc_mrr_at_3_std": -0.0, + "nauc_mrr_at_3_diff1": -0.0, + "nauc_mrr_at_5_max": 0.0, + "nauc_mrr_at_5_std": 0.0, + "nauc_mrr_at_5_diff1": 0.0, + "nauc_mrr_at_10_max": 0.0, + "nauc_mrr_at_10_std": 0.0, + "nauc_mrr_at_10_diff1": 0.0, + "nauc_mrr_at_20_max": 0.0, + "nauc_mrr_at_20_std": 0.0, + "nauc_mrr_at_20_diff1": 0.0, + "nauc_mrr_at_100_max": 0.0, + "nauc_mrr_at_100_std": 0.0, + "nauc_mrr_at_100_diff1": 0.0, + "nauc_mrr_at_1000_max": 0.0, + "nauc_mrr_at_1000_std": 0.0, + "nauc_mrr_at_1000_diff1": 0.0, + "hit_rate_at_1": 0.45482, + "hit_rate_at_3": 0.7259, + "hit_rate_at_5": 0.8012, + "hit_rate_at_10": 0.88554, + "hit_rate_at_20": 0.92771, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.5116, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.4262, + "ndcg_at_3": 0.44468, + "ndcg_at_5": 0.45864, + "ndcg_at_10": 0.49377, + "ndcg_at_20": 0.53686, + "ndcg_at_100": 0.575, + "ndcg_at_1000": 0.575, + "map_at_1": 0.19493, + "map_at_3": 0.29373, + "map_at_5": 0.33497, + "map_at_10": 0.37494, + "map_at_20": 0.40364, + "map_at_100": 0.41897, + "map_at_1000": 0.41897, + "recall_at_1": 0.19493, + "recall_at_3": 0.35924, + "recall_at_5": 0.44779, + "recall_at_10": 0.56666, + "recall_at_20": 0.68467, + "recall_at_100": 0.79964, + "recall_at_1000": 0.79964, + "accuracy": 0.19493, + "precision_at_1": 0.43675, + "precision_at_3": 0.32631, + "precision_at_5": 0.26446, + "precision_at_10": 0.19096, + "precision_at_20": 0.12922, + "precision_at_100": 0.0328, + "precision_at_1000": 0.00328, + "mrr_at_1": 0.436747, + "mrr_at_3": 0.559739, + "mrr_at_5": 0.571335, + "mrr_at_10": 0.584588, + "mrr_at_20": 0.588002, + "mrr_at_100": 0.589321, + "mrr_at_1000": 0.589321, + "nauc_ndcg_at_1_max": 0.0, + "nauc_ndcg_at_1_std": 0.0, + "nauc_ndcg_at_1_diff1": 0.0, + "nauc_ndcg_at_3_max": 0.0, + "nauc_ndcg_at_3_std": 0.0, + "nauc_ndcg_at_3_diff1": 0.0, + "nauc_ndcg_at_5_max": 0.0, + "nauc_ndcg_at_5_std": 0.0, + "nauc_ndcg_at_5_diff1": 0.0, + "nauc_ndcg_at_10_max": 0.0, + "nauc_ndcg_at_10_std": 0.0, + "nauc_ndcg_at_10_diff1": 0.0, + "nauc_ndcg_at_20_max": -0.0, + "nauc_ndcg_at_20_std": -0.0, + "nauc_ndcg_at_20_diff1": -0.0, + "nauc_ndcg_at_100_max": 0.0, + "nauc_ndcg_at_100_std": 0.0, + "nauc_ndcg_at_100_diff1": 0.0, + "nauc_ndcg_at_1000_max": 0.0, + "nauc_ndcg_at_1000_std": 0.0, + "nauc_ndcg_at_1000_diff1": 0.0, + "nauc_map_at_1_max": -0.0, + "nauc_map_at_1_std": -0.0, + "nauc_map_at_1_diff1": -0.0, + "nauc_map_at_3_max": 0.0, + "nauc_map_at_3_std": 0.0, + "nauc_map_at_3_diff1": 0.0, + "nauc_map_at_5_max": 0.0, + "nauc_map_at_5_std": 0.0, + "nauc_map_at_5_diff1": 0.0, + "nauc_map_at_10_max": 0.0, + "nauc_map_at_10_std": 0.0, + "nauc_map_at_10_diff1": 0.0, + "nauc_map_at_20_max": 0.0, + "nauc_map_at_20_std": 0.0, + "nauc_map_at_20_diff1": 0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": -0.0, + "nauc_recall_at_1_std": -0.0, + "nauc_recall_at_1_diff1": -0.0, + "nauc_recall_at_3_max": 0.0, + "nauc_recall_at_3_std": 0.0, + "nauc_recall_at_3_diff1": 0.0, + "nauc_recall_at_5_max": -0.0, + "nauc_recall_at_5_std": -0.0, + "nauc_recall_at_5_diff1": -0.0, + "nauc_recall_at_10_max": 0.0, + "nauc_recall_at_10_std": 0.0, + "nauc_recall_at_10_diff1": 0.0, + "nauc_recall_at_20_max": 0.0, + "nauc_recall_at_20_std": 0.0, + "nauc_recall_at_20_diff1": 0.0, + "nauc_recall_at_100_max": 0.0, + "nauc_recall_at_100_std": 0.0, + "nauc_recall_at_100_diff1": 0.0, + "nauc_recall_at_1000_max": 0.0, + "nauc_recall_at_1000_std": 0.0, + "nauc_recall_at_1000_diff1": 0.0, + "nauc_precision_at_1_max": 0.0, + "nauc_precision_at_1_std": 0.0, + "nauc_precision_at_1_diff1": 0.0, + "nauc_precision_at_3_max": 0.0, + "nauc_precision_at_3_std": 0.0, + "nauc_precision_at_3_diff1": 0.0, + "nauc_precision_at_5_max": 0.0, + "nauc_precision_at_5_std": 0.0, + "nauc_precision_at_5_diff1": 0.0, + "nauc_precision_at_10_max": 0.0, + "nauc_precision_at_10_std": 0.0, + "nauc_precision_at_10_diff1": 0.0, + "nauc_precision_at_20_max": -0.0, + "nauc_precision_at_20_std": -0.0, + "nauc_precision_at_20_diff1": -0.0, + "nauc_precision_at_100_max": 0.0, + "nauc_precision_at_100_std": 0.0, + "nauc_precision_at_100_diff1": 0.0, + "nauc_precision_at_1000_max": -0.0, + "nauc_precision_at_1000_std": -0.0, + "nauc_precision_at_1000_diff1": -0.0, + "nauc_mrr_at_1_max": 0.0, + "nauc_mrr_at_1_std": 0.0, + "nauc_mrr_at_1_diff1": 0.0, + "nauc_mrr_at_3_max": -0.0, + "nauc_mrr_at_3_std": -0.0, + "nauc_mrr_at_3_diff1": -0.0, + "nauc_mrr_at_5_max": 0.0, + "nauc_mrr_at_5_std": 0.0, + "nauc_mrr_at_5_diff1": 0.0, + "nauc_mrr_at_10_max": 0.0, + "nauc_mrr_at_10_std": 0.0, + "nauc_mrr_at_10_diff1": 0.0, + "nauc_mrr_at_20_max": -0.0, + "nauc_mrr_at_20_std": -0.0, + "nauc_mrr_at_20_diff1": -0.0, + "nauc_mrr_at_100_max": 0.0, + "nauc_mrr_at_100_std": 0.0, + "nauc_mrr_at_100_diff1": 0.0, + "nauc_mrr_at_1000_max": 0.0, + "nauc_mrr_at_1000_std": 0.0, + "nauc_mrr_at_1000_diff1": 0.0, + "hit_rate_at_1": 0.43675, + "hit_rate_at_3": 0.71386, + "hit_rate_at_5": 0.76205, + "hit_rate_at_10": 0.85542, + "hit_rate_at_20": 0.90361, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.49377, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.43223, + "ndcg_at_3": 0.44836, + "ndcg_at_5": 0.46027, + "ndcg_at_10": 0.49316, + "ndcg_at_20": 0.53577, + "ndcg_at_100": 0.5773, + "ndcg_at_1000": 0.5773, + "map_at_1": 0.19882, + "map_at_3": 0.29528, + "map_at_5": 0.33373, + "map_at_10": 0.37408, + "map_at_20": 0.40153, + "map_at_100": 0.41851, + "map_at_1000": 0.41851, + "recall_at_1": 0.19882, + "recall_at_3": 0.3547, + "recall_at_5": 0.44287, + "recall_at_10": 0.56305, + "recall_at_20": 0.68094, + "recall_at_100": 0.80754, + "recall_at_1000": 0.80754, + "accuracy": 0.19882, + "precision_at_1": 0.44578, + "precision_at_3": 0.33032, + "precision_at_5": 0.26506, + "precision_at_10": 0.19096, + "precision_at_20": 0.12771, + "precision_at_100": 0.03304, + "precision_at_1000": 0.0033, + "mrr_at_1": 0.445783, + "mrr_at_3": 0.559237, + "mrr_at_5": 0.579719, + "mrr_at_10": 0.589228, + "mrr_at_20": 0.592793, + "mrr_at_100": 0.594288, + "mrr_at_1000": 0.594288, + "nauc_ndcg_at_1_max": 0.0, + "nauc_ndcg_at_1_std": 0.0, + "nauc_ndcg_at_1_diff1": 0.0, + "nauc_ndcg_at_3_max": 0.0, + "nauc_ndcg_at_3_std": 0.0, + "nauc_ndcg_at_3_diff1": 0.0, + "nauc_ndcg_at_5_max": 0.0, + "nauc_ndcg_at_5_std": 0.0, + "nauc_ndcg_at_5_diff1": 0.0, + "nauc_ndcg_at_10_max": 0.0, + "nauc_ndcg_at_10_std": 0.0, + "nauc_ndcg_at_10_diff1": 0.0, + "nauc_ndcg_at_20_max": 0.0, + "nauc_ndcg_at_20_std": 0.0, + "nauc_ndcg_at_20_diff1": 0.0, + "nauc_ndcg_at_100_max": -0.0, + "nauc_ndcg_at_100_std": -0.0, + "nauc_ndcg_at_100_diff1": -0.0, + "nauc_ndcg_at_1000_max": -0.0, + "nauc_ndcg_at_1000_std": -0.0, + "nauc_ndcg_at_1000_diff1": -0.0, + "nauc_map_at_1_max": 0.0, + "nauc_map_at_1_std": 0.0, + "nauc_map_at_1_diff1": 0.0, + "nauc_map_at_3_max": 0.0, + "nauc_map_at_3_std": 0.0, + "nauc_map_at_3_diff1": 0.0, + "nauc_map_at_5_max": 0.0, + "nauc_map_at_5_std": 0.0, + "nauc_map_at_5_diff1": 0.0, + "nauc_map_at_10_max": 0.0, + "nauc_map_at_10_std": 0.0, + "nauc_map_at_10_diff1": 0.0, + "nauc_map_at_20_max": 0.0, + "nauc_map_at_20_std": 0.0, + "nauc_map_at_20_diff1": 0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": 0.0, + "nauc_recall_at_1_std": 0.0, + "nauc_recall_at_1_diff1": 0.0, + "nauc_recall_at_3_max": 0.0, + "nauc_recall_at_3_std": 0.0, + "nauc_recall_at_3_diff1": 0.0, + "nauc_recall_at_5_max": 0.0, + "nauc_recall_at_5_std": 0.0, + "nauc_recall_at_5_diff1": 0.0, + "nauc_recall_at_10_max": 0.0, + "nauc_recall_at_10_std": 0.0, + "nauc_recall_at_10_diff1": 0.0, + "nauc_recall_at_20_max": 0.0, + "nauc_recall_at_20_std": 0.0, + "nauc_recall_at_20_diff1": 0.0, + "nauc_recall_at_100_max": 0.0, + "nauc_recall_at_100_std": 0.0, + "nauc_recall_at_100_diff1": 0.0, + "nauc_recall_at_1000_max": 0.0, + "nauc_recall_at_1000_std": 0.0, + "nauc_recall_at_1000_diff1": 0.0, + "nauc_precision_at_1_max": 0.0, + "nauc_precision_at_1_std": 0.0, + "nauc_precision_at_1_diff1": 0.0, + "nauc_precision_at_3_max": 0.0, + "nauc_precision_at_3_std": 0.0, + "nauc_precision_at_3_diff1": 0.0, + "nauc_precision_at_5_max": 0.0, + "nauc_precision_at_5_std": 0.0, + "nauc_precision_at_5_diff1": 0.0, + "nauc_precision_at_10_max": 0.0, + "nauc_precision_at_10_std": 0.0, + "nauc_precision_at_10_diff1": 0.0, + "nauc_precision_at_20_max": 0.0, + "nauc_precision_at_20_std": 0.0, + "nauc_precision_at_20_diff1": 0.0, + "nauc_precision_at_100_max": -0.0, + "nauc_precision_at_100_std": -0.0, + "nauc_precision_at_100_diff1": -0.0, + "nauc_precision_at_1000_max": -0.0, + "nauc_precision_at_1000_std": -0.0, + "nauc_precision_at_1000_diff1": -0.0, + "nauc_mrr_at_1_max": 0.0, + "nauc_mrr_at_1_std": 0.0, + "nauc_mrr_at_1_diff1": 0.0, + "nauc_mrr_at_3_max": 0.0, + "nauc_mrr_at_3_std": 0.0, + "nauc_mrr_at_3_diff1": 0.0, + "nauc_mrr_at_5_max": 0.0, + "nauc_mrr_at_5_std": 0.0, + "nauc_mrr_at_5_diff1": 0.0, + "nauc_mrr_at_10_max": -0.0, + "nauc_mrr_at_10_std": -0.0, + "nauc_mrr_at_10_diff1": -0.0, + "nauc_mrr_at_20_max": 0.0, + "nauc_mrr_at_20_std": 0.0, + "nauc_mrr_at_20_diff1": 0.0, + "nauc_mrr_at_100_max": 0.0, + "nauc_mrr_at_100_std": 0.0, + "nauc_mrr_at_100_diff1": 0.0, + "nauc_mrr_at_1000_max": 0.0, + "nauc_mrr_at_1000_std": 0.0, + "nauc_mrr_at_1000_diff1": 0.0, + "hit_rate_at_1": 0.44578, + "hit_rate_at_3": 0.6988, + "hit_rate_at_5": 0.78916, + "hit_rate_at_10": 0.85843, + "hit_rate_at_20": 0.90964, + "hit_rate_at_100": 0.95181, + "hit_rate_at_1000": 0.95181, + "main_score": 0.49316, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.4262, + "ndcg_at_3": 0.45852, + "ndcg_at_5": 0.47153, + "ndcg_at_10": 0.49961, + "ndcg_at_20": 0.54233, + "ndcg_at_100": 0.5826, + "ndcg_at_1000": 0.5826, + "map_at_1": 0.192, + "map_at_3": 0.30086, + "map_at_5": 0.33961, + "map_at_10": 0.37856, + "map_at_20": 0.407, + "map_at_100": 0.42307, + "map_at_1000": 0.42307, + "recall_at_1": 0.192, + "recall_at_3": 0.37208, + "recall_at_5": 0.46018, + "recall_at_10": 0.57216, + "recall_at_20": 0.68498, + "recall_at_100": 0.80792, + "recall_at_1000": 0.80792, + "accuracy": 0.192, + "precision_at_1": 0.43373, + "precision_at_3": 0.33534, + "precision_at_5": 0.27289, + "precision_at_10": 0.19157, + "precision_at_20": 0.13057, + "precision_at_100": 0.03334, + "precision_at_1000": 0.00333, + "mrr_at_1": 0.433735, + "mrr_at_3": 0.562249, + "mrr_at_5": 0.583484, + "mrr_at_10": 0.591075, + "mrr_at_20": 0.594224, + "mrr_at_100": 0.595615, + "mrr_at_1000": 0.595615, + "nauc_ndcg_at_1_max": 0.0, + "nauc_ndcg_at_1_std": 0.0, + "nauc_ndcg_at_1_diff1": 0.0, + "nauc_ndcg_at_3_max": 0.0, + "nauc_ndcg_at_3_std": 0.0, + "nauc_ndcg_at_3_diff1": 0.0, + "nauc_ndcg_at_5_max": 0.0, + "nauc_ndcg_at_5_std": 0.0, + "nauc_ndcg_at_5_diff1": 0.0, + "nauc_ndcg_at_10_max": 0.0, + "nauc_ndcg_at_10_std": 0.0, + "nauc_ndcg_at_10_diff1": 0.0, + "nauc_ndcg_at_20_max": 0.0, + "nauc_ndcg_at_20_std": 0.0, + "nauc_ndcg_at_20_diff1": 0.0, + "nauc_ndcg_at_100_max": -0.0, + "nauc_ndcg_at_100_std": -0.0, + "nauc_ndcg_at_100_diff1": -0.0, + "nauc_ndcg_at_1000_max": -0.0, + "nauc_ndcg_at_1000_std": -0.0, + "nauc_ndcg_at_1000_diff1": -0.0, + "nauc_map_at_1_max": 0.0, + "nauc_map_at_1_std": 0.0, + "nauc_map_at_1_diff1": 0.0, + "nauc_map_at_3_max": 0.0, + "nauc_map_at_3_std": 0.0, + "nauc_map_at_3_diff1": 0.0, + "nauc_map_at_5_max": 0.0, + "nauc_map_at_5_std": 0.0, + "nauc_map_at_5_diff1": 0.0, + "nauc_map_at_10_max": 0.0, + "nauc_map_at_10_std": 0.0, + "nauc_map_at_10_diff1": 0.0, + "nauc_map_at_20_max": -0.0, + "nauc_map_at_20_std": -0.0, + "nauc_map_at_20_diff1": -0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": 0.0, + "nauc_recall_at_1_std": 0.0, + "nauc_recall_at_1_diff1": 0.0, + "nauc_recall_at_3_max": 0.0, + "nauc_recall_at_3_std": 0.0, + "nauc_recall_at_3_diff1": 0.0, + "nauc_recall_at_5_max": 0.0, + "nauc_recall_at_5_std": 0.0, + "nauc_recall_at_5_diff1": 0.0, + "nauc_recall_at_10_max": 0.0, + "nauc_recall_at_10_std": 0.0, + "nauc_recall_at_10_diff1": 0.0, + "nauc_recall_at_20_max": 0.0, + "nauc_recall_at_20_std": 0.0, + "nauc_recall_at_20_diff1": 0.0, + "nauc_recall_at_100_max": 0.0, + "nauc_recall_at_100_std": 0.0, + "nauc_recall_at_100_diff1": 0.0, + "nauc_recall_at_1000_max": 0.0, + "nauc_recall_at_1000_std": 0.0, + "nauc_recall_at_1000_diff1": 0.0, + "nauc_precision_at_1_max": 0.0, + "nauc_precision_at_1_std": 0.0, + "nauc_precision_at_1_diff1": 0.0, + "nauc_precision_at_3_max": -0.0, + "nauc_precision_at_3_std": -0.0, + "nauc_precision_at_3_diff1": -0.0, + "nauc_precision_at_5_max": 0.0, + "nauc_precision_at_5_std": 0.0, + "nauc_precision_at_5_diff1": 0.0, + "nauc_precision_at_10_max": 0.0, + "nauc_precision_at_10_std": 0.0, + "nauc_precision_at_10_diff1": 0.0, + "nauc_precision_at_20_max": 0.0, + "nauc_precision_at_20_std": 0.0, + "nauc_precision_at_20_diff1": 0.0, + "nauc_precision_at_100_max": -0.0, + "nauc_precision_at_100_std": -0.0, + "nauc_precision_at_100_diff1": -0.0, + "nauc_precision_at_1000_max": -0.0, + "nauc_precision_at_1000_std": -0.0, + "nauc_precision_at_1000_diff1": -0.0, + "nauc_mrr_at_1_max": 0.0, + "nauc_mrr_at_1_std": 0.0, + "nauc_mrr_at_1_diff1": 0.0, + "nauc_mrr_at_3_max": 0.0, + "nauc_mrr_at_3_std": 0.0, + "nauc_mrr_at_3_diff1": 0.0, + "nauc_mrr_at_5_max": 0.0, + "nauc_mrr_at_5_std": 0.0, + "nauc_mrr_at_5_diff1": 0.0, + "nauc_mrr_at_10_max": 0.0, + "nauc_mrr_at_10_std": 0.0, + "nauc_mrr_at_10_diff1": 0.0, + "nauc_mrr_at_20_max": -0.0, + "nauc_mrr_at_20_std": -0.0, + "nauc_mrr_at_20_diff1": -0.0, + "nauc_mrr_at_100_max": 0.0, + "nauc_mrr_at_100_std": 0.0, + "nauc_mrr_at_100_diff1": 0.0, + "nauc_mrr_at_1000_max": 0.0, + "nauc_mrr_at_1000_std": 0.0, + "nauc_mrr_at_1000_diff1": 0.0, + "hit_rate_at_1": 0.43373, + "hit_rate_at_3": 0.71988, + "hit_rate_at_5": 0.81325, + "hit_rate_at_10": 0.87048, + "hit_rate_at_20": 0.91265, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.49961, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.43072, + "ndcg_at_3": 0.45906, + "ndcg_at_5": 0.47487, + "ndcg_at_10": 0.50486, + "ndcg_at_20": 0.54562, + "ndcg_at_100": 0.58412, + "ndcg_at_1000": 0.58412, + "map_at_1": 0.19452, + "map_at_3": 0.30409, + "map_at_5": 0.3448, + "map_at_10": 0.38374, + "map_at_20": 0.41121, + "map_at_100": 0.42711, + "map_at_1000": 0.42711, + "recall_at_1": 0.19452, + "recall_at_3": 0.37651, + "recall_at_5": 0.46633, + "recall_at_10": 0.57862, + "recall_at_20": 0.6882, + "recall_at_100": 0.80575, + "recall_at_1000": 0.80575, + "accuracy": 0.19452, + "precision_at_1": 0.43976, + "precision_at_3": 0.33735, + "precision_at_5": 0.27711, + "precision_at_10": 0.19729, + "precision_at_20": 0.13102, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.439759, + "mrr_at_3": 0.565763, + "mrr_at_5": 0.583685, + "mrr_at_10": 0.592166, + "mrr_at_20": 0.595166, + "mrr_at_100": 0.596463, + "mrr_at_1000": 0.596463, + "nauc_ndcg_at_1_max": -0.0, + "nauc_ndcg_at_1_std": -0.0, + "nauc_ndcg_at_1_diff1": -0.0, + "nauc_ndcg_at_3_max": 0.0, + "nauc_ndcg_at_3_std": 0.0, + "nauc_ndcg_at_3_diff1": 0.0, + "nauc_ndcg_at_5_max": 0.0, + "nauc_ndcg_at_5_std": 0.0, + "nauc_ndcg_at_5_diff1": 0.0, + "nauc_ndcg_at_10_max": 0.0, + "nauc_ndcg_at_10_std": 0.0, + "nauc_ndcg_at_10_diff1": 0.0, + "nauc_ndcg_at_20_max": 0.0, + "nauc_ndcg_at_20_std": 0.0, + "nauc_ndcg_at_20_diff1": 0.0, + "nauc_ndcg_at_100_max": 0.0, + "nauc_ndcg_at_100_std": 0.0, + "nauc_ndcg_at_100_diff1": 0.0, + "nauc_ndcg_at_1000_max": 0.0, + "nauc_ndcg_at_1000_std": 0.0, + "nauc_ndcg_at_1000_diff1": 0.0, + "nauc_map_at_1_max": -0.0, + "nauc_map_at_1_std": -0.0, + "nauc_map_at_1_diff1": -0.0, + "nauc_map_at_3_max": 0.0, + "nauc_map_at_3_std": 0.0, + "nauc_map_at_3_diff1": 0.0, + "nauc_map_at_5_max": 0.0, + "nauc_map_at_5_std": 0.0, + "nauc_map_at_5_diff1": 0.0, + "nauc_map_at_10_max": -0.0, + "nauc_map_at_10_std": -0.0, + "nauc_map_at_10_diff1": -0.0, + "nauc_map_at_20_max": 0.0, + "nauc_map_at_20_std": 0.0, + "nauc_map_at_20_diff1": 0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": -0.0, + "nauc_recall_at_1_std": -0.0, + "nauc_recall_at_1_diff1": -0.0, + "nauc_recall_at_3_max": 0.0, + "nauc_recall_at_3_std": 0.0, + "nauc_recall_at_3_diff1": 0.0, + "nauc_recall_at_5_max": 0.0, + "nauc_recall_at_5_std": 0.0, + "nauc_recall_at_5_diff1": 0.0, + "nauc_recall_at_10_max": 0.0, + "nauc_recall_at_10_std": 0.0, + "nauc_recall_at_10_diff1": 0.0, + "nauc_recall_at_20_max": -0.0, + "nauc_recall_at_20_std": -0.0, + "nauc_recall_at_20_diff1": -0.0, + "nauc_recall_at_100_max": 0.0, + "nauc_recall_at_100_std": 0.0, + "nauc_recall_at_100_diff1": 0.0, + "nauc_recall_at_1000_max": 0.0, + "nauc_recall_at_1000_std": 0.0, + "nauc_recall_at_1000_diff1": 0.0, + "nauc_precision_at_1_max": 0.0, + "nauc_precision_at_1_std": 0.0, + "nauc_precision_at_1_diff1": 0.0, + "nauc_precision_at_3_max": -0.0, + "nauc_precision_at_3_std": -0.0, + "nauc_precision_at_3_diff1": -0.0, + "nauc_precision_at_5_max": 0.0, + "nauc_precision_at_5_std": 0.0, + "nauc_precision_at_5_diff1": 0.0, + "nauc_precision_at_10_max": 0.0, + "nauc_precision_at_10_std": 0.0, + "nauc_precision_at_10_diff1": 0.0, + "nauc_precision_at_20_max": -0.0, + "nauc_precision_at_20_std": -0.0, + "nauc_precision_at_20_diff1": -0.0, + "nauc_precision_at_100_max": 0.0, + "nauc_precision_at_100_std": 0.0, + "nauc_precision_at_100_diff1": 0.0, + "nauc_precision_at_1000_max": -0.0, + "nauc_precision_at_1000_std": -0.0, + "nauc_precision_at_1000_diff1": -0.0, + "nauc_mrr_at_1_max": 0.0, + "nauc_mrr_at_1_std": 0.0, + "nauc_mrr_at_1_diff1": 0.0, + "nauc_mrr_at_3_max": 0.0, + "nauc_mrr_at_3_std": 0.0, + "nauc_mrr_at_3_diff1": 0.0, + "nauc_mrr_at_5_max": 0.0, + "nauc_mrr_at_5_std": 0.0, + "nauc_mrr_at_5_diff1": 0.0, + "nauc_mrr_at_10_max": 0.0, + "nauc_mrr_at_10_std": 0.0, + "nauc_mrr_at_10_diff1": 0.0, + "nauc_mrr_at_20_max": 0.0, + "nauc_mrr_at_20_std": 0.0, + "nauc_mrr_at_20_diff1": 0.0, + "nauc_mrr_at_100_max": 0.0, + "nauc_mrr_at_100_std": 0.0, + "nauc_mrr_at_100_diff1": 0.0, + "nauc_mrr_at_1000_max": 0.0, + "nauc_mrr_at_1000_std": 0.0, + "nauc_mrr_at_1000_diff1": 0.0, + "hit_rate_at_1": 0.43976, + "hit_rate_at_3": 0.72289, + "hit_rate_at_5": 0.8012, + "hit_rate_at_10": 0.86446, + "hit_rate_at_20": 0.90964, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.50486, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.43072, + "ndcg_at_3": 0.46276, + "ndcg_at_5": 0.47723, + "ndcg_at_10": 0.50533, + "ndcg_at_20": 0.54787, + "ndcg_at_100": 0.58638, + "ndcg_at_1000": 0.58638, + "map_at_1": 0.19934, + "map_at_3": 0.3071, + "map_at_5": 0.34765, + "map_at_10": 0.38497, + "map_at_20": 0.41422, + "map_at_100": 0.43017, + "map_at_1000": 0.43017, + "recall_at_1": 0.19934, + "recall_at_3": 0.38419, + "recall_at_5": 0.47115, + "recall_at_10": 0.57936, + "recall_at_20": 0.69513, + "recall_at_100": 0.80599, + "recall_at_1000": 0.80599, + "accuracy": 0.19934, + "precision_at_1": 0.43976, + "precision_at_3": 0.34137, + "precision_at_5": 0.27711, + "precision_at_10": 0.19367, + "precision_at_20": 0.13087, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.439759, + "mrr_at_3": 0.571787, + "mrr_at_5": 0.584287, + "mrr_at_10": 0.59378, + "mrr_at_20": 0.596612, + "mrr_at_100": 0.598294, + "mrr_at_1000": 0.598294, + "nauc_ndcg_at_1_max": -0.0, + "nauc_ndcg_at_1_std": -0.0, + "nauc_ndcg_at_1_diff1": -0.0, + "nauc_ndcg_at_3_max": 0.0, + "nauc_ndcg_at_3_std": 0.0, + "nauc_ndcg_at_3_diff1": 0.0, + "nauc_ndcg_at_5_max": 0.0, + "nauc_ndcg_at_5_std": 0.0, + "nauc_ndcg_at_5_diff1": 0.0, + "nauc_ndcg_at_10_max": 0.0, + "nauc_ndcg_at_10_std": 0.0, + "nauc_ndcg_at_10_diff1": 0.0, + "nauc_ndcg_at_20_max": 0.0, + "nauc_ndcg_at_20_std": 0.0, + "nauc_ndcg_at_20_diff1": 0.0, + "nauc_ndcg_at_100_max": -0.0, + "nauc_ndcg_at_100_std": -0.0, + "nauc_ndcg_at_100_diff1": -0.0, + "nauc_ndcg_at_1000_max": -0.0, + "nauc_ndcg_at_1000_std": -0.0, + "nauc_ndcg_at_1000_diff1": -0.0, + "nauc_map_at_1_max": 0.0, + "nauc_map_at_1_std": 0.0, + "nauc_map_at_1_diff1": 0.0, + "nauc_map_at_3_max": 0.0, + "nauc_map_at_3_std": 0.0, + "nauc_map_at_3_diff1": 0.0, + "nauc_map_at_5_max": 0.0, + "nauc_map_at_5_std": 0.0, + "nauc_map_at_5_diff1": 0.0, + "nauc_map_at_10_max": 0.0, + "nauc_map_at_10_std": 0.0, + "nauc_map_at_10_diff1": 0.0, + "nauc_map_at_20_max": -0.0, + "nauc_map_at_20_std": -0.0, + "nauc_map_at_20_diff1": -0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": 0.0, + "nauc_recall_at_1_std": 0.0, + "nauc_recall_at_1_diff1": 0.0, + "nauc_recall_at_3_max": -0.0, + "nauc_recall_at_3_std": -0.0, + "nauc_recall_at_3_diff1": -0.0, + "nauc_recall_at_5_max": 0.0, + "nauc_recall_at_5_std": 0.0, + "nauc_recall_at_5_diff1": 0.0, + "nauc_recall_at_10_max": 0.0, + "nauc_recall_at_10_std": 0.0, + "nauc_recall_at_10_diff1": 0.0, + "nauc_recall_at_20_max": -0.0, + "nauc_recall_at_20_std": -0.0, + "nauc_recall_at_20_diff1": -0.0, + "nauc_recall_at_100_max": 0.0, + "nauc_recall_at_100_std": 0.0, + "nauc_recall_at_100_diff1": 0.0, + "nauc_recall_at_1000_max": 0.0, + "nauc_recall_at_1000_std": 0.0, + "nauc_recall_at_1000_diff1": 0.0, + "nauc_precision_at_1_max": 0.0, + "nauc_precision_at_1_std": 0.0, + "nauc_precision_at_1_diff1": 0.0, + "nauc_precision_at_3_max": -0.0, + "nauc_precision_at_3_std": -0.0, + "nauc_precision_at_3_diff1": -0.0, + "nauc_precision_at_5_max": 0.0, + "nauc_precision_at_5_std": 0.0, + "nauc_precision_at_5_diff1": 0.0, + "nauc_precision_at_10_max": 0.0, + "nauc_precision_at_10_std": 0.0, + "nauc_precision_at_10_diff1": 0.0, + "nauc_precision_at_20_max": -0.0, + "nauc_precision_at_20_std": -0.0, + "nauc_precision_at_20_diff1": -0.0, + "nauc_precision_at_100_max": -0.0, + "nauc_precision_at_100_std": -0.0, + "nauc_precision_at_100_diff1": -0.0, + "nauc_precision_at_1000_max": -0.0, + "nauc_precision_at_1000_std": -0.0, + "nauc_precision_at_1000_diff1": -0.0, + "nauc_mrr_at_1_max": 0.0, + "nauc_mrr_at_1_std": 0.0, + "nauc_mrr_at_1_diff1": 0.0, + "nauc_mrr_at_3_max": -0.0, + "nauc_mrr_at_3_std": -0.0, + "nauc_mrr_at_3_diff1": -0.0, + "nauc_mrr_at_5_max": 0.0, + "nauc_mrr_at_5_std": 0.0, + "nauc_mrr_at_5_diff1": 0.0, + "nauc_mrr_at_10_max": 0.0, + "nauc_mrr_at_10_std": 0.0, + "nauc_mrr_at_10_diff1": 0.0, + "nauc_mrr_at_20_max": 0.0, + "nauc_mrr_at_20_std": 0.0, + "nauc_mrr_at_20_diff1": 0.0, + "nauc_mrr_at_100_max": -0.0, + "nauc_mrr_at_100_std": -0.0, + "nauc_mrr_at_100_diff1": -0.0, + "nauc_mrr_at_1000_max": -0.0, + "nauc_mrr_at_1000_std": -0.0, + "nauc_mrr_at_1000_diff1": -0.0, + "hit_rate_at_1": 0.43976, + "hit_rate_at_3": 0.74096, + "hit_rate_at_5": 0.79518, + "hit_rate_at_10": 0.86145, + "hit_rate_at_20": 0.9006, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.50533, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 45.449772772029974, + "kg_co2_emissions": null, + "date": 1792440763.648874, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 5.768029950559139e-06, + "end": 32.74266313994303, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 32.74269673996605, + "end": 32.742698963033035, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries and documents", + "start": 32.74703257903457, + "end": 34.90295077499468, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 34.9029697739752, + "end": 34.94779616501182, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries and documents", + "start": 34.95848434197251, + "end": 36.97320925502572, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 36.97328525898047, + "end": 37.01344270200934, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries and documents", + "start": 37.03408805199433, + "end": 39.061605321010575, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 39.0616400729632, + "end": 39.10052892798558, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries and documents", + "start": 39.11239247396588, + "end": 41.14616882102564, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 41.14622196997516, + "end": 41.18618567404337, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries and documents", + "start": 41.202507872949354, + "end": 43.24475708196405, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 43.24479061202146, + "end": 43.28426540293731, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries and documents", + "start": 43.2971780099906, + "end": 45.402164465980604, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 45.40217810694594, + "end": 45.44020198704675, + "split": "test", + "subset": "portuguese" + } + ] +} diff --git a/results/mteb__baseline-no-reranking/2/Vidore3TelecomNemotron35Embed8BReranking.v2.json b/results/mteb__baseline-no-reranking/2/Vidore3TelecomNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..d0ed552bb1 --- /dev/null +++ b/results/mteb__baseline-no-reranking/2/Vidore3TelecomNemotron35Embed8BReranking.v2.json @@ -0,0 +1,1048 @@ +{ + "dataset_revision": "47c9b703527792f3fbf0a082f4589ff40c1aa8cc", + "task_name": "Vidore3TelecomNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.70402, + "ndcg_at_3": 0.68287, + "ndcg_at_5": 0.68639, + "ndcg_at_10": 0.71247, + "ndcg_at_20": 0.73941, + "ndcg_at_100": 0.76846, + "ndcg_at_1000": 0.76846, + "map_at_1": 0.29433, + "map_at_3": 0.47917, + "map_at_5": 0.53778, + "map_at_10": 0.58944, + "map_at_20": 0.61739, + "map_at_100": 0.63316, + "map_at_1000": 0.63316, + "recall_at_1": 0.29433, + "recall_at_3": 0.54146, + "recall_at_5": 0.63739, + "recall_at_10": 0.75784, + "recall_at_20": 0.84383, + "recall_at_100": 0.92961, + "recall_at_1000": 0.92961, + "accuracy": 0.29433, + "precision_at_1": 0.72701, + "precision_at_3": 0.51724, + "precision_at_5": 0.40287, + "precision_at_10": 0.2681, + "precision_at_20": 0.16336, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.727011, + "mrr_at_3": 0.808429, + "mrr_at_5": 0.816475, + "mrr_at_10": 0.820999, + "mrr_at_20": 0.821963, + "mrr_at_100": 0.822165, + "mrr_at_1000": 0.822165, + "nauc_ndcg_at_1_max": -0.0, + "nauc_ndcg_at_1_std": -0.0, + "nauc_ndcg_at_1_diff1": -0.0, + "nauc_ndcg_at_3_max": 0.0, + "nauc_ndcg_at_3_std": 0.0, + "nauc_ndcg_at_3_diff1": 0.0, + "nauc_ndcg_at_5_max": 0.0, + "nauc_ndcg_at_5_std": 0.0, + "nauc_ndcg_at_5_diff1": 0.0, + "nauc_ndcg_at_10_max": 0.0, + "nauc_ndcg_at_10_std": 0.0, + "nauc_ndcg_at_10_diff1": 0.0, + "nauc_ndcg_at_20_max": -0.0, + "nauc_ndcg_at_20_std": -0.0, + "nauc_ndcg_at_20_diff1": -0.0, + "nauc_ndcg_at_100_max": 0.0, + "nauc_ndcg_at_100_std": 0.0, + "nauc_ndcg_at_100_diff1": 0.0, + "nauc_ndcg_at_1000_max": 0.0, + "nauc_ndcg_at_1000_std": 0.0, + "nauc_ndcg_at_1000_diff1": 0.0, + "nauc_map_at_1_max": -0.0, + "nauc_map_at_1_std": -0.0, + "nauc_map_at_1_diff1": -0.0, + "nauc_map_at_3_max": 0.0, + "nauc_map_at_3_std": 0.0, + "nauc_map_at_3_diff1": 0.0, + "nauc_map_at_5_max": 0.0, + "nauc_map_at_5_std": 0.0, + "nauc_map_at_5_diff1": 0.0, + "nauc_map_at_10_max": 0.0, + "nauc_map_at_10_std": 0.0, + "nauc_map_at_10_diff1": 0.0, + "nauc_map_at_20_max": -0.0, + "nauc_map_at_20_std": -0.0, + "nauc_map_at_20_diff1": -0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": -0.0, + "nauc_recall_at_1_std": -0.0, + "nauc_recall_at_1_diff1": -0.0, + "nauc_recall_at_3_max": 0.0, + "nauc_recall_at_3_std": 0.0, + "nauc_recall_at_3_diff1": 0.0, + "nauc_recall_at_5_max": 0.0, + "nauc_recall_at_5_std": 0.0, + "nauc_recall_at_5_diff1": 0.0, + "nauc_recall_at_10_max": -0.0, + "nauc_recall_at_10_std": -0.0, + "nauc_recall_at_10_diff1": -0.0, + "nauc_recall_at_20_max": 0.0, + "nauc_recall_at_20_std": 0.0, + "nauc_recall_at_20_diff1": 0.0, + "nauc_recall_at_100_max": 0.0, + "nauc_recall_at_100_std": 0.0, + "nauc_recall_at_100_diff1": 0.0, + "nauc_recall_at_1000_max": 0.0, + "nauc_recall_at_1000_std": 0.0, + "nauc_recall_at_1000_diff1": 0.0, + "nauc_precision_at_1_max": 0.0, + "nauc_precision_at_1_std": 0.0, + "nauc_precision_at_1_diff1": 0.0, + "nauc_precision_at_3_max": 0.0, + "nauc_precision_at_3_std": 0.0, + "nauc_precision_at_3_diff1": 0.0, + "nauc_precision_at_5_max": 0.0, + "nauc_precision_at_5_std": 0.0, + "nauc_precision_at_5_diff1": 0.0, + "nauc_precision_at_10_max": 0.0, + "nauc_precision_at_10_std": 0.0, + "nauc_precision_at_10_diff1": 0.0, + "nauc_precision_at_20_max": 0.0, + "nauc_precision_at_20_std": 0.0, + "nauc_precision_at_20_diff1": 0.0, + "nauc_precision_at_100_max": 0.0, + "nauc_precision_at_100_std": 0.0, + "nauc_precision_at_100_diff1": 0.0, + "nauc_precision_at_1000_max": 0.0, + "nauc_precision_at_1000_std": 0.0, + "nauc_precision_at_1000_diff1": 0.0, + "nauc_mrr_at_1_max": 0.0, + "nauc_mrr_at_1_std": 0.0, + "nauc_mrr_at_1_diff1": 0.0, + "nauc_mrr_at_3_max": -0.0, + "nauc_mrr_at_3_std": -0.0, + "nauc_mrr_at_3_diff1": -0.0, + "nauc_mrr_at_5_max": 0.0, + "nauc_mrr_at_5_std": 0.0, + "nauc_mrr_at_5_diff1": 0.0, + "nauc_mrr_at_10_max": -0.0, + "nauc_mrr_at_10_std": -0.0, + "nauc_mrr_at_10_diff1": -0.0, + "nauc_mrr_at_20_max": 0.0, + "nauc_mrr_at_20_std": 0.0, + "nauc_mrr_at_20_diff1": 0.0, + "nauc_mrr_at_100_max": -0.0, + "nauc_mrr_at_100_std": -0.0, + "nauc_mrr_at_100_diff1": -0.0, + "nauc_mrr_at_1000_max": -0.0, + "nauc_mrr_at_1000_std": -0.0, + "nauc_mrr_at_1000_diff1": -0.0, + "hit_rate_at_1": 0.72701, + "hit_rate_at_3": 0.90805, + "hit_rate_at_5": 0.94253, + "hit_rate_at_10": 0.97414, + "hit_rate_at_20": 0.98851, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.71247, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.67816, + "ndcg_at_3": 0.66715, + "ndcg_at_5": 0.67255, + "ndcg_at_10": 0.6921, + "ndcg_at_20": 0.72464, + "ndcg_at_100": 0.75599, + "ndcg_at_1000": 0.75599, + "map_at_1": 0.29531, + "map_at_3": 0.46974, + "map_at_5": 0.52443, + "map_at_10": 0.57054, + "map_at_20": 0.6003, + "map_at_100": 0.61694, + "map_at_1000": 0.61694, + "recall_at_1": 0.29531, + "recall_at_3": 0.52675, + "recall_at_5": 0.6248, + "recall_at_10": 0.73072, + "recall_at_20": 0.82902, + "recall_at_100": 0.92187, + "recall_at_1000": 0.92187, + "accuracy": 0.29531, + "precision_at_1": 0.7069, + "precision_at_3": 0.50383, + "precision_at_5": 0.38908, + "precision_at_10": 0.25345, + "precision_at_20": 0.15934, + "precision_at_100": 0.03882, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.706897, + "mrr_at_3": 0.792146, + "mrr_at_5": 0.80091, + "mrr_at_10": 0.805873, + "mrr_at_20": 0.806898, + "mrr_at_100": 0.807258, + "mrr_at_1000": 0.807258, + "nauc_ndcg_at_1_max": 0.0, + "nauc_ndcg_at_1_std": 0.0, + "nauc_ndcg_at_1_diff1": 0.0, + "nauc_ndcg_at_3_max": 0.0, + "nauc_ndcg_at_3_std": 0.0, + "nauc_ndcg_at_3_diff1": 0.0, + "nauc_ndcg_at_5_max": 0.0, + "nauc_ndcg_at_5_std": 0.0, + "nauc_ndcg_at_5_diff1": 0.0, + "nauc_ndcg_at_10_max": 0.0, + "nauc_ndcg_at_10_std": 0.0, + "nauc_ndcg_at_10_diff1": 0.0, + "nauc_ndcg_at_20_max": 0.0, + "nauc_ndcg_at_20_std": 0.0, + "nauc_ndcg_at_20_diff1": 0.0, + "nauc_ndcg_at_100_max": -0.0, + "nauc_ndcg_at_100_std": -0.0, + "nauc_ndcg_at_100_diff1": -0.0, + "nauc_ndcg_at_1000_max": -0.0, + "nauc_ndcg_at_1000_std": -0.0, + "nauc_ndcg_at_1000_diff1": -0.0, + "nauc_map_at_1_max": 0.0, + "nauc_map_at_1_std": 0.0, + "nauc_map_at_1_diff1": 0.0, + "nauc_map_at_3_max": 0.0, + "nauc_map_at_3_std": 0.0, + "nauc_map_at_3_diff1": 0.0, + "nauc_map_at_5_max": -0.0, + "nauc_map_at_5_std": -0.0, + "nauc_map_at_5_diff1": -0.0, + "nauc_map_at_10_max": 0.0, + "nauc_map_at_10_std": 0.0, + "nauc_map_at_10_diff1": 0.0, + "nauc_map_at_20_max": -0.0, + "nauc_map_at_20_std": -0.0, + "nauc_map_at_20_diff1": -0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": 0.0, + "nauc_recall_at_1_std": 0.0, + "nauc_recall_at_1_diff1": 0.0, + "nauc_recall_at_3_max": 0.0, + "nauc_recall_at_3_std": 0.0, + "nauc_recall_at_3_diff1": 0.0, + "nauc_recall_at_5_max": 0.0, + "nauc_recall_at_5_std": 0.0, + "nauc_recall_at_5_diff1": 0.0, + "nauc_recall_at_10_max": -0.0, + "nauc_recall_at_10_std": -0.0, + "nauc_recall_at_10_diff1": -0.0, + "nauc_recall_at_20_max": -0.0, + "nauc_recall_at_20_std": -0.0, + "nauc_recall_at_20_diff1": -0.0, + "nauc_recall_at_100_max": 0.0, + "nauc_recall_at_100_std": 0.0, + "nauc_recall_at_100_diff1": 0.0, + "nauc_recall_at_1000_max": 0.0, + "nauc_recall_at_1000_std": 0.0, + "nauc_recall_at_1000_diff1": 0.0, + "nauc_precision_at_1_max": 0.0, + "nauc_precision_at_1_std": 0.0, + "nauc_precision_at_1_diff1": 0.0, + "nauc_precision_at_3_max": -0.0, + "nauc_precision_at_3_std": -0.0, + "nauc_precision_at_3_diff1": -0.0, + "nauc_precision_at_5_max": 0.0, + "nauc_precision_at_5_std": 0.0, + "nauc_precision_at_5_diff1": 0.0, + "nauc_precision_at_10_max": 0.0, + "nauc_precision_at_10_std": 0.0, + "nauc_precision_at_10_diff1": 0.0, + "nauc_precision_at_20_max": -0.0, + "nauc_precision_at_20_std": -0.0, + "nauc_precision_at_20_diff1": -0.0, + "nauc_precision_at_100_max": 0.0, + "nauc_precision_at_100_std": 0.0, + "nauc_precision_at_100_diff1": 0.0, + "nauc_precision_at_1000_max": -0.0, + "nauc_precision_at_1000_std": -0.0, + "nauc_precision_at_1000_diff1": -0.0, + "nauc_mrr_at_1_max": 0.0, + "nauc_mrr_at_1_std": 0.0, + "nauc_mrr_at_1_diff1": 0.0, + "nauc_mrr_at_3_max": 0.0, + "nauc_mrr_at_3_std": 0.0, + "nauc_mrr_at_3_diff1": 0.0, + "nauc_mrr_at_5_max": -0.0, + "nauc_mrr_at_5_std": -0.0, + "nauc_mrr_at_5_diff1": -0.0, + "nauc_mrr_at_10_max": -0.0, + "nauc_mrr_at_10_std": -0.0, + "nauc_mrr_at_10_diff1": -0.0, + "nauc_mrr_at_20_max": 0.0, + "nauc_mrr_at_20_std": 0.0, + "nauc_mrr_at_20_diff1": 0.0, + "nauc_mrr_at_100_max": 0.0, + "nauc_mrr_at_100_std": 0.0, + "nauc_mrr_at_100_diff1": 0.0, + "nauc_mrr_at_1000_max": 0.0, + "nauc_mrr_at_1000_std": 0.0, + "nauc_mrr_at_1000_diff1": 0.0, + "hit_rate_at_1": 0.7069, + "hit_rate_at_3": 0.89655, + "hit_rate_at_5": 0.93391, + "hit_rate_at_10": 0.96839, + "hit_rate_at_20": 0.98276, + "hit_rate_at_100": 0.99138, + "hit_rate_at_1000": 0.99138, + "main_score": 0.6921, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.69109, + "ndcg_at_3": 0.67055, + "ndcg_at_5": 0.67407, + "ndcg_at_10": 0.7016, + "ndcg_at_20": 0.73073, + "ndcg_at_100": 0.76237, + "ndcg_at_1000": 0.76237, + "map_at_1": 0.29968, + "map_at_3": 0.4702, + "map_at_5": 0.52619, + "map_at_10": 0.57797, + "map_at_20": 0.60601, + "map_at_100": 0.62284, + "map_at_1000": 0.62284, + "recall_at_1": 0.29968, + "recall_at_3": 0.52488, + "recall_at_5": 0.62134, + "recall_at_10": 0.74589, + "recall_at_20": 0.8343, + "recall_at_100": 0.93008, + "recall_at_1000": 0.93008, + "accuracy": 0.29968, + "precision_at_1": 0.71264, + "precision_at_3": 0.50287, + "precision_at_5": 0.38966, + "precision_at_10": 0.26121, + "precision_at_20": 0.16092, + "precision_at_100": 0.03905, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.712644, + "mrr_at_3": 0.791667, + "mrr_at_5": 0.801006, + "mrr_at_10": 0.806891, + "mrr_at_20": 0.807557, + "mrr_at_100": 0.808142, + "mrr_at_1000": 0.808142, + "nauc_ndcg_at_1_max": 0.0, + "nauc_ndcg_at_1_std": 0.0, + "nauc_ndcg_at_1_diff1": 0.0, + "nauc_ndcg_at_3_max": 0.0, + "nauc_ndcg_at_3_std": 0.0, + "nauc_ndcg_at_3_diff1": 0.0, + "nauc_ndcg_at_5_max": -0.0, + "nauc_ndcg_at_5_std": -0.0, + "nauc_ndcg_at_5_diff1": -0.0, + "nauc_ndcg_at_10_max": -0.0, + "nauc_ndcg_at_10_std": -0.0, + "nauc_ndcg_at_10_diff1": -0.0, + "nauc_ndcg_at_20_max": 0.0, + "nauc_ndcg_at_20_std": 0.0, + "nauc_ndcg_at_20_diff1": 0.0, + "nauc_ndcg_at_100_max": 0.0, + "nauc_ndcg_at_100_std": 0.0, + "nauc_ndcg_at_100_diff1": 0.0, + "nauc_ndcg_at_1000_max": 0.0, + "nauc_ndcg_at_1000_std": 0.0, + "nauc_ndcg_at_1000_diff1": 0.0, + "nauc_map_at_1_max": 0.0, + "nauc_map_at_1_std": 0.0, + "nauc_map_at_1_diff1": 0.0, + "nauc_map_at_3_max": -0.0, + "nauc_map_at_3_std": -0.0, + "nauc_map_at_3_diff1": -0.0, + "nauc_map_at_5_max": 0.0, + "nauc_map_at_5_std": 0.0, + "nauc_map_at_5_diff1": 0.0, + "nauc_map_at_10_max": -0.0, + "nauc_map_at_10_std": -0.0, + "nauc_map_at_10_diff1": -0.0, + "nauc_map_at_20_max": -0.0, + "nauc_map_at_20_std": -0.0, + "nauc_map_at_20_diff1": -0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": 0.0, + "nauc_recall_at_1_std": 0.0, + "nauc_recall_at_1_diff1": 0.0, + "nauc_recall_at_3_max": 0.0, + "nauc_recall_at_3_std": 0.0, + "nauc_recall_at_3_diff1": 0.0, + "nauc_recall_at_5_max": 0.0, + "nauc_recall_at_5_std": 0.0, + "nauc_recall_at_5_diff1": 0.0, + "nauc_recall_at_10_max": -0.0, + "nauc_recall_at_10_std": -0.0, + "nauc_recall_at_10_diff1": -0.0, + "nauc_recall_at_20_max": 0.0, + "nauc_recall_at_20_std": 0.0, + "nauc_recall_at_20_diff1": 0.0, + "nauc_recall_at_100_max": 0.0, + "nauc_recall_at_100_std": 0.0, + "nauc_recall_at_100_diff1": 0.0, + "nauc_recall_at_1000_max": 0.0, + "nauc_recall_at_1000_std": 0.0, + "nauc_recall_at_1000_diff1": 0.0, + "nauc_precision_at_1_max": -0.0, + "nauc_precision_at_1_std": -0.0, + "nauc_precision_at_1_diff1": -0.0, + "nauc_precision_at_3_max": -0.0, + "nauc_precision_at_3_std": -0.0, + "nauc_precision_at_3_diff1": -0.0, + "nauc_precision_at_5_max": -0.0, + "nauc_precision_at_5_std": -0.0, + "nauc_precision_at_5_diff1": -0.0, + "nauc_precision_at_10_max": -0.0, + "nauc_precision_at_10_std": -0.0, + "nauc_precision_at_10_diff1": -0.0, + "nauc_precision_at_20_max": -0.0, + "nauc_precision_at_20_std": -0.0, + "nauc_precision_at_20_diff1": -0.0, + "nauc_precision_at_100_max": -0.0, + "nauc_precision_at_100_std": -0.0, + "nauc_precision_at_100_diff1": -0.0, + "nauc_precision_at_1000_max": -0.0, + "nauc_precision_at_1000_std": -0.0, + "nauc_precision_at_1000_diff1": -0.0, + "nauc_mrr_at_1_max": -0.0, + "nauc_mrr_at_1_std": -0.0, + "nauc_mrr_at_1_diff1": -0.0, + "nauc_mrr_at_3_max": 0.0, + "nauc_mrr_at_3_std": 0.0, + "nauc_mrr_at_3_diff1": 0.0, + "nauc_mrr_at_5_max": 0.0, + "nauc_mrr_at_5_std": 0.0, + "nauc_mrr_at_5_diff1": 0.0, + "nauc_mrr_at_10_max": 0.0, + "nauc_mrr_at_10_std": 0.0, + "nauc_mrr_at_10_diff1": 0.0, + "nauc_mrr_at_20_max": -0.0, + "nauc_mrr_at_20_std": -0.0, + "nauc_mrr_at_20_diff1": -0.0, + "nauc_mrr_at_100_max": -0.0, + "nauc_mrr_at_100_std": -0.0, + "nauc_mrr_at_100_diff1": -0.0, + "nauc_mrr_at_1000_max": -0.0, + "nauc_mrr_at_1000_std": -0.0, + "nauc_mrr_at_1000_diff1": -0.0, + "hit_rate_at_1": 0.71264, + "hit_rate_at_3": 0.88793, + "hit_rate_at_5": 0.92816, + "hit_rate_at_10": 0.97126, + "hit_rate_at_20": 0.97989, + "hit_rate_at_100": 0.99713, + "hit_rate_at_1000": 0.99713, + "main_score": 0.7016, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.68822, + "ndcg_at_3": 0.67072, + "ndcg_at_5": 0.67193, + "ndcg_at_10": 0.69941, + "ndcg_at_20": 0.72958, + "ndcg_at_100": 0.76203, + "ndcg_at_1000": 0.76203, + "map_at_1": 0.30044, + "map_at_3": 0.47351, + "map_at_5": 0.52589, + "map_at_10": 0.57799, + "map_at_20": 0.60693, + "map_at_100": 0.62329, + "map_at_1000": 0.62329, + "recall_at_1": 0.30044, + "recall_at_3": 0.53049, + "recall_at_5": 0.61955, + "recall_at_10": 0.74251, + "recall_at_20": 0.83412, + "recall_at_100": 0.93189, + "recall_at_1000": 0.93189, + "accuracy": 0.30044, + "precision_at_1": 0.71552, + "precision_at_3": 0.50575, + "precision_at_5": 0.38621, + "precision_at_10": 0.25948, + "precision_at_20": 0.16078, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.715517, + "mrr_at_3": 0.795977, + "mrr_at_5": 0.802874, + "mrr_at_10": 0.808894, + "mrr_at_20": 0.809708, + "mrr_at_100": 0.810184, + "mrr_at_1000": 0.810184, + "nauc_ndcg_at_1_max": 0.0, + "nauc_ndcg_at_1_std": 0.0, + "nauc_ndcg_at_1_diff1": 0.0, + "nauc_ndcg_at_3_max": 0.0, + "nauc_ndcg_at_3_std": 0.0, + "nauc_ndcg_at_3_diff1": 0.0, + "nauc_ndcg_at_5_max": 0.0, + "nauc_ndcg_at_5_std": 0.0, + "nauc_ndcg_at_5_diff1": 0.0, + "nauc_ndcg_at_10_max": 0.0, + "nauc_ndcg_at_10_std": 0.0, + "nauc_ndcg_at_10_diff1": 0.0, + "nauc_ndcg_at_20_max": 0.0, + "nauc_ndcg_at_20_std": 0.0, + "nauc_ndcg_at_20_diff1": 0.0, + "nauc_ndcg_at_100_max": -0.0, + "nauc_ndcg_at_100_std": -0.0, + "nauc_ndcg_at_100_diff1": -0.0, + "nauc_ndcg_at_1000_max": -0.0, + "nauc_ndcg_at_1000_std": -0.0, + "nauc_ndcg_at_1000_diff1": -0.0, + "nauc_map_at_1_max": -0.0, + "nauc_map_at_1_std": -0.0, + "nauc_map_at_1_diff1": -0.0, + "nauc_map_at_3_max": -0.0, + "nauc_map_at_3_std": -0.0, + "nauc_map_at_3_diff1": -0.0, + "nauc_map_at_5_max": 0.0, + "nauc_map_at_5_std": 0.0, + "nauc_map_at_5_diff1": 0.0, + "nauc_map_at_10_max": 0.0, + "nauc_map_at_10_std": 0.0, + "nauc_map_at_10_diff1": 0.0, + "nauc_map_at_20_max": 0.0, + "nauc_map_at_20_std": 0.0, + "nauc_map_at_20_diff1": 0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": -0.0, + "nauc_recall_at_1_std": -0.0, + "nauc_recall_at_1_diff1": -0.0, + "nauc_recall_at_3_max": 0.0, + "nauc_recall_at_3_std": 0.0, + "nauc_recall_at_3_diff1": 0.0, + "nauc_recall_at_5_max": 0.0, + "nauc_recall_at_5_std": 0.0, + "nauc_recall_at_5_diff1": 0.0, + "nauc_recall_at_10_max": -0.0, + "nauc_recall_at_10_std": -0.0, + "nauc_recall_at_10_diff1": -0.0, + "nauc_recall_at_20_max": -0.0, + "nauc_recall_at_20_std": -0.0, + "nauc_recall_at_20_diff1": -0.0, + "nauc_recall_at_100_max": 0.0, + "nauc_recall_at_100_std": 0.0, + "nauc_recall_at_100_diff1": 0.0, + "nauc_recall_at_1000_max": 0.0, + "nauc_recall_at_1000_std": 0.0, + "nauc_recall_at_1000_diff1": 0.0, + "nauc_precision_at_1_max": -0.0, + "nauc_precision_at_1_std": -0.0, + "nauc_precision_at_1_diff1": -0.0, + "nauc_precision_at_3_max": -0.0, + "nauc_precision_at_3_std": -0.0, + "nauc_precision_at_3_diff1": -0.0, + "nauc_precision_at_5_max": -0.0, + "nauc_precision_at_5_std": -0.0, + "nauc_precision_at_5_diff1": -0.0, + "nauc_precision_at_10_max": 0.0, + "nauc_precision_at_10_std": 0.0, + "nauc_precision_at_10_diff1": 0.0, + "nauc_precision_at_20_max": 0.0, + "nauc_precision_at_20_std": 0.0, + "nauc_precision_at_20_diff1": 0.0, + "nauc_precision_at_100_max": -0.0, + "nauc_precision_at_100_std": -0.0, + "nauc_precision_at_100_diff1": -0.0, + "nauc_precision_at_1000_max": 0.0, + "nauc_precision_at_1000_std": 0.0, + "nauc_precision_at_1000_diff1": 0.0, + "nauc_mrr_at_1_max": -0.0, + "nauc_mrr_at_1_std": -0.0, + "nauc_mrr_at_1_diff1": -0.0, + "nauc_mrr_at_3_max": 0.0, + "nauc_mrr_at_3_std": 0.0, + "nauc_mrr_at_3_diff1": 0.0, + "nauc_mrr_at_5_max": 0.0, + "nauc_mrr_at_5_std": 0.0, + "nauc_mrr_at_5_diff1": 0.0, + "nauc_mrr_at_10_max": 0.0, + "nauc_mrr_at_10_std": 0.0, + "nauc_mrr_at_10_diff1": 0.0, + "nauc_mrr_at_20_max": 0.0, + "nauc_mrr_at_20_std": 0.0, + "nauc_mrr_at_20_diff1": 0.0, + "nauc_mrr_at_100_max": 0.0, + "nauc_mrr_at_100_std": 0.0, + "nauc_mrr_at_100_diff1": 0.0, + "nauc_mrr_at_1000_max": 0.0, + "nauc_mrr_at_1000_std": 0.0, + "nauc_mrr_at_1000_diff1": 0.0, + "hit_rate_at_1": 0.71552, + "hit_rate_at_3": 0.89943, + "hit_rate_at_5": 0.92816, + "hit_rate_at_10": 0.96839, + "hit_rate_at_20": 0.97989, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.69941, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.68822, + "ndcg_at_3": 0.67112, + "ndcg_at_5": 0.67586, + "ndcg_at_10": 0.69722, + "ndcg_at_20": 0.72953, + "ndcg_at_100": 0.76077, + "ndcg_at_1000": 0.76077, + "map_at_1": 0.29248, + "map_at_3": 0.47052, + "map_at_5": 0.52536, + "map_at_10": 0.575, + "map_at_20": 0.60566, + "map_at_100": 0.62189, + "map_at_1000": 0.62189, + "recall_at_1": 0.29248, + "recall_at_3": 0.52771, + "recall_at_5": 0.62462, + "recall_at_10": 0.73692, + "recall_at_20": 0.83649, + "recall_at_100": 0.93007, + "recall_at_1000": 0.93007, + "accuracy": 0.29248, + "precision_at_1": 0.71264, + "precision_at_3": 0.50766, + "precision_at_5": 0.3931, + "precision_at_10": 0.26006, + "precision_at_20": 0.16193, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.712644, + "mrr_at_3": 0.794061, + "mrr_at_5": 0.802969, + "mrr_at_10": 0.808311, + "mrr_at_20": 0.809383, + "mrr_at_100": 0.809704, + "mrr_at_1000": 0.809704, + "nauc_ndcg_at_1_max": 0.0, + "nauc_ndcg_at_1_std": 0.0, + "nauc_ndcg_at_1_diff1": 0.0, + "nauc_ndcg_at_3_max": -0.0, + "nauc_ndcg_at_3_std": -0.0, + "nauc_ndcg_at_3_diff1": -0.0, + "nauc_ndcg_at_5_max": 0.0, + "nauc_ndcg_at_5_std": 0.0, + "nauc_ndcg_at_5_diff1": 0.0, + "nauc_ndcg_at_10_max": -0.0, + "nauc_ndcg_at_10_std": -0.0, + "nauc_ndcg_at_10_diff1": -0.0, + "nauc_ndcg_at_20_max": 0.0, + "nauc_ndcg_at_20_std": 0.0, + "nauc_ndcg_at_20_diff1": 0.0, + "nauc_ndcg_at_100_max": -0.0, + "nauc_ndcg_at_100_std": -0.0, + "nauc_ndcg_at_100_diff1": -0.0, + "nauc_ndcg_at_1000_max": -0.0, + "nauc_ndcg_at_1000_std": -0.0, + "nauc_ndcg_at_1000_diff1": -0.0, + "nauc_map_at_1_max": 0.0, + "nauc_map_at_1_std": 0.0, + "nauc_map_at_1_diff1": 0.0, + "nauc_map_at_3_max": 0.0, + "nauc_map_at_3_std": 0.0, + "nauc_map_at_3_diff1": 0.0, + "nauc_map_at_5_max": 0.0, + "nauc_map_at_5_std": 0.0, + "nauc_map_at_5_diff1": 0.0, + "nauc_map_at_10_max": -0.0, + "nauc_map_at_10_std": -0.0, + "nauc_map_at_10_diff1": -0.0, + "nauc_map_at_20_max": -0.0, + "nauc_map_at_20_std": -0.0, + "nauc_map_at_20_diff1": -0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": 0.0, + "nauc_recall_at_1_std": 0.0, + "nauc_recall_at_1_diff1": 0.0, + "nauc_recall_at_3_max": 0.0, + "nauc_recall_at_3_std": 0.0, + "nauc_recall_at_3_diff1": 0.0, + "nauc_recall_at_5_max": 0.0, + "nauc_recall_at_5_std": 0.0, + "nauc_recall_at_5_diff1": 0.0, + "nauc_recall_at_10_max": -0.0, + "nauc_recall_at_10_std": -0.0, + "nauc_recall_at_10_diff1": -0.0, + "nauc_recall_at_20_max": -0.0, + "nauc_recall_at_20_std": -0.0, + "nauc_recall_at_20_diff1": -0.0, + "nauc_recall_at_100_max": -0.0, + "nauc_recall_at_100_std": -0.0, + "nauc_recall_at_100_diff1": -0.0, + "nauc_recall_at_1000_max": -0.0, + "nauc_recall_at_1000_std": -0.0, + "nauc_recall_at_1000_diff1": -0.0, + "nauc_precision_at_1_max": -0.0, + "nauc_precision_at_1_std": -0.0, + "nauc_precision_at_1_diff1": -0.0, + "nauc_precision_at_3_max": 0.0, + "nauc_precision_at_3_std": 0.0, + "nauc_precision_at_3_diff1": 0.0, + "nauc_precision_at_5_max": 0.0, + "nauc_precision_at_5_std": 0.0, + "nauc_precision_at_5_diff1": 0.0, + "nauc_precision_at_10_max": 0.0, + "nauc_precision_at_10_std": 0.0, + "nauc_precision_at_10_diff1": 0.0, + "nauc_precision_at_20_max": -0.0, + "nauc_precision_at_20_std": -0.0, + "nauc_precision_at_20_diff1": -0.0, + "nauc_precision_at_100_max": -0.0, + "nauc_precision_at_100_std": -0.0, + "nauc_precision_at_100_diff1": -0.0, + "nauc_precision_at_1000_max": 0.0, + "nauc_precision_at_1000_std": 0.0, + "nauc_precision_at_1000_diff1": 0.0, + "nauc_mrr_at_1_max": -0.0, + "nauc_mrr_at_1_std": -0.0, + "nauc_mrr_at_1_diff1": -0.0, + "nauc_mrr_at_3_max": 0.0, + "nauc_mrr_at_3_std": 0.0, + "nauc_mrr_at_3_diff1": 0.0, + "nauc_mrr_at_5_max": -0.0, + "nauc_mrr_at_5_std": -0.0, + "nauc_mrr_at_5_diff1": -0.0, + "nauc_mrr_at_10_max": 0.0, + "nauc_mrr_at_10_std": 0.0, + "nauc_mrr_at_10_diff1": 0.0, + "nauc_mrr_at_20_max": -0.0, + "nauc_mrr_at_20_std": -0.0, + "nauc_mrr_at_20_diff1": -0.0, + "nauc_mrr_at_100_max": 0.0, + "nauc_mrr_at_100_std": 0.0, + "nauc_mrr_at_100_diff1": 0.0, + "nauc_mrr_at_1000_max": 0.0, + "nauc_mrr_at_1000_std": 0.0, + "nauc_mrr_at_1000_diff1": 0.0, + "hit_rate_at_1": 0.71264, + "hit_rate_at_3": 0.89655, + "hit_rate_at_5": 0.93391, + "hit_rate_at_10": 0.97126, + "hit_rate_at_20": 0.98563, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.69722, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.67385, + "ndcg_at_3": 0.66799, + "ndcg_at_5": 0.66829, + "ndcg_at_10": 0.69101, + "ndcg_at_20": 0.72332, + "ndcg_at_100": 0.75697, + "ndcg_at_1000": 0.75697, + "map_at_1": 0.28928, + "map_at_3": 0.46672, + "map_at_5": 0.51938, + "map_at_10": 0.56811, + "map_at_20": 0.59837, + "map_at_100": 0.61607, + "map_at_1000": 0.61607, + "recall_at_1": 0.28928, + "recall_at_3": 0.53297, + "recall_at_5": 0.62214, + "recall_at_10": 0.73613, + "recall_at_20": 0.83163, + "recall_at_100": 0.93268, + "recall_at_1000": 0.93268, + "accuracy": 0.28928, + "precision_at_1": 0.70402, + "precision_at_3": 0.50958, + "precision_at_5": 0.38793, + "precision_at_10": 0.25603, + "precision_at_20": 0.16006, + "precision_at_100": 0.03937, + "precision_at_1000": 0.00394, + "mrr_at_1": 0.704023, + "mrr_at_3": 0.795498, + "mrr_at_5": 0.801533, + "mrr_at_10": 0.805951, + "mrr_at_20": 0.806822, + "mrr_at_100": 0.80728, + "mrr_at_1000": 0.80728, + "nauc_ndcg_at_1_max": -0.0, + "nauc_ndcg_at_1_std": -0.0, + "nauc_ndcg_at_1_diff1": -0.0, + "nauc_ndcg_at_3_max": 0.0, + "nauc_ndcg_at_3_std": 0.0, + "nauc_ndcg_at_3_diff1": 0.0, + "nauc_ndcg_at_5_max": 0.0, + "nauc_ndcg_at_5_std": 0.0, + "nauc_ndcg_at_5_diff1": 0.0, + "nauc_ndcg_at_10_max": 0.0, + "nauc_ndcg_at_10_std": 0.0, + "nauc_ndcg_at_10_diff1": 0.0, + "nauc_ndcg_at_20_max": 0.0, + "nauc_ndcg_at_20_std": 0.0, + "nauc_ndcg_at_20_diff1": 0.0, + "nauc_ndcg_at_100_max": 0.0, + "nauc_ndcg_at_100_std": 0.0, + "nauc_ndcg_at_100_diff1": 0.0, + "nauc_ndcg_at_1000_max": 0.0, + "nauc_ndcg_at_1000_std": 0.0, + "nauc_ndcg_at_1000_diff1": 0.0, + "nauc_map_at_1_max": 0.0, + "nauc_map_at_1_std": 0.0, + "nauc_map_at_1_diff1": 0.0, + "nauc_map_at_3_max": 0.0, + "nauc_map_at_3_std": 0.0, + "nauc_map_at_3_diff1": 0.0, + "nauc_map_at_5_max": 0.0, + "nauc_map_at_5_std": 0.0, + "nauc_map_at_5_diff1": 0.0, + "nauc_map_at_10_max": 0.0, + "nauc_map_at_10_std": 0.0, + "nauc_map_at_10_diff1": 0.0, + "nauc_map_at_20_max": 0.0, + "nauc_map_at_20_std": 0.0, + "nauc_map_at_20_diff1": 0.0, + "nauc_map_at_100_max": 0.0, + "nauc_map_at_100_std": 0.0, + "nauc_map_at_100_diff1": 0.0, + "nauc_map_at_1000_max": 0.0, + "nauc_map_at_1000_std": 0.0, + "nauc_map_at_1000_diff1": 0.0, + "nauc_recall_at_1_max": 0.0, + "nauc_recall_at_1_std": 0.0, + "nauc_recall_at_1_diff1": 0.0, + "nauc_recall_at_3_max": 0.0, + "nauc_recall_at_3_std": 0.0, + "nauc_recall_at_3_diff1": 0.0, + "nauc_recall_at_5_max": -0.0, + "nauc_recall_at_5_std": -0.0, + "nauc_recall_at_5_diff1": -0.0, + "nauc_recall_at_10_max": -0.0, + "nauc_recall_at_10_std": -0.0, + "nauc_recall_at_10_diff1": -0.0, + "nauc_recall_at_20_max": 0.0, + "nauc_recall_at_20_std": 0.0, + "nauc_recall_at_20_diff1": 0.0, + "nauc_recall_at_100_max": -0.0, + "nauc_recall_at_100_std": -0.0, + "nauc_recall_at_100_diff1": -0.0, + "nauc_recall_at_1000_max": -0.0, + "nauc_recall_at_1000_std": -0.0, + "nauc_recall_at_1000_diff1": -0.0, + "nauc_precision_at_1_max": -0.0, + "nauc_precision_at_1_std": -0.0, + "nauc_precision_at_1_diff1": -0.0, + "nauc_precision_at_3_max": 0.0, + "nauc_precision_at_3_std": 0.0, + "nauc_precision_at_3_diff1": 0.0, + "nauc_precision_at_5_max": -0.0, + "nauc_precision_at_5_std": -0.0, + "nauc_precision_at_5_diff1": -0.0, + "nauc_precision_at_10_max": 0.0, + "nauc_precision_at_10_std": 0.0, + "nauc_precision_at_10_diff1": 0.0, + "nauc_precision_at_20_max": 0.0, + "nauc_precision_at_20_std": 0.0, + "nauc_precision_at_20_diff1": 0.0, + "nauc_precision_at_100_max": 0.0, + "nauc_precision_at_100_std": 0.0, + "nauc_precision_at_100_diff1": 0.0, + "nauc_precision_at_1000_max": 0.0, + "nauc_precision_at_1000_std": 0.0, + "nauc_precision_at_1000_diff1": 0.0, + "nauc_mrr_at_1_max": -0.0, + "nauc_mrr_at_1_std": -0.0, + "nauc_mrr_at_1_diff1": -0.0, + "nauc_mrr_at_3_max": -0.0, + "nauc_mrr_at_3_std": -0.0, + "nauc_mrr_at_3_diff1": -0.0, + "nauc_mrr_at_5_max": 0.0, + "nauc_mrr_at_5_std": 0.0, + "nauc_mrr_at_5_diff1": 0.0, + "nauc_mrr_at_10_max": 0.0, + "nauc_mrr_at_10_std": 0.0, + "nauc_mrr_at_10_diff1": 0.0, + "nauc_mrr_at_20_max": 0.0, + "nauc_mrr_at_20_std": 0.0, + "nauc_mrr_at_20_diff1": 0.0, + "nauc_mrr_at_100_max": 0.0, + "nauc_mrr_at_100_std": 0.0, + "nauc_mrr_at_100_diff1": 0.0, + "nauc_mrr_at_1000_max": 0.0, + "nauc_mrr_at_1000_std": 0.0, + "nauc_mrr_at_1000_diff1": 0.0, + "hit_rate_at_1": 0.70402, + "hit_rate_at_3": 0.91379, + "hit_rate_at_5": 0.93966, + "hit_rate_at_10": 0.97126, + "hit_rate_at_20": 0.98276, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.69101, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 44.94659286108799, + "kg_co2_emissions": null, + "date": 1792440718.183964, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 1.097691711038351e-05, + "end": 31.635659248917364, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 31.635750534944236, + "end": 31.635759708937258, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries and documents", + "start": 31.641796426963992, + "end": 33.80661089497153, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 33.80671104497742, + "end": 33.85451251396444, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries and documents", + "start": 33.86884893395472, + "end": 36.00806529994588, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 36.00807875988539, + "end": 36.047437191940844, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries and documents", + "start": 36.05852649593726, + "end": 38.21714125294238, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 38.21719214890618, + "end": 38.25959751789924, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries and documents", + "start": 38.272112984908745, + "end": 40.42021722299978, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 40.420257282909006, + "end": 40.45970962499268, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries and documents", + "start": 40.472832149942406, + "end": 42.60427073389292, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 42.604283322929405, + "end": 42.64336339791771, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries and documents", + "start": 42.65707904996816, + "end": 44.896652502939105, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 44.896689057932235, + "end": 44.93666622496676, + "split": "test", + "subset": "portuguese" + } + ] +} diff --git a/results/mteb__baseline-no-reranking/2/run_settings.jsonl b/results/mteb__baseline-no-reranking/2/run_settings.jsonl index 1e20a62ad3..626d7e5cc7 100644 --- a/results/mteb__baseline-no-reranking/2/run_settings.jsonl +++ b/results/mteb__baseline-no-reranking/2/run_settings.jsonl @@ -38,3 +38,5 @@ {"task": "Vidore3PharmaceuticalsNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.0"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3PhysicsNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.0"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.0"}, "encode_kwargs": {"batch_size": 16}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} diff --git a/results/mteb__baseline-random-cross-encoder/2/Vidore3NuclearNemotron35Embed8BReranking.v2.json b/results/mteb__baseline-random-cross-encoder/2/Vidore3NuclearNemotron35Embed8BReranking.v2.json index f2325fabd7..6fecbac8b4 100644 --- a/results/mteb__baseline-random-cross-encoder/2/Vidore3NuclearNemotron35Embed8BReranking.v2.json +++ b/results/mteb__baseline-random-cross-encoder/2/Vidore3NuclearNemotron35Embed8BReranking.v2.json @@ -471,6 +471,474 @@ "deu-Latn" ], "mteb_version": "2.26.4" + }, + { + "ndcg_at_1": 0.0497, + "ndcg_at_3": 0.06329, + "ndcg_at_5": 0.07339, + "ndcg_at_10": 0.10566, + "ndcg_at_20": 0.15907, + "ndcg_at_100": 0.29818, + "ndcg_at_1000": 0.29818, + "map_at_1": 0.01483, + "map_at_3": 0.02739, + "map_at_5": 0.03511, + "map_at_10": 0.05104, + "map_at_20": 0.06985, + "map_at_100": 0.10782, + "map_at_1000": 0.10782, + "recall_at_1": 0.01483, + "recall_at_3": 0.0411, + "recall_at_5": 0.07016, + "recall_at_10": 0.15906, + "recall_at_20": 0.31785, + "recall_at_100": 0.80792, + "recall_at_1000": 0.80792, + "accuracy": 0.01483, + "precision_at_1": 0.05723, + "precision_at_3": 0.06325, + "precision_at_5": 0.06506, + "precision_at_10": 0.07018, + "precision_at_20": 0.06732, + "precision_at_100": 0.03334, + "precision_at_1000": 0.00333, + "mrr_at_1": 0.057229, + "mrr_at_3": 0.097892, + "mrr_at_5": 0.119578, + "mrr_at_10": 0.146314, + "mrr_at_20": 0.16034, + "mrr_at_100": 0.170027, + "mrr_at_1000": 0.170027, + "nauc_ndcg_at_1_max": 0.006839, + "nauc_ndcg_at_1_std": -0.12643, + "nauc_ndcg_at_1_diff1": 0.041729, + "nauc_ndcg_at_3_max": 0.042461, + "nauc_ndcg_at_3_std": -0.082273, + "nauc_ndcg_at_3_diff1": 0.08263, + "nauc_ndcg_at_5_max": 0.057743, + "nauc_ndcg_at_5_std": -0.058534, + "nauc_ndcg_at_5_diff1": 0.09409, + "nauc_ndcg_at_10_max": 0.043862, + "nauc_ndcg_at_10_std": 0.000413, + "nauc_ndcg_at_10_diff1": 0.121367, + "nauc_ndcg_at_20_max": 0.023872, + "nauc_ndcg_at_20_std": 0.078403, + "nauc_ndcg_at_20_diff1": 0.121937, + "nauc_ndcg_at_100_max": 0.028504, + "nauc_ndcg_at_100_std": -0.008503, + "nauc_ndcg_at_100_diff1": 0.071081, + "nauc_ndcg_at_1000_max": 0.028504, + "nauc_ndcg_at_1000_std": -0.008503, + "nauc_ndcg_at_1000_diff1": 0.071081, + "nauc_map_at_1_max": -0.003847, + "nauc_map_at_1_std": 0.045475, + "nauc_map_at_1_diff1": 0.146773, + "nauc_map_at_3_max": 0.008732, + "nauc_map_at_3_std": 0.002697, + "nauc_map_at_3_diff1": 0.137908, + "nauc_map_at_5_max": 0.028931, + "nauc_map_at_5_std": -0.001859, + "nauc_map_at_5_diff1": 0.142769, + "nauc_map_at_10_max": 0.035385, + "nauc_map_at_10_std": 0.013978, + "nauc_map_at_10_diff1": 0.149307, + "nauc_map_at_20_max": 0.026938, + "nauc_map_at_20_std": 0.041835, + "nauc_map_at_20_diff1": 0.134737, + "nauc_map_at_100_max": 0.026737, + "nauc_map_at_100_std": 0.005077, + "nauc_map_at_100_diff1": 0.098869, + "nauc_map_at_1000_max": 0.026737, + "nauc_map_at_1000_std": 0.005077, + "nauc_map_at_1000_diff1": 0.098869, + "nauc_recall_at_1_max": -0.003847, + "nauc_recall_at_1_std": 0.045475, + "nauc_recall_at_1_diff1": 0.146773, + "nauc_recall_at_3_max": 0.031107, + "nauc_recall_at_3_std": -0.008288, + "nauc_recall_at_3_diff1": 0.158856, + "nauc_recall_at_5_max": 0.079247, + "nauc_recall_at_5_std": 0.015717, + "nauc_recall_at_5_diff1": 0.168443, + "nauc_recall_at_10_max": 0.099627, + "nauc_recall_at_10_std": 0.074637, + "nauc_recall_at_10_diff1": 0.186733, + "nauc_recall_at_20_max": 0.035132, + "nauc_recall_at_20_std": 0.187854, + "nauc_recall_at_20_diff1": 0.157321, + "nauc_recall_at_100_max": 0.068835, + "nauc_recall_at_100_std": 0.034852, + "nauc_recall_at_100_diff1": 0.137865, + "nauc_recall_at_1000_max": 0.068835, + "nauc_recall_at_1000_std": 0.034852, + "nauc_recall_at_1000_diff1": 0.137865, + "nauc_precision_at_1_max": -0.043347, + "nauc_precision_at_1_std": -0.126487, + "nauc_precision_at_1_diff1": 0.057703, + "nauc_precision_at_3_max": 0.044691, + "nauc_precision_at_3_std": -0.107404, + "nauc_precision_at_3_diff1": 0.068008, + "nauc_precision_at_5_max": 0.070583, + "nauc_precision_at_5_std": -0.126931, + "nauc_precision_at_5_diff1": 0.038825, + "nauc_precision_at_10_max": 0.015369, + "nauc_precision_at_10_std": -0.077185, + "nauc_precision_at_10_diff1": 0.003909, + "nauc_precision_at_20_max": 0.011703, + "nauc_precision_at_20_std": 0.010326, + "nauc_precision_at_20_diff1": -0.013674, + "nauc_precision_at_100_max": 0.016475, + "nauc_precision_at_100_std": -0.060142, + "nauc_precision_at_100_diff1": -0.072182, + "nauc_precision_at_1000_max": 0.016475, + "nauc_precision_at_1000_std": -0.060142, + "nauc_precision_at_1000_diff1": -0.072182, + "nauc_mrr_at_1_max": -0.043347, + "nauc_mrr_at_1_std": -0.126487, + "nauc_mrr_at_1_diff1": 0.057703, + "nauc_mrr_at_3_max": -0.013064, + "nauc_mrr_at_3_std": -0.09505, + "nauc_mrr_at_3_diff1": 0.097839, + "nauc_mrr_at_5_max": 0.003417, + "nauc_mrr_at_5_std": -0.094635, + "nauc_mrr_at_5_diff1": 0.086863, + "nauc_mrr_at_10_max": -0.00922, + "nauc_mrr_at_10_std": -0.071526, + "nauc_mrr_at_10_diff1": 0.079111, + "nauc_mrr_at_20_max": -0.006842, + "nauc_mrr_at_20_std": -0.052442, + "nauc_mrr_at_20_diff1": 0.084266, + "nauc_mrr_at_100_max": -0.008499, + "nauc_mrr_at_100_std": -0.063321, + "nauc_mrr_at_100_diff1": 0.085438, + "nauc_mrr_at_1000_max": -0.008499, + "nauc_mrr_at_1000_std": -0.063321, + "nauc_mrr_at_1000_diff1": 0.085438, + "hit_rate_at_1": 0.05723, + "hit_rate_at_3": 0.15663, + "hit_rate_at_5": 0.25301, + "hit_rate_at_10": 0.45181, + "hit_rate_at_20": 0.65663, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.10566, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.06325, + "ndcg_at_3": 0.06613, + "ndcg_at_5": 0.08402, + "ndcg_at_10": 0.10703, + "ndcg_at_20": 0.16424, + "ndcg_at_100": 0.30177, + "ndcg_at_1000": 0.30177, + "map_at_1": 0.0156, + "map_at_3": 0.02942, + "map_at_5": 0.04063, + "map_at_10": 0.05293, + "map_at_20": 0.07206, + "map_at_100": 0.11021, + "map_at_1000": 0.11021, + "recall_at_1": 0.0156, + "recall_at_3": 0.0463, + "recall_at_5": 0.0918, + "recall_at_10": 0.15495, + "recall_at_20": 0.3223, + "recall_at_100": 0.80575, + "recall_at_1000": 0.80575, + "accuracy": 0.0156, + "precision_at_1": 0.07229, + "precision_at_3": 0.06124, + "precision_at_5": 0.06807, + "precision_at_10": 0.06536, + "precision_at_20": 0.06581, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.072289, + "mrr_at_3": 0.110944, + "mrr_at_5": 0.138805, + "mrr_at_10": 0.159656, + "mrr_at_20": 0.174139, + "mrr_at_100": 0.183871, + "mrr_at_1000": 0.183871, + "nauc_ndcg_at_1_max": -0.123201, + "nauc_ndcg_at_1_std": -0.147426, + "nauc_ndcg_at_1_diff1": 0.0411, + "nauc_ndcg_at_3_max": -0.112169, + "nauc_ndcg_at_3_std": -0.043756, + "nauc_ndcg_at_3_diff1": 0.070662, + "nauc_ndcg_at_5_max": -0.012054, + "nauc_ndcg_at_5_std": -0.075984, + "nauc_ndcg_at_5_diff1": 0.128985, + "nauc_ndcg_at_10_max": -0.011344, + "nauc_ndcg_at_10_std": -0.068691, + "nauc_ndcg_at_10_diff1": 0.08616, + "nauc_ndcg_at_20_max": -0.016095, + "nauc_ndcg_at_20_std": -0.044366, + "nauc_ndcg_at_20_diff1": 0.07897, + "nauc_ndcg_at_100_max": -0.02061, + "nauc_ndcg_at_100_std": -0.00978, + "nauc_ndcg_at_100_diff1": 0.057979, + "nauc_ndcg_at_1000_max": -0.02061, + "nauc_ndcg_at_1000_std": -0.00978, + "nauc_ndcg_at_1000_diff1": 0.057979, + "nauc_map_at_1_max": -0.169967, + "nauc_map_at_1_std": -0.192229, + "nauc_map_at_1_diff1": 0.110598, + "nauc_map_at_3_max": -0.113897, + "nauc_map_at_3_std": -0.089439, + "nauc_map_at_3_diff1": 0.119516, + "nauc_map_at_5_max": -0.024411, + "nauc_map_at_5_std": -0.110308, + "nauc_map_at_5_diff1": 0.144174, + "nauc_map_at_10_max": -0.033276, + "nauc_map_at_10_std": -0.089981, + "nauc_map_at_10_diff1": 0.106802, + "nauc_map_at_20_max": -0.041545, + "nauc_map_at_20_std": -0.067349, + "nauc_map_at_20_diff1": 0.086667, + "nauc_map_at_100_max": -0.041973, + "nauc_map_at_100_std": -0.040111, + "nauc_map_at_100_diff1": 0.059621, + "nauc_map_at_1000_max": -0.041973, + "nauc_map_at_1000_std": -0.040111, + "nauc_map_at_1000_diff1": 0.059621, + "nauc_recall_at_1_max": -0.169967, + "nauc_recall_at_1_std": -0.192229, + "nauc_recall_at_1_diff1": 0.110598, + "nauc_recall_at_3_max": -0.101158, + "nauc_recall_at_3_std": 0.000334, + "nauc_recall_at_3_diff1": 0.115065, + "nauc_recall_at_5_max": 0.081941, + "nauc_recall_at_5_std": -0.07689, + "nauc_recall_at_5_diff1": 0.164193, + "nauc_recall_at_10_max": 0.04098, + "nauc_recall_at_10_std": -0.04028, + "nauc_recall_at_10_diff1": 0.07319, + "nauc_recall_at_20_max": 0.01046, + "nauc_recall_at_20_std": 0.006341, + "nauc_recall_at_20_diff1": 0.053018, + "nauc_recall_at_100_max": -0.009768, + "nauc_recall_at_100_std": 0.103832, + "nauc_recall_at_100_diff1": 0.024903, + "nauc_recall_at_1000_max": -0.009768, + "nauc_recall_at_1000_std": 0.103832, + "nauc_recall_at_1000_diff1": 0.024903, + "nauc_precision_at_1_max": -0.110913, + "nauc_precision_at_1_std": -0.122901, + "nauc_precision_at_1_diff1": 0.004122, + "nauc_precision_at_3_max": -0.106341, + "nauc_precision_at_3_std": -0.021561, + "nauc_precision_at_3_diff1": 0.04408, + "nauc_precision_at_5_max": 0.011502, + "nauc_precision_at_5_std": -0.06659, + "nauc_precision_at_5_diff1": 0.087446, + "nauc_precision_at_10_max": -0.009647, + "nauc_precision_at_10_std": -0.052858, + "nauc_precision_at_10_diff1": -0.016351, + "nauc_precision_at_20_max": -0.01061, + "nauc_precision_at_20_std": 0.006557, + "nauc_precision_at_20_diff1": -0.023276, + "nauc_precision_at_100_max": 0.000704, + "nauc_precision_at_100_std": 0.047701, + "nauc_precision_at_100_diff1": -0.056656, + "nauc_precision_at_1000_max": 0.000704, + "nauc_precision_at_1000_std": 0.047701, + "nauc_precision_at_1000_diff1": -0.056656, + "nauc_mrr_at_1_max": -0.110913, + "nauc_mrr_at_1_std": -0.122901, + "nauc_mrr_at_1_diff1": 0.004122, + "nauc_mrr_at_3_max": -0.086172, + "nauc_mrr_at_3_std": -0.064527, + "nauc_mrr_at_3_diff1": 0.027709, + "nauc_mrr_at_5_max": -0.038613, + "nauc_mrr_at_5_std": -0.077913, + "nauc_mrr_at_5_diff1": 0.057569, + "nauc_mrr_at_10_max": -0.034525, + "nauc_mrr_at_10_std": -0.089762, + "nauc_mrr_at_10_diff1": 0.03498, + "nauc_mrr_at_20_max": -0.031388, + "nauc_mrr_at_20_std": -0.089268, + "nauc_mrr_at_20_diff1": 0.037228, + "nauc_mrr_at_100_max": -0.036194, + "nauc_mrr_at_100_std": -0.088477, + "nauc_mrr_at_100_diff1": 0.038422, + "nauc_mrr_at_1000_max": -0.036194, + "nauc_mrr_at_1000_std": -0.088477, + "nauc_mrr_at_1000_diff1": 0.038422, + "hit_rate_at_1": 0.07229, + "hit_rate_at_3": 0.16566, + "hit_rate_at_5": 0.28614, + "hit_rate_at_10": 0.44277, + "hit_rate_at_20": 0.6506, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.10703, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.04217, + "ndcg_at_3": 0.05554, + "ndcg_at_5": 0.06686, + "ndcg_at_10": 0.09242, + "ndcg_at_20": 0.14914, + "ndcg_at_100": 0.29103, + "ndcg_at_1000": 0.29103, + "map_at_1": 0.01112, + "map_at_3": 0.02292, + "map_at_5": 0.03173, + "map_at_10": 0.04388, + "map_at_20": 0.06227, + "map_at_100": 0.10131, + "map_at_1000": 0.10131, + "recall_at_1": 0.01112, + "recall_at_3": 0.03838, + "recall_at_5": 0.0685, + "recall_at_10": 0.1379, + "recall_at_20": 0.30705, + "recall_at_100": 0.80599, + "recall_at_1000": 0.80599, + "accuracy": 0.01112, + "precision_at_1": 0.0512, + "precision_at_3": 0.05924, + "precision_at_5": 0.06145, + "precision_at_10": 0.06114, + "precision_at_20": 0.06355, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.051205, + "mrr_at_3": 0.102912, + "mrr_at_5": 0.119779, + "mrr_at_10": 0.14033, + "mrr_at_20": 0.15678, + "mrr_at_100": 0.166628, + "mrr_at_1000": 0.166628, + "nauc_ndcg_at_1_max": -0.001048, + "nauc_ndcg_at_1_std": 0.051418, + "nauc_ndcg_at_1_diff1": -0.150655, + "nauc_ndcg_at_3_max": -0.03317, + "nauc_ndcg_at_3_std": 0.009584, + "nauc_ndcg_at_3_diff1": -0.114731, + "nauc_ndcg_at_5_max": 0.000828, + "nauc_ndcg_at_5_std": -0.035495, + "nauc_ndcg_at_5_diff1": -0.040429, + "nauc_ndcg_at_10_max": 0.024901, + "nauc_ndcg_at_10_std": -0.005065, + "nauc_ndcg_at_10_diff1": -0.060113, + "nauc_ndcg_at_20_max": -0.040959, + "nauc_ndcg_at_20_std": -0.00225, + "nauc_ndcg_at_20_diff1": -0.116071, + "nauc_ndcg_at_100_max": -0.012933, + "nauc_ndcg_at_100_std": -0.028532, + "nauc_ndcg_at_100_diff1": -0.089718, + "nauc_ndcg_at_1000_max": -0.012933, + "nauc_ndcg_at_1000_std": -0.028532, + "nauc_ndcg_at_1000_diff1": -0.089718, + "nauc_map_at_1_max": -0.010671, + "nauc_map_at_1_std": 0.062879, + "nauc_map_at_1_diff1": -0.241489, + "nauc_map_at_3_max": -0.057534, + "nauc_map_at_3_std": 0.025973, + "nauc_map_at_3_diff1": -0.170133, + "nauc_map_at_5_max": -0.026574, + "nauc_map_at_5_std": -0.016403, + "nauc_map_at_5_diff1": -0.063084, + "nauc_map_at_10_max": 0.009292, + "nauc_map_at_10_std": -0.001309, + "nauc_map_at_10_diff1": -0.056251, + "nauc_map_at_20_max": -0.032338, + "nauc_map_at_20_std": 0.012015, + "nauc_map_at_20_diff1": -0.094024, + "nauc_map_at_100_max": -0.031339, + "nauc_map_at_100_std": -0.005377, + "nauc_map_at_100_diff1": -0.082445, + "nauc_map_at_1000_max": -0.031339, + "nauc_map_at_1000_std": -0.005377, + "nauc_map_at_1000_diff1": -0.082445, + "nauc_recall_at_1_max": -0.010671, + "nauc_recall_at_1_std": 0.062879, + "nauc_recall_at_1_diff1": -0.241489, + "nauc_recall_at_3_max": -0.088719, + "nauc_recall_at_3_std": 0.026846, + "nauc_recall_at_3_diff1": -0.160924, + "nauc_recall_at_5_max": -0.016959, + "nauc_recall_at_5_std": -0.050896, + "nauc_recall_at_5_diff1": 0.054519, + "nauc_recall_at_10_max": 0.04906, + "nauc_recall_at_10_std": -0.02076, + "nauc_recall_at_10_diff1": 0.006332, + "nauc_recall_at_20_max": -0.072836, + "nauc_recall_at_20_std": -0.009786, + "nauc_recall_at_20_diff1": -0.105987, + "nauc_recall_at_100_max": -0.046152, + "nauc_recall_at_100_std": -0.029171, + "nauc_recall_at_100_diff1": -0.134917, + "nauc_recall_at_1000_max": -0.046152, + "nauc_recall_at_1000_std": -0.029171, + "nauc_recall_at_1000_diff1": -0.134917, + "nauc_precision_at_1_max": -0.022804, + "nauc_precision_at_1_std": 0.069942, + "nauc_precision_at_1_diff1": -0.144876, + "nauc_precision_at_3_max": -0.021455, + "nauc_precision_at_3_std": -0.013316, + "nauc_precision_at_3_diff1": -0.061411, + "nauc_precision_at_5_max": 0.035675, + "nauc_precision_at_5_std": -0.05171, + "nauc_precision_at_5_diff1": -0.016684, + "nauc_precision_at_10_max": 0.008018, + "nauc_precision_at_10_std": 0.003392, + "nauc_precision_at_10_diff1": -0.076311, + "nauc_precision_at_20_max": -0.059729, + "nauc_precision_at_20_std": -0.014944, + "nauc_precision_at_20_diff1": -0.122872, + "nauc_precision_at_100_max": 0.000151, + "nauc_precision_at_100_std": -0.03718, + "nauc_precision_at_100_diff1": -0.051031, + "nauc_precision_at_1000_max": 0.000151, + "nauc_precision_at_1000_std": -0.03718, + "nauc_precision_at_1000_diff1": -0.051031, + "nauc_mrr_at_1_max": -0.022804, + "nauc_mrr_at_1_std": 0.069942, + "nauc_mrr_at_1_diff1": -0.144876, + "nauc_mrr_at_3_max": -0.017043, + "nauc_mrr_at_3_std": 0.003987, + "nauc_mrr_at_3_diff1": -0.072149, + "nauc_mrr_at_5_max": 0.002368, + "nauc_mrr_at_5_std": -0.013386, + "nauc_mrr_at_5_diff1": -0.038757, + "nauc_mrr_at_10_max": 0.008211, + "nauc_mrr_at_10_std": -0.004003, + "nauc_mrr_at_10_diff1": -0.054179, + "nauc_mrr_at_20_max": 0.004343, + "nauc_mrr_at_20_std": -0.012137, + "nauc_mrr_at_20_diff1": -0.060216, + "nauc_mrr_at_100_max": 0.007383, + "nauc_mrr_at_100_std": -0.010708, + "nauc_mrr_at_100_diff1": -0.054433, + "nauc_mrr_at_1000_max": 0.007383, + "nauc_mrr_at_1000_std": -0.010708, + "nauc_mrr_at_1000_diff1": -0.054433, + "hit_rate_at_1": 0.0512, + "hit_rate_at_3": 0.17169, + "hit_rate_at_5": 0.24699, + "hit_rate_at_10": 0.40361, + "hit_rate_at_20": 0.64458, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.09242, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" } ] }, diff --git a/results/mteb__baseline-random-cross-encoder/2/Vidore3TelecomNemotron35Embed8BReranking.v2.json b/results/mteb__baseline-random-cross-encoder/2/Vidore3TelecomNemotron35Embed8BReranking.v2.json index 7f78eb8fd8..0f50702bea 100644 --- a/results/mteb__baseline-random-cross-encoder/2/Vidore3TelecomNemotron35Embed8BReranking.v2.json +++ b/results/mteb__baseline-random-cross-encoder/2/Vidore3TelecomNemotron35Embed8BReranking.v2.json @@ -471,6 +471,474 @@ "deu-Latn" ], "mteb_version": "2.26.4" + }, + { + "ndcg_at_1": 0.09339, + "ndcg_at_3": 0.10305, + "ndcg_at_5": 0.11167, + "ndcg_at_10": 0.14081, + "ndcg_at_20": 0.20552, + "ndcg_at_100": 0.35971, + "ndcg_at_1000": 0.35971, + "map_at_1": 0.02542, + "map_at_3": 0.04674, + "map_at_5": 0.05735, + "map_at_10": 0.07396, + "map_at_20": 0.09749, + "map_at_100": 0.14504, + "map_at_1000": 0.14504, + "recall_at_1": 0.02542, + "recall_at_3": 0.07129, + "recall_at_5": 0.10857, + "recall_at_10": 0.20129, + "recall_at_20": 0.38007, + "recall_at_100": 0.93189, + "recall_at_1000": 0.93189, + "accuracy": 0.02542, + "precision_at_1": 0.10345, + "precision_at_3": 0.09866, + "precision_at_5": 0.09253, + "precision_at_10": 0.0819, + "precision_at_20": 0.08305, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.103448, + "mrr_at_3": 0.163793, + "mrr_at_5": 0.183908, + "mrr_at_10": 0.206267, + "mrr_at_20": 0.222717, + "mrr_at_100": 0.231208, + "mrr_at_1000": 0.231208, + "nauc_ndcg_at_1_max": 0.049201, + "nauc_ndcg_at_1_std": -0.046224, + "nauc_ndcg_at_1_diff1": 0.045132, + "nauc_ndcg_at_3_max": 0.008242, + "nauc_ndcg_at_3_std": -0.004322, + "nauc_ndcg_at_3_diff1": -0.010802, + "nauc_ndcg_at_5_max": -0.007488, + "nauc_ndcg_at_5_std": 0.006911, + "nauc_ndcg_at_5_diff1": -0.039868, + "nauc_ndcg_at_10_max": -0.01454, + "nauc_ndcg_at_10_std": -0.033503, + "nauc_ndcg_at_10_diff1": -0.069603, + "nauc_ndcg_at_20_max": 0.01617, + "nauc_ndcg_at_20_std": -0.013424, + "nauc_ndcg_at_20_diff1": -0.019841, + "nauc_ndcg_at_100_max": 0.007295, + "nauc_ndcg_at_100_std": -0.011549, + "nauc_ndcg_at_100_diff1": -0.007702, + "nauc_ndcg_at_1000_max": 0.007295, + "nauc_ndcg_at_1000_std": -0.011549, + "nauc_ndcg_at_1000_diff1": -0.007702, + "nauc_map_at_1_max": -0.043972, + "nauc_map_at_1_std": 0.004125, + "nauc_map_at_1_diff1": -0.068468, + "nauc_map_at_3_max": -0.026781, + "nauc_map_at_3_std": 0.037044, + "nauc_map_at_3_diff1": -0.050659, + "nauc_map_at_5_max": -0.033441, + "nauc_map_at_5_std": 0.038847, + "nauc_map_at_5_diff1": -0.063329, + "nauc_map_at_10_max": -0.031542, + "nauc_map_at_10_std": 0.016592, + "nauc_map_at_10_diff1": -0.065178, + "nauc_map_at_20_max": -0.009178, + "nauc_map_at_20_std": 0.026789, + "nauc_map_at_20_diff1": -0.031314, + "nauc_map_at_100_max": -0.003977, + "nauc_map_at_100_std": 0.011779, + "nauc_map_at_100_diff1": -0.022873, + "nauc_map_at_1000_max": -0.003977, + "nauc_map_at_1000_std": 0.011779, + "nauc_map_at_1000_diff1": -0.022873, + "nauc_recall_at_1_max": -0.043972, + "nauc_recall_at_1_std": 0.004125, + "nauc_recall_at_1_diff1": -0.068468, + "nauc_recall_at_3_max": -0.031427, + "nauc_recall_at_3_std": 0.021302, + "nauc_recall_at_3_diff1": -0.063852, + "nauc_recall_at_5_max": -0.052803, + "nauc_recall_at_5_std": 0.037483, + "nauc_recall_at_5_diff1": -0.087946, + "nauc_recall_at_10_max": -0.04202, + "nauc_recall_at_10_std": -0.05218, + "nauc_recall_at_10_diff1": -0.111873, + "nauc_recall_at_20_max": 0.009101, + "nauc_recall_at_20_std": -0.003155, + "nauc_recall_at_20_diff1": -0.030616, + "nauc_recall_at_100_max": -0.008233, + "nauc_recall_at_100_std": 0.113123, + "nauc_recall_at_100_diff1": 0.034937, + "nauc_recall_at_1000_max": -0.008233, + "nauc_recall_at_1000_std": 0.113123, + "nauc_recall_at_1000_diff1": 0.034937, + "nauc_precision_at_1_max": 0.033909, + "nauc_precision_at_1_std": -0.031215, + "nauc_precision_at_1_diff1": 0.041258, + "nauc_precision_at_3_max": 0.013634, + "nauc_precision_at_3_std": 0.01599, + "nauc_precision_at_3_diff1": 0.016488, + "nauc_precision_at_5_max": 0.031399, + "nauc_precision_at_5_std": -0.00162, + "nauc_precision_at_5_diff1": -2.3e-05, + "nauc_precision_at_10_max": 0.043553, + "nauc_precision_at_10_std": -0.053855, + "nauc_precision_at_10_diff1": 0.002595, + "nauc_precision_at_20_max": 0.081173, + "nauc_precision_at_20_std": -0.024931, + "nauc_precision_at_20_diff1": 0.070743, + "nauc_precision_at_100_max": 0.038995, + "nauc_precision_at_100_std": -0.048209, + "nauc_precision_at_100_diff1": 0.040683, + "nauc_precision_at_1000_max": 0.038995, + "nauc_precision_at_1000_std": -0.048209, + "nauc_precision_at_1000_diff1": 0.040683, + "nauc_mrr_at_1_max": 0.033909, + "nauc_mrr_at_1_std": -0.031215, + "nauc_mrr_at_1_diff1": 0.041258, + "nauc_mrr_at_3_max": 0.008857, + "nauc_mrr_at_3_std": -0.006629, + "nauc_mrr_at_3_diff1": 0.008222, + "nauc_mrr_at_5_max": 0.012078, + "nauc_mrr_at_5_std": 0.003287, + "nauc_mrr_at_5_diff1": 0.012119, + "nauc_mrr_at_10_max": 0.013499, + "nauc_mrr_at_10_std": -0.020815, + "nauc_mrr_at_10_diff1": 0.004598, + "nauc_mrr_at_20_max": 0.011928, + "nauc_mrr_at_20_std": -0.012878, + "nauc_mrr_at_20_diff1": 0.006046, + "nauc_mrr_at_100_max": 0.009968, + "nauc_mrr_at_100_std": -0.011018, + "nauc_mrr_at_100_diff1": 0.006026, + "nauc_mrr_at_1000_max": 0.009968, + "nauc_mrr_at_1000_std": -0.011018, + "nauc_mrr_at_1000_diff1": 0.006026, + "hit_rate_at_1": 0.10345, + "hit_rate_at_3": 0.25, + "hit_rate_at_5": 0.34195, + "hit_rate_at_10": 0.51437, + "hit_rate_at_20": 0.74138, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.14081, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.10489, + "ndcg_at_3": 0.10593, + "ndcg_at_5": 0.11503, + "ndcg_at_10": 0.14609, + "ndcg_at_20": 0.21215, + "ndcg_at_100": 0.36437, + "ndcg_at_1000": 0.36437, + "map_at_1": 0.03067, + "map_at_3": 0.0497, + "map_at_5": 0.05941, + "map_at_10": 0.07579, + "map_at_20": 0.09993, + "map_at_100": 0.14701, + "map_at_1000": 0.14701, + "recall_at_1": 0.03067, + "recall_at_3": 0.07414, + "recall_at_5": 0.10741, + "recall_at_10": 0.19964, + "recall_at_20": 0.39559, + "recall_at_100": 0.93007, + "recall_at_1000": 0.93007, + "accuracy": 0.03067, + "precision_at_1": 0.1092, + "precision_at_3": 0.0977, + "precision_at_5": 0.09138, + "precision_at_10": 0.08132, + "precision_at_20": 0.08204, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.109195, + "mrr_at_3": 0.176245, + "mrr_at_5": 0.197941, + "mrr_at_10": 0.220357, + "mrr_at_20": 0.235112, + "mrr_at_100": 0.243051, + "mrr_at_1000": 0.243051, + "nauc_ndcg_at_1_max": -0.073108, + "nauc_ndcg_at_1_std": 0.0574, + "nauc_ndcg_at_1_diff1": 0.018958, + "nauc_ndcg_at_3_max": -0.054626, + "nauc_ndcg_at_3_std": 0.017989, + "nauc_ndcg_at_3_diff1": -0.009901, + "nauc_ndcg_at_5_max": -0.013724, + "nauc_ndcg_at_5_std": -0.006482, + "nauc_ndcg_at_5_diff1": 0.003262, + "nauc_ndcg_at_10_max": 0.049123, + "nauc_ndcg_at_10_std": -0.018396, + "nauc_ndcg_at_10_diff1": -0.032834, + "nauc_ndcg_at_20_max": 0.053777, + "nauc_ndcg_at_20_std": 0.021913, + "nauc_ndcg_at_20_diff1": 0.040648, + "nauc_ndcg_at_100_max": 0.021358, + "nauc_ndcg_at_100_std": 0.024403, + "nauc_ndcg_at_100_diff1": 0.038353, + "nauc_ndcg_at_1000_max": 0.021358, + "nauc_ndcg_at_1000_std": 0.024403, + "nauc_ndcg_at_1000_diff1": 0.038353, + "nauc_map_at_1_max": 0.026965, + "nauc_map_at_1_std": 0.01938, + "nauc_map_at_1_diff1": -0.027989, + "nauc_map_at_3_max": 0.010477, + "nauc_map_at_3_std": 0.008251, + "nauc_map_at_3_diff1": -0.072752, + "nauc_map_at_5_max": 0.022375, + "nauc_map_at_5_std": -0.009359, + "nauc_map_at_5_diff1": -0.042116, + "nauc_map_at_10_max": 0.044964, + "nauc_map_at_10_std": -0.013054, + "nauc_map_at_10_diff1": -0.057713, + "nauc_map_at_20_max": 0.041135, + "nauc_map_at_20_std": 0.001841, + "nauc_map_at_20_diff1": -0.018417, + "nauc_map_at_100_max": 0.034564, + "nauc_map_at_100_std": 0.008924, + "nauc_map_at_100_diff1": 0.003041, + "nauc_map_at_1000_max": 0.034564, + "nauc_map_at_1000_std": 0.008924, + "nauc_map_at_1000_diff1": 0.003041, + "nauc_recall_at_1_max": 0.026965, + "nauc_recall_at_1_std": 0.01938, + "nauc_recall_at_1_diff1": -0.027989, + "nauc_recall_at_3_max": 0.006439, + "nauc_recall_at_3_std": 0.012334, + "nauc_recall_at_3_diff1": -0.101912, + "nauc_recall_at_5_max": 0.049312, + "nauc_recall_at_5_std": -0.025029, + "nauc_recall_at_5_diff1": -0.043866, + "nauc_recall_at_10_max": 0.111416, + "nauc_recall_at_10_std": -0.016856, + "nauc_recall_at_10_diff1": -0.077095, + "nauc_recall_at_20_max": 0.107213, + "nauc_recall_at_20_std": 0.052974, + "nauc_recall_at_20_diff1": 0.075901, + "nauc_recall_at_100_max": 0.022094, + "nauc_recall_at_100_std": 0.089267, + "nauc_recall_at_100_diff1": -0.131139, + "nauc_recall_at_1000_max": 0.022094, + "nauc_recall_at_1000_std": 0.089267, + "nauc_recall_at_1000_diff1": -0.131139, + "nauc_precision_at_1_max": -0.065931, + "nauc_precision_at_1_std": 0.042791, + "nauc_precision_at_1_diff1": 0.01632, + "nauc_precision_at_3_max": -0.068641, + "nauc_precision_at_3_std": 0.004763, + "nauc_precision_at_3_diff1": 0.036232, + "nauc_precision_at_5_max": -0.019465, + "nauc_precision_at_5_std": -0.00826, + "nauc_precision_at_5_diff1": 0.100194, + "nauc_precision_at_10_max": 0.064501, + "nauc_precision_at_10_std": -0.033742, + "nauc_precision_at_10_diff1": 0.044812, + "nauc_precision_at_20_max": 0.039779, + "nauc_precision_at_20_std": 0.01765, + "nauc_precision_at_20_diff1": 0.114192, + "nauc_precision_at_100_max": 0.018167, + "nauc_precision_at_100_std": 0.017602, + "nauc_precision_at_100_diff1": 0.099336, + "nauc_precision_at_1000_max": 0.018167, + "nauc_precision_at_1000_std": 0.017602, + "nauc_precision_at_1000_diff1": 0.099336, + "nauc_mrr_at_1_max": -0.065931, + "nauc_mrr_at_1_std": 0.042791, + "nauc_mrr_at_1_diff1": 0.01632, + "nauc_mrr_at_3_max": -0.083009, + "nauc_mrr_at_3_std": 0.039394, + "nauc_mrr_at_3_diff1": 0.027832, + "nauc_mrr_at_5_max": -0.052179, + "nauc_mrr_at_5_std": 0.026796, + "nauc_mrr_at_5_diff1": 0.033483, + "nauc_mrr_at_10_max": -0.030728, + "nauc_mrr_at_10_std": 0.0222, + "nauc_mrr_at_10_diff1": 0.029249, + "nauc_mrr_at_20_max": -0.034738, + "nauc_mrr_at_20_std": 0.033497, + "nauc_mrr_at_20_diff1": 0.043033, + "nauc_mrr_at_100_max": -0.040099, + "nauc_mrr_at_100_std": 0.03037, + "nauc_mrr_at_100_diff1": 0.035754, + "nauc_mrr_at_1000_max": -0.040099, + "nauc_mrr_at_1000_std": 0.03037, + "nauc_mrr_at_1000_diff1": 0.035754, + "hit_rate_at_1": 0.1092, + "hit_rate_at_3": 0.27011, + "hit_rate_at_5": 0.36782, + "hit_rate_at_10": 0.53448, + "hit_rate_at_20": 0.75, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.14609, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.06322, + "ndcg_at_3": 0.07649, + "ndcg_at_5": 0.09057, + "ndcg_at_10": 0.12315, + "ndcg_at_20": 0.19342, + "ndcg_at_100": 0.34995, + "ndcg_at_1000": 0.34995, + "map_at_1": 0.02146, + "map_at_3": 0.035, + "map_at_5": 0.04584, + "map_at_10": 0.06176, + "map_at_20": 0.0864, + "map_at_100": 0.13504, + "map_at_1000": 0.13504, + "recall_at_1": 0.02146, + "recall_at_3": 0.05275, + "recall_at_5": 0.08975, + "recall_at_10": 0.17953, + "recall_at_20": 0.38028, + "recall_at_100": 0.93268, + "recall_at_1000": 0.93268, + "accuracy": 0.02146, + "precision_at_1": 0.07184, + "precision_at_3": 0.07759, + "precision_at_5": 0.07701, + "precision_at_10": 0.07471, + "precision_at_20": 0.07989, + "precision_at_100": 0.03937, + "precision_at_1000": 0.00394, + "mrr_at_1": 0.071839, + "mrr_at_3": 0.126916, + "mrr_at_5": 0.148611, + "mrr_at_10": 0.172419, + "mrr_at_20": 0.190068, + "mrr_at_100": 0.199041, + "mrr_at_1000": 0.199041, + "nauc_ndcg_at_1_max": 0.214882, + "nauc_ndcg_at_1_std": -0.075522, + "nauc_ndcg_at_1_diff1": 0.120865, + "nauc_ndcg_at_3_max": 0.12944, + "nauc_ndcg_at_3_std": -0.055501, + "nauc_ndcg_at_3_diff1": 0.119782, + "nauc_ndcg_at_5_max": 0.103742, + "nauc_ndcg_at_5_std": -0.046564, + "nauc_ndcg_at_5_diff1": 0.112061, + "nauc_ndcg_at_10_max": 0.08666, + "nauc_ndcg_at_10_std": -0.035423, + "nauc_ndcg_at_10_diff1": 0.104504, + "nauc_ndcg_at_20_max": 0.031001, + "nauc_ndcg_at_20_std": -0.038736, + "nauc_ndcg_at_20_diff1": 0.065405, + "nauc_ndcg_at_100_max": 0.0635, + "nauc_ndcg_at_100_std": -0.04453, + "nauc_ndcg_at_100_diff1": 0.072604, + "nauc_ndcg_at_1000_max": 0.0635, + "nauc_ndcg_at_1000_std": -0.04453, + "nauc_ndcg_at_1000_diff1": 0.072604, + "nauc_map_at_1_max": 0.219444, + "nauc_map_at_1_std": -0.171337, + "nauc_map_at_1_diff1": 0.153229, + "nauc_map_at_3_max": 0.140469, + "nauc_map_at_3_std": -0.106079, + "nauc_map_at_3_diff1": 0.141172, + "nauc_map_at_5_max": 0.134994, + "nauc_map_at_5_std": -0.089694, + "nauc_map_at_5_diff1": 0.131208, + "nauc_map_at_10_max": 0.134887, + "nauc_map_at_10_std": -0.085929, + "nauc_map_at_10_diff1": 0.12653, + "nauc_map_at_20_max": 0.100729, + "nauc_map_at_20_std": -0.074448, + "nauc_map_at_20_diff1": 0.102733, + "nauc_map_at_100_max": 0.101322, + "nauc_map_at_100_std": -0.081173, + "nauc_map_at_100_diff1": 0.086973, + "nauc_map_at_1000_max": 0.101322, + "nauc_map_at_1000_std": -0.081173, + "nauc_map_at_1000_diff1": 0.086973, + "nauc_recall_at_1_max": 0.219444, + "nauc_recall_at_1_std": -0.171337, + "nauc_recall_at_1_diff1": 0.153229, + "nauc_recall_at_3_max": 0.059264, + "nauc_recall_at_3_std": -0.057826, + "nauc_recall_at_3_diff1": 0.125434, + "nauc_recall_at_5_max": 0.038586, + "nauc_recall_at_5_std": -0.05602, + "nauc_recall_at_5_diff1": 0.076223, + "nauc_recall_at_10_max": 0.03916, + "nauc_recall_at_10_std": -0.038009, + "nauc_recall_at_10_diff1": 0.062488, + "nauc_recall_at_20_max": -0.056109, + "nauc_recall_at_20_std": -0.02644, + "nauc_recall_at_20_diff1": -0.01183, + "nauc_recall_at_100_max": -0.061487, + "nauc_recall_at_100_std": -0.138607, + "nauc_recall_at_100_diff1": -0.008495, + "nauc_recall_at_1000_max": -0.061487, + "nauc_recall_at_1000_std": -0.138607, + "nauc_recall_at_1000_diff1": -0.008495, + "nauc_precision_at_1_max": 0.209936, + "nauc_precision_at_1_std": -0.06001, + "nauc_precision_at_1_diff1": 0.116157, + "nauc_precision_at_3_max": 0.129281, + "nauc_precision_at_3_std": -0.016174, + "nauc_precision_at_3_diff1": 0.10208, + "nauc_precision_at_5_max": 0.107619, + "nauc_precision_at_5_std": 0.003765, + "nauc_precision_at_5_diff1": 0.107516, + "nauc_precision_at_10_max": 0.084772, + "nauc_precision_at_10_std": -0.00553, + "nauc_precision_at_10_diff1": 0.086508, + "nauc_precision_at_20_max": -0.002677, + "nauc_precision_at_20_std": 0.009312, + "nauc_precision_at_20_diff1": 0.000282, + "nauc_precision_at_100_max": -0.002182, + "nauc_precision_at_100_std": 0.006732, + "nauc_precision_at_100_diff1": -0.035262, + "nauc_precision_at_1000_max": -0.002182, + "nauc_precision_at_1000_std": 0.006732, + "nauc_precision_at_1000_diff1": -0.035262, + "nauc_mrr_at_1_max": 0.209936, + "nauc_mrr_at_1_std": -0.06001, + "nauc_mrr_at_1_diff1": 0.116157, + "nauc_mrr_at_3_max": 0.17231, + "nauc_mrr_at_3_std": -0.022774, + "nauc_mrr_at_3_diff1": 0.125968, + "nauc_mrr_at_5_max": 0.150803, + "nauc_mrr_at_5_std": -0.025119, + "nauc_mrr_at_5_diff1": 0.11284, + "nauc_mrr_at_10_max": 0.132395, + "nauc_mrr_at_10_std": -0.018236, + "nauc_mrr_at_10_diff1": 0.10969, + "nauc_mrr_at_20_max": 0.130057, + "nauc_mrr_at_20_std": -0.015656, + "nauc_mrr_at_20_diff1": 0.104828, + "nauc_mrr_at_100_max": 0.138116, + "nauc_mrr_at_100_std": -0.018994, + "nauc_mrr_at_100_diff1": 0.109095, + "nauc_mrr_at_1000_max": 0.138116, + "nauc_mrr_at_1000_std": -0.018994, + "nauc_mrr_at_1000_diff1": 0.109095, + "hit_rate_at_1": 0.07184, + "hit_rate_at_3": 0.2069, + "hit_rate_at_5": 0.30172, + "hit_rate_at_10": 0.47989, + "hit_rate_at_20": 0.72989, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.12315, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" } ] }, diff --git a/results/mteb__baseline-random-cross-encoder/2/run_settings.jsonl b/results/mteb__baseline-random-cross-encoder/2/run_settings.jsonl index 9f4a9b31fc..abcc0a20f8 100644 --- a/results/mteb__baseline-random-cross-encoder/2/run_settings.jsonl +++ b/results/mteb__baseline-random-cross-encoder/2/run_settings.jsonl @@ -38,3 +38,7 @@ {"task": "Vidore3PhysicsQwen3VLEmbedding2BReranking.v2", "splits": ["test"], "version": {"mteb": "2.24.9", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.19.0"}, "encode_kwargs": {"batch_size": 64}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3TelecomQwen3VLEmbedding2BReranking.v2", "splits": ["test"], "version": {"mteb": "2.24.9", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.19.0"}, "encode_kwargs": {"batch_size": 64}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3NuclearQwen3VLEmbedding2BReranking.v2", "splits": ["test"], "version": {"mteb": "2.24.9", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.19.0"}, "encode_kwargs": {"batch_size": 64}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.4", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.19.0"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "german"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.4", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.19.0"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 64}, "subsets": ["italian", "portuguese", "spanish"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.2"}, "encode_kwargs": {"batch_size": 64}, "subsets": ["italian", "portuguese", "spanish"]} diff --git a/results/nvidia__Nemotron-3.5-Rerank-8B-BF16/708e844c3da0bbaa923e9dc310b07da4ffb6a6b5/Vidore3NuclearNemotron35Embed8BReranking.v2.json b/results/nvidia__Nemotron-3.5-Rerank-8B-BF16/708e844c3da0bbaa923e9dc310b07da4ffb6a6b5/Vidore3NuclearNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..b66d05d61c --- /dev/null +++ b/results/nvidia__Nemotron-3.5-Rerank-8B-BF16/708e844c3da0bbaa923e9dc310b07da4ffb6a6b5/Vidore3NuclearNemotron35Embed8BReranking.v2.json @@ -0,0 +1,978 @@ +{ + "dataset_revision": "97ff9e10e265e64abece589ff40c552215754f26", + "task_name": "Vidore3NuclearNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.52861, + "ndcg_at_3": 0.55245, + "ndcg_at_5": 0.56162, + "ndcg_at_10": 0.58254, + "ndcg_at_20": 0.62061, + "ndcg_at_100": 0.64761, + "ndcg_at_1000": 0.64761, + "map_at_1": 0.25443, + "map_at_3": 0.37871, + "map_at_5": 0.42413, + "map_at_10": 0.46513, + "map_at_20": 0.49397, + "map_at_100": 0.50678, + "map_at_1000": 0.50678, + "recall_at_1": 0.25443, + "recall_at_3": 0.44374, + "recall_at_5": 0.52945, + "recall_at_10": 0.63222, + "recall_at_20": 0.7369, + "recall_at_100": 0.81132, + "recall_at_1000": 0.81132, + "accuracy": 0.25443, + "precision_at_1": 0.54518, + "precision_at_3": 0.40462, + "precision_at_5": 0.32289, + "precision_at_10": 0.21988, + "precision_at_20": 0.14202, + "precision_at_100": 0.0338, + "precision_at_1000": 0.00338, + "mrr_at_1": 0.545181, + "mrr_at_3": 0.667169, + "mrr_at_5": 0.679669, + "mrr_at_10": 0.685797, + "mrr_at_20": 0.687515, + "mrr_at_100": 0.687787, + "mrr_at_1000": 0.687787, + "nauc_ndcg_at_1_max": 0.119942, + "nauc_ndcg_at_1_std": 0.154041, + "nauc_ndcg_at_1_diff1": 0.440248, + "nauc_ndcg_at_3_max": 0.17287, + "nauc_ndcg_at_3_std": 0.242926, + "nauc_ndcg_at_3_diff1": 0.399912, + "nauc_ndcg_at_5_max": 0.184966, + "nauc_ndcg_at_5_std": 0.253283, + "nauc_ndcg_at_5_diff1": 0.421792, + "nauc_ndcg_at_10_max": 0.196103, + "nauc_ndcg_at_10_std": 0.301957, + "nauc_ndcg_at_10_diff1": 0.441953, + "nauc_ndcg_at_20_max": 0.18993, + "nauc_ndcg_at_20_std": 0.331083, + "nauc_ndcg_at_20_diff1": 0.438399, + "nauc_ndcg_at_100_max": 0.191299, + "nauc_ndcg_at_100_std": 0.340035, + "nauc_ndcg_at_100_diff1": 0.428961, + "nauc_ndcg_at_1000_max": 0.191299, + "nauc_ndcg_at_1000_std": 0.340035, + "nauc_ndcg_at_1000_diff1": 0.428961, + "nauc_map_at_1_max": 0.172256, + "nauc_map_at_1_std": 0.065344, + "nauc_map_at_1_diff1": 0.499517, + "nauc_map_at_3_max": 0.194486, + "nauc_map_at_3_std": 0.157841, + "nauc_map_at_3_diff1": 0.461791, + "nauc_map_at_5_max": 0.195177, + "nauc_map_at_5_std": 0.192356, + "nauc_map_at_5_diff1": 0.451407, + "nauc_map_at_10_max": 0.193731, + "nauc_map_at_10_std": 0.249395, + "nauc_map_at_10_diff1": 0.444132, + "nauc_map_at_20_max": 0.177565, + "nauc_map_at_20_std": 0.267327, + "nauc_map_at_20_diff1": 0.435931, + "nauc_map_at_100_max": 0.175849, + "nauc_map_at_100_std": 0.273257, + "nauc_map_at_100_diff1": 0.431702, + "nauc_map_at_1000_max": 0.175849, + "nauc_map_at_1000_std": 0.273257, + "nauc_map_at_1000_diff1": 0.431702, + "nauc_recall_at_1_max": 0.172256, + "nauc_recall_at_1_std": 0.065344, + "nauc_recall_at_1_diff1": 0.499517, + "nauc_recall_at_3_max": 0.203603, + "nauc_recall_at_3_std": 0.183553, + "nauc_recall_at_3_diff1": 0.417057, + "nauc_recall_at_5_max": 0.207777, + "nauc_recall_at_5_std": 0.233075, + "nauc_recall_at_5_diff1": 0.402338, + "nauc_recall_at_10_max": 0.216125, + "nauc_recall_at_10_std": 0.362104, + "nauc_recall_at_10_diff1": 0.3954, + "nauc_recall_at_20_max": 0.193978, + "nauc_recall_at_20_std": 0.441148, + "nauc_recall_at_20_diff1": 0.355245, + "nauc_recall_at_100_max": 0.229575, + "nauc_recall_at_100_std": 0.524463, + "nauc_recall_at_100_diff1": 0.33347, + "nauc_recall_at_1000_max": 0.229575, + "nauc_recall_at_1000_std": 0.524463, + "nauc_recall_at_1000_diff1": 0.33347, + "nauc_precision_at_1_max": 0.122183, + "nauc_precision_at_1_std": 0.17377, + "nauc_precision_at_1_diff1": 0.448708, + "nauc_precision_at_3_max": 0.055097, + "nauc_precision_at_3_std": 0.275845, + "nauc_precision_at_3_diff1": 0.103031, + "nauc_precision_at_5_max": 0.015369, + "nauc_precision_at_5_std": 0.264005, + "nauc_precision_at_5_diff1": 0.012023, + "nauc_precision_at_10_max": -0.049556, + "nauc_precision_at_10_std": 0.268921, + "nauc_precision_at_10_diff1": -0.077006, + "nauc_precision_at_20_max": -0.097734, + "nauc_precision_at_20_std": 0.212896, + "nauc_precision_at_20_diff1": -0.1337, + "nauc_precision_at_100_max": -0.109777, + "nauc_precision_at_100_std": 0.148369, + "nauc_precision_at_100_diff1": -0.19065, + "nauc_precision_at_1000_max": -0.109777, + "nauc_precision_at_1000_std": 0.148369, + "nauc_precision_at_1000_diff1": -0.19065, + "nauc_mrr_at_1_max": 0.122183, + "nauc_mrr_at_1_std": 0.17377, + "nauc_mrr_at_1_diff1": 0.448708, + "nauc_mrr_at_3_max": 0.147858, + "nauc_mrr_at_3_std": 0.263822, + "nauc_mrr_at_3_diff1": 0.45456, + "nauc_mrr_at_5_max": 0.138864, + "nauc_mrr_at_5_std": 0.262601, + "nauc_mrr_at_5_diff1": 0.452448, + "nauc_mrr_at_10_max": 0.133307, + "nauc_mrr_at_10_std": 0.258284, + "nauc_mrr_at_10_diff1": 0.44858, + "nauc_mrr_at_20_max": 0.137121, + "nauc_mrr_at_20_std": 0.258973, + "nauc_mrr_at_20_diff1": 0.44924, + "nauc_mrr_at_100_max": 0.136752, + "nauc_mrr_at_100_std": 0.259028, + "nauc_mrr_at_100_diff1": 0.44921, + "nauc_mrr_at_1000_max": 0.136752, + "nauc_mrr_at_1000_std": 0.259028, + "nauc_mrr_at_1000_diff1": 0.44921, + "hit_rate_at_1": 0.54518, + "hit_rate_at_3": 0.81325, + "hit_rate_at_5": 0.86747, + "hit_rate_at_10": 0.91265, + "hit_rate_at_20": 0.93976, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.58254, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.49849, + "ndcg_at_3": 0.51571, + "ndcg_at_5": 0.52915, + "ndcg_at_10": 0.55932, + "ndcg_at_20": 0.59489, + "ndcg_at_100": 0.62204, + "ndcg_at_1000": 0.62204, + "map_at_1": 0.2367, + "map_at_3": 0.35085, + "map_at_5": 0.3994, + "map_at_10": 0.44381, + "map_at_20": 0.4703, + "map_at_100": 0.4823, + "map_at_1000": 0.4823, + "recall_at_1": 0.2367, + "recall_at_3": 0.41597, + "recall_at_5": 0.51011, + "recall_at_10": 0.62593, + "recall_at_20": 0.72331, + "recall_at_100": 0.79964, + "recall_at_1000": 0.79964, + "accuracy": 0.2367, + "precision_at_1": 0.51807, + "precision_at_3": 0.3755, + "precision_at_5": 0.30482, + "precision_at_10": 0.21355, + "precision_at_20": 0.13735, + "precision_at_100": 0.0328, + "precision_at_1000": 0.00328, + "mrr_at_1": 0.518072, + "mrr_at_3": 0.629518, + "mrr_at_5": 0.63991, + "mrr_at_10": 0.646543, + "mrr_at_20": 0.649585, + "mrr_at_100": 0.650646, + "mrr_at_1000": 0.650646, + "nauc_ndcg_at_1_max": 0.231978, + "nauc_ndcg_at_1_std": 0.325662, + "nauc_ndcg_at_1_diff1": 0.551911, + "nauc_ndcg_at_3_max": 0.237722, + "nauc_ndcg_at_3_std": 0.332167, + "nauc_ndcg_at_3_diff1": 0.488686, + "nauc_ndcg_at_5_max": 0.269712, + "nauc_ndcg_at_5_std": 0.352911, + "nauc_ndcg_at_5_diff1": 0.495678, + "nauc_ndcg_at_10_max": 0.261625, + "nauc_ndcg_at_10_std": 0.38801, + "nauc_ndcg_at_10_diff1": 0.531241, + "nauc_ndcg_at_20_max": 0.257229, + "nauc_ndcg_at_20_std": 0.430053, + "nauc_ndcg_at_20_diff1": 0.522292, + "nauc_ndcg_at_100_max": 0.26341, + "nauc_ndcg_at_100_std": 0.436122, + "nauc_ndcg_at_100_diff1": 0.5195, + "nauc_ndcg_at_1000_max": 0.26341, + "nauc_ndcg_at_1000_std": 0.436122, + "nauc_ndcg_at_1000_diff1": 0.5195, + "nauc_map_at_1_max": 0.238784, + "nauc_map_at_1_std": 0.169992, + "nauc_map_at_1_diff1": 0.560917, + "nauc_map_at_3_max": 0.264547, + "nauc_map_at_3_std": 0.2439, + "nauc_map_at_3_diff1": 0.526114, + "nauc_map_at_5_max": 0.275546, + "nauc_map_at_5_std": 0.29067, + "nauc_map_at_5_diff1": 0.500246, + "nauc_map_at_10_max": 0.26258, + "nauc_map_at_10_std": 0.328291, + "nauc_map_at_10_diff1": 0.50963, + "nauc_map_at_20_max": 0.251626, + "nauc_map_at_20_std": 0.354603, + "nauc_map_at_20_diff1": 0.507949, + "nauc_map_at_100_max": 0.251071, + "nauc_map_at_100_std": 0.361164, + "nauc_map_at_100_diff1": 0.506516, + "nauc_map_at_1000_max": 0.251071, + "nauc_map_at_1000_std": 0.361164, + "nauc_map_at_1000_diff1": 0.506516, + "nauc_recall_at_1_max": 0.238784, + "nauc_recall_at_1_std": 0.169992, + "nauc_recall_at_1_diff1": 0.560917, + "nauc_recall_at_3_max": 0.249994, + "nauc_recall_at_3_std": 0.246744, + "nauc_recall_at_3_diff1": 0.450603, + "nauc_recall_at_5_max": 0.282208, + "nauc_recall_at_5_std": 0.315521, + "nauc_recall_at_5_diff1": 0.420332, + "nauc_recall_at_10_max": 0.254425, + "nauc_recall_at_10_std": 0.409872, + "nauc_recall_at_10_diff1": 0.452034, + "nauc_recall_at_20_max": 0.220436, + "nauc_recall_at_20_std": 0.513657, + "nauc_recall_at_20_diff1": 0.422349, + "nauc_recall_at_100_max": 0.251553, + "nauc_recall_at_100_std": 0.572932, + "nauc_recall_at_100_diff1": 0.40964, + "nauc_recall_at_1000_max": 0.251553, + "nauc_recall_at_1000_std": 0.572932, + "nauc_recall_at_1000_diff1": 0.40964, + "nauc_precision_at_1_max": 0.238425, + "nauc_precision_at_1_std": 0.358917, + "nauc_precision_at_1_diff1": 0.558252, + "nauc_precision_at_3_max": 0.095313, + "nauc_precision_at_3_std": 0.332248, + "nauc_precision_at_3_diff1": 0.201097, + "nauc_precision_at_5_max": 0.086828, + "nauc_precision_at_5_std": 0.316935, + "nauc_precision_at_5_diff1": 0.083139, + "nauc_precision_at_10_max": -0.025393, + "nauc_precision_at_10_std": 0.274178, + "nauc_precision_at_10_diff1": 0.027653, + "nauc_precision_at_20_max": -0.074581, + "nauc_precision_at_20_std": 0.251691, + "nauc_precision_at_20_diff1": -0.051619, + "nauc_precision_at_100_max": -0.096113, + "nauc_precision_at_100_std": 0.171671, + "nauc_precision_at_100_diff1": -0.12327, + "nauc_precision_at_1000_max": -0.096113, + "nauc_precision_at_1000_std": 0.171671, + "nauc_precision_at_1000_diff1": -0.12327, + "nauc_mrr_at_1_max": 0.238425, + "nauc_mrr_at_1_std": 0.358917, + "nauc_mrr_at_1_diff1": 0.558252, + "nauc_mrr_at_3_max": 0.22368, + "nauc_mrr_at_3_std": 0.39466, + "nauc_mrr_at_3_diff1": 0.549104, + "nauc_mrr_at_5_max": 0.231898, + "nauc_mrr_at_5_std": 0.404018, + "nauc_mrr_at_5_diff1": 0.553865, + "nauc_mrr_at_10_max": 0.236308, + "nauc_mrr_at_10_std": 0.411378, + "nauc_mrr_at_10_diff1": 0.551317, + "nauc_mrr_at_20_max": 0.23593, + "nauc_mrr_at_20_std": 0.411083, + "nauc_mrr_at_20_diff1": 0.550837, + "nauc_mrr_at_100_max": 0.235979, + "nauc_mrr_at_100_std": 0.409314, + "nauc_mrr_at_100_diff1": 0.551862, + "nauc_mrr_at_1000_max": 0.235979, + "nauc_mrr_at_1000_std": 0.409314, + "nauc_mrr_at_1000_diff1": 0.551862, + "hit_rate_at_1": 0.51807, + "hit_rate_at_3": 0.77711, + "hit_rate_at_5": 0.8253, + "hit_rate_at_10": 0.87651, + "hit_rate_at_20": 0.91867, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.55932, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.51958, + "ndcg_at_3": 0.5247, + "ndcg_at_5": 0.53915, + "ndcg_at_10": 0.56602, + "ndcg_at_20": 0.60451, + "ndcg_at_100": 0.6313, + "ndcg_at_1000": 0.6313, + "map_at_1": 0.24484, + "map_at_3": 0.35809, + "map_at_5": 0.40577, + "map_at_10": 0.4506, + "map_at_20": 0.47913, + "map_at_100": 0.49063, + "map_at_1000": 0.49063, + "recall_at_1": 0.24484, + "recall_at_3": 0.41761, + "recall_at_5": 0.51268, + "recall_at_10": 0.6223, + "recall_at_20": 0.72811, + "recall_at_100": 0.80754, + "recall_at_1000": 0.80754, + "accuracy": 0.24484, + "precision_at_1": 0.53916, + "precision_at_3": 0.38052, + "precision_at_5": 0.30964, + "precision_at_10": 0.21596, + "precision_at_20": 0.14021, + "precision_at_100": 0.03304, + "precision_at_1000": 0.0033, + "mrr_at_1": 0.539157, + "mrr_at_3": 0.650602, + "mrr_at_5": 0.661446, + "mrr_at_10": 0.667664, + "mrr_at_20": 0.671061, + "mrr_at_100": 0.671662, + "mrr_at_1000": 0.671662, + "nauc_ndcg_at_1_max": 0.179678, + "nauc_ndcg_at_1_std": 0.22546, + "nauc_ndcg_at_1_diff1": 0.518095, + "nauc_ndcg_at_3_max": 0.191996, + "nauc_ndcg_at_3_std": 0.274759, + "nauc_ndcg_at_3_diff1": 0.449132, + "nauc_ndcg_at_5_max": 0.230402, + "nauc_ndcg_at_5_std": 0.292291, + "nauc_ndcg_at_5_diff1": 0.466968, + "nauc_ndcg_at_10_max": 0.236988, + "nauc_ndcg_at_10_std": 0.337136, + "nauc_ndcg_at_10_diff1": 0.501183, + "nauc_ndcg_at_20_max": 0.225278, + "nauc_ndcg_at_20_std": 0.385197, + "nauc_ndcg_at_20_diff1": 0.483807, + "nauc_ndcg_at_100_max": 0.229555, + "nauc_ndcg_at_100_std": 0.385914, + "nauc_ndcg_at_100_diff1": 0.473395, + "nauc_ndcg_at_1000_max": 0.229555, + "nauc_ndcg_at_1000_std": 0.385914, + "nauc_ndcg_at_1000_diff1": 0.473395, + "nauc_map_at_1_max": 0.206352, + "nauc_map_at_1_std": 0.094619, + "nauc_map_at_1_diff1": 0.549203, + "nauc_map_at_3_max": 0.22534, + "nauc_map_at_3_std": 0.17986, + "nauc_map_at_3_diff1": 0.518102, + "nauc_map_at_5_max": 0.240578, + "nauc_map_at_5_std": 0.229839, + "nauc_map_at_5_diff1": 0.493214, + "nauc_map_at_10_max": 0.235248, + "nauc_map_at_10_std": 0.279617, + "nauc_map_at_10_diff1": 0.492694, + "nauc_map_at_20_max": 0.220791, + "nauc_map_at_20_std": 0.310539, + "nauc_map_at_20_diff1": 0.482694, + "nauc_map_at_100_max": 0.220814, + "nauc_map_at_100_std": 0.315866, + "nauc_map_at_100_diff1": 0.47996, + "nauc_map_at_1000_max": 0.220814, + "nauc_map_at_1000_std": 0.315866, + "nauc_map_at_1000_diff1": 0.47996, + "nauc_recall_at_1_max": 0.206352, + "nauc_recall_at_1_std": 0.094619, + "nauc_recall_at_1_diff1": 0.549203, + "nauc_recall_at_3_max": 0.218308, + "nauc_recall_at_3_std": 0.199247, + "nauc_recall_at_3_diff1": 0.452643, + "nauc_recall_at_5_max": 0.27385, + "nauc_recall_at_5_std": 0.275626, + "nauc_recall_at_5_diff1": 0.409761, + "nauc_recall_at_10_max": 0.261448, + "nauc_recall_at_10_std": 0.382849, + "nauc_recall_at_10_diff1": 0.425981, + "nauc_recall_at_20_max": 0.222606, + "nauc_recall_at_20_std": 0.524226, + "nauc_recall_at_20_diff1": 0.364779, + "nauc_recall_at_100_max": 0.254076, + "nauc_recall_at_100_std": 0.581078, + "nauc_recall_at_100_diff1": 0.311439, + "nauc_recall_at_1000_max": 0.254076, + "nauc_recall_at_1000_std": 0.581078, + "nauc_recall_at_1000_diff1": 0.311439, + "nauc_precision_at_1_max": 0.186357, + "nauc_precision_at_1_std": 0.249926, + "nauc_precision_at_1_diff1": 0.530727, + "nauc_precision_at_3_max": 0.047405, + "nauc_precision_at_3_std": 0.299666, + "nauc_precision_at_3_diff1": 0.157803, + "nauc_precision_at_5_max": 0.026249, + "nauc_precision_at_5_std": 0.291846, + "nauc_precision_at_5_diff1": 0.028786, + "nauc_precision_at_10_max": -0.050493, + "nauc_precision_at_10_std": 0.29097, + "nauc_precision_at_10_diff1": -0.035256, + "nauc_precision_at_20_max": -0.101145, + "nauc_precision_at_20_std": 0.268001, + "nauc_precision_at_20_diff1": -0.118026, + "nauc_precision_at_100_max": -0.110096, + "nauc_precision_at_100_std": 0.189063, + "nauc_precision_at_100_diff1": -0.181896, + "nauc_precision_at_1000_max": -0.110096, + "nauc_precision_at_1000_std": 0.189063, + "nauc_precision_at_1000_diff1": -0.181896, + "nauc_mrr_at_1_max": 0.186357, + "nauc_mrr_at_1_std": 0.249926, + "nauc_mrr_at_1_diff1": 0.530727, + "nauc_mrr_at_3_max": 0.174078, + "nauc_mrr_at_3_std": 0.302244, + "nauc_mrr_at_3_diff1": 0.519979, + "nauc_mrr_at_5_max": 0.175192, + "nauc_mrr_at_5_std": 0.300898, + "nauc_mrr_at_5_diff1": 0.520753, + "nauc_mrr_at_10_max": 0.174572, + "nauc_mrr_at_10_std": 0.302615, + "nauc_mrr_at_10_diff1": 0.521481, + "nauc_mrr_at_20_max": 0.173763, + "nauc_mrr_at_20_std": 0.303179, + "nauc_mrr_at_20_diff1": 0.521902, + "nauc_mrr_at_100_max": 0.175388, + "nauc_mrr_at_100_std": 0.302258, + "nauc_mrr_at_100_diff1": 0.521607, + "nauc_mrr_at_1000_max": 0.175388, + "nauc_mrr_at_1000_std": 0.302258, + "nauc_mrr_at_1000_diff1": 0.521607, + "hit_rate_at_1": 0.53916, + "hit_rate_at_3": 0.78916, + "hit_rate_at_5": 0.83735, + "hit_rate_at_10": 0.88253, + "hit_rate_at_20": 0.93373, + "hit_rate_at_100": 0.95181, + "hit_rate_at_1000": 0.95181, + "main_score": 0.56602, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.50301, + "ndcg_at_3": 0.52539, + "ndcg_at_5": 0.53784, + "ndcg_at_10": 0.56633, + "ndcg_at_20": 0.60653, + "ndcg_at_100": 0.63228, + "ndcg_at_1000": 0.63228, + "map_at_1": 0.24418, + "map_at_3": 0.35969, + "map_at_5": 0.40721, + "map_at_10": 0.45147, + "map_at_20": 0.48041, + "map_at_100": 0.49196, + "map_at_1000": 0.49196, + "recall_at_1": 0.24418, + "recall_at_3": 0.42123, + "recall_at_5": 0.51216, + "recall_at_10": 0.62645, + "recall_at_20": 0.73317, + "recall_at_100": 0.80792, + "recall_at_1000": 0.80792, + "accuracy": 0.24418, + "precision_at_1": 0.52108, + "precision_at_3": 0.38253, + "precision_at_5": 0.30964, + "precision_at_10": 0.21596, + "precision_at_20": 0.14142, + "precision_at_100": 0.03334, + "precision_at_1000": 0.00333, + "mrr_at_1": 0.521084, + "mrr_at_3": 0.634538, + "mrr_at_5": 0.647189, + "mrr_at_10": 0.654455, + "mrr_at_20": 0.657709, + "mrr_at_100": 0.658344, + "mrr_at_1000": 0.658344, + "nauc_ndcg_at_1_max": 0.17889, + "nauc_ndcg_at_1_std": 0.259055, + "nauc_ndcg_at_1_diff1": 0.579272, + "nauc_ndcg_at_3_max": 0.218429, + "nauc_ndcg_at_3_std": 0.294629, + "nauc_ndcg_at_3_diff1": 0.495362, + "nauc_ndcg_at_5_max": 0.239186, + "nauc_ndcg_at_5_std": 0.307379, + "nauc_ndcg_at_5_diff1": 0.505626, + "nauc_ndcg_at_10_max": 0.246509, + "nauc_ndcg_at_10_std": 0.351578, + "nauc_ndcg_at_10_diff1": 0.538968, + "nauc_ndcg_at_20_max": 0.22638, + "nauc_ndcg_at_20_std": 0.384889, + "nauc_ndcg_at_20_diff1": 0.540331, + "nauc_ndcg_at_100_max": 0.230083, + "nauc_ndcg_at_100_std": 0.388997, + "nauc_ndcg_at_100_diff1": 0.537891, + "nauc_ndcg_at_1000_max": 0.230083, + "nauc_ndcg_at_1000_std": 0.388997, + "nauc_ndcg_at_1000_diff1": 0.537891, + "nauc_map_at_1_max": 0.200182, + "nauc_map_at_1_std": 0.116023, + "nauc_map_at_1_diff1": 0.558982, + "nauc_map_at_3_max": 0.236276, + "nauc_map_at_3_std": 0.214002, + "nauc_map_at_3_diff1": 0.534157, + "nauc_map_at_5_max": 0.244204, + "nauc_map_at_5_std": 0.255723, + "nauc_map_at_5_diff1": 0.514025, + "nauc_map_at_10_max": 0.242061, + "nauc_map_at_10_std": 0.296137, + "nauc_map_at_10_diff1": 0.522925, + "nauc_map_at_20_max": 0.221091, + "nauc_map_at_20_std": 0.315806, + "nauc_map_at_20_diff1": 0.520172, + "nauc_map_at_100_max": 0.219166, + "nauc_map_at_100_std": 0.32183, + "nauc_map_at_100_diff1": 0.519822, + "nauc_map_at_1000_max": 0.219166, + "nauc_map_at_1000_std": 0.32183, + "nauc_map_at_1000_diff1": 0.519822, + "nauc_recall_at_1_max": 0.200182, + "nauc_recall_at_1_std": 0.116023, + "nauc_recall_at_1_diff1": 0.558982, + "nauc_recall_at_3_max": 0.248681, + "nauc_recall_at_3_std": 0.240717, + "nauc_recall_at_3_diff1": 0.469709, + "nauc_recall_at_5_max": 0.274671, + "nauc_recall_at_5_std": 0.295429, + "nauc_recall_at_5_diff1": 0.438406, + "nauc_recall_at_10_max": 0.273507, + "nauc_recall_at_10_std": 0.400624, + "nauc_recall_at_10_diff1": 0.456557, + "nauc_recall_at_20_max": 0.199795, + "nauc_recall_at_20_std": 0.500312, + "nauc_recall_at_20_diff1": 0.44701, + "nauc_recall_at_100_max": 0.231955, + "nauc_recall_at_100_std": 0.565051, + "nauc_recall_at_100_diff1": 0.439948, + "nauc_recall_at_1000_max": 0.231955, + "nauc_recall_at_1000_std": 0.565051, + "nauc_recall_at_1000_diff1": 0.439948, + "nauc_precision_at_1_max": 0.192341, + "nauc_precision_at_1_std": 0.285954, + "nauc_precision_at_1_diff1": 0.590099, + "nauc_precision_at_3_max": 0.079058, + "nauc_precision_at_3_std": 0.289052, + "nauc_precision_at_3_diff1": 0.209137, + "nauc_precision_at_5_max": 0.046211, + "nauc_precision_at_5_std": 0.272913, + "nauc_precision_at_5_diff1": 0.083027, + "nauc_precision_at_10_max": -0.029327, + "nauc_precision_at_10_std": 0.251302, + "nauc_precision_at_10_diff1": 0.010091, + "nauc_precision_at_20_max": -0.110764, + "nauc_precision_at_20_std": 0.204425, + "nauc_precision_at_20_diff1": -0.084063, + "nauc_precision_at_100_max": -0.121265, + "nauc_precision_at_100_std": 0.141821, + "nauc_precision_at_100_diff1": -0.142545, + "nauc_precision_at_1000_max": -0.121265, + "nauc_precision_at_1000_std": 0.141821, + "nauc_precision_at_1000_diff1": -0.142545, + "nauc_mrr_at_1_max": 0.192341, + "nauc_mrr_at_1_std": 0.285954, + "nauc_mrr_at_1_diff1": 0.590099, + "nauc_mrr_at_3_max": 0.199544, + "nauc_mrr_at_3_std": 0.335156, + "nauc_mrr_at_3_diff1": 0.570469, + "nauc_mrr_at_5_max": 0.206803, + "nauc_mrr_at_5_std": 0.35171, + "nauc_mrr_at_5_diff1": 0.569865, + "nauc_mrr_at_10_max": 0.203222, + "nauc_mrr_at_10_std": 0.350306, + "nauc_mrr_at_10_diff1": 0.56978, + "nauc_mrr_at_20_max": 0.202325, + "nauc_mrr_at_20_std": 0.349581, + "nauc_mrr_at_20_diff1": 0.570845, + "nauc_mrr_at_100_max": 0.203258, + "nauc_mrr_at_100_std": 0.348877, + "nauc_mrr_at_100_diff1": 0.570812, + "nauc_mrr_at_1000_max": 0.203258, + "nauc_mrr_at_1000_std": 0.348877, + "nauc_mrr_at_1000_diff1": 0.570812, + "hit_rate_at_1": 0.52108, + "hit_rate_at_3": 0.77711, + "hit_rate_at_5": 0.83133, + "hit_rate_at_10": 0.88554, + "hit_rate_at_20": 0.93072, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.56633, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.50452, + "ndcg_at_3": 0.52779, + "ndcg_at_5": 0.53262, + "ndcg_at_10": 0.56525, + "ndcg_at_20": 0.60499, + "ndcg_at_100": 0.63105, + "ndcg_at_1000": 0.63105, + "map_at_1": 0.24181, + "map_at_3": 0.36316, + "map_at_5": 0.40582, + "map_at_10": 0.45144, + "map_at_20": 0.48071, + "map_at_100": 0.49258, + "map_at_1000": 0.49258, + "recall_at_1": 0.24181, + "recall_at_3": 0.42689, + "recall_at_5": 0.50488, + "recall_at_10": 0.62475, + "recall_at_20": 0.73287, + "recall_at_100": 0.80575, + "recall_at_1000": 0.80575, + "accuracy": 0.24181, + "precision_at_1": 0.52711, + "precision_at_3": 0.38554, + "precision_at_5": 0.30422, + "precision_at_10": 0.21627, + "precision_at_20": 0.14157, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.527108, + "mrr_at_3": 0.638052, + "mrr_at_5": 0.647992, + "mrr_at_10": 0.656762, + "mrr_at_20": 0.659625, + "mrr_at_100": 0.660617, + "mrr_at_1000": 0.660617, + "nauc_ndcg_at_1_max": 0.235777, + "nauc_ndcg_at_1_std": 0.286191, + "nauc_ndcg_at_1_diff1": 0.553448, + "nauc_ndcg_at_3_max": 0.240828, + "nauc_ndcg_at_3_std": 0.317847, + "nauc_ndcg_at_3_diff1": 0.474999, + "nauc_ndcg_at_5_max": 0.264983, + "nauc_ndcg_at_5_std": 0.333974, + "nauc_ndcg_at_5_diff1": 0.499616, + "nauc_ndcg_at_10_max": 0.267478, + "nauc_ndcg_at_10_std": 0.371845, + "nauc_ndcg_at_10_diff1": 0.531661, + "nauc_ndcg_at_20_max": 0.253028, + "nauc_ndcg_at_20_std": 0.405793, + "nauc_ndcg_at_20_diff1": 0.535354, + "nauc_ndcg_at_100_max": 0.258669, + "nauc_ndcg_at_100_std": 0.408764, + "nauc_ndcg_at_100_diff1": 0.530675, + "nauc_ndcg_at_1000_max": 0.258669, + "nauc_ndcg_at_1000_std": 0.408764, + "nauc_ndcg_at_1000_diff1": 0.530675, + "nauc_map_at_1_max": 0.239817, + "nauc_map_at_1_std": 0.167706, + "nauc_map_at_1_diff1": 0.56721, + "nauc_map_at_3_max": 0.255469, + "nauc_map_at_3_std": 0.227743, + "nauc_map_at_3_diff1": 0.531538, + "nauc_map_at_5_max": 0.268043, + "nauc_map_at_5_std": 0.274916, + "nauc_map_at_5_diff1": 0.513207, + "nauc_map_at_10_max": 0.259832, + "nauc_map_at_10_std": 0.307392, + "nauc_map_at_10_diff1": 0.518339, + "nauc_map_at_20_max": 0.243242, + "nauc_map_at_20_std": 0.327362, + "nauc_map_at_20_diff1": 0.518477, + "nauc_map_at_100_max": 0.243181, + "nauc_map_at_100_std": 0.334214, + "nauc_map_at_100_diff1": 0.516686, + "nauc_map_at_1000_max": 0.243181, + "nauc_map_at_1000_std": 0.334214, + "nauc_map_at_1000_diff1": 0.516686, + "nauc_recall_at_1_max": 0.239817, + "nauc_recall_at_1_std": 0.167706, + "nauc_recall_at_1_diff1": 0.56721, + "nauc_recall_at_3_max": 0.24865, + "nauc_recall_at_3_std": 0.239975, + "nauc_recall_at_3_diff1": 0.45716, + "nauc_recall_at_5_max": 0.269866, + "nauc_recall_at_5_std": 0.312051, + "nauc_recall_at_5_diff1": 0.445086, + "nauc_recall_at_10_max": 0.267044, + "nauc_recall_at_10_std": 0.398836, + "nauc_recall_at_10_diff1": 0.473728, + "nauc_recall_at_20_max": 0.208072, + "nauc_recall_at_20_std": 0.491728, + "nauc_recall_at_20_diff1": 0.476233, + "nauc_recall_at_100_max": 0.245669, + "nauc_recall_at_100_std": 0.535799, + "nauc_recall_at_100_diff1": 0.468733, + "nauc_recall_at_1000_max": 0.245669, + "nauc_recall_at_1000_std": 0.535799, + "nauc_recall_at_1000_diff1": 0.468733, + "nauc_precision_at_1_max": 0.24059, + "nauc_precision_at_1_std": 0.307001, + "nauc_precision_at_1_diff1": 0.5548, + "nauc_precision_at_3_max": 0.08839, + "nauc_precision_at_3_std": 0.281655, + "nauc_precision_at_3_diff1": 0.18474, + "nauc_precision_at_5_max": 0.081177, + "nauc_precision_at_5_std": 0.286114, + "nauc_precision_at_5_diff1": 0.093731, + "nauc_precision_at_10_max": -0.029738, + "nauc_precision_at_10_std": 0.243621, + "nauc_precision_at_10_diff1": -0.002071, + "nauc_precision_at_20_max": -0.102324, + "nauc_precision_at_20_std": 0.194209, + "nauc_precision_at_20_diff1": -0.07422, + "nauc_precision_at_100_max": -0.111146, + "nauc_precision_at_100_std": 0.129771, + "nauc_precision_at_100_diff1": -0.144916, + "nauc_precision_at_1000_max": -0.111146, + "nauc_precision_at_1000_std": 0.129771, + "nauc_precision_at_1000_diff1": -0.144916, + "nauc_mrr_at_1_max": 0.24059, + "nauc_mrr_at_1_std": 0.307001, + "nauc_mrr_at_1_diff1": 0.5548, + "nauc_mrr_at_3_max": 0.249091, + "nauc_mrr_at_3_std": 0.386206, + "nauc_mrr_at_3_diff1": 0.533331, + "nauc_mrr_at_5_max": 0.247621, + "nauc_mrr_at_5_std": 0.391272, + "nauc_mrr_at_5_diff1": 0.542005, + "nauc_mrr_at_10_max": 0.244487, + "nauc_mrr_at_10_std": 0.388792, + "nauc_mrr_at_10_diff1": 0.538574, + "nauc_mrr_at_20_max": 0.246498, + "nauc_mrr_at_20_std": 0.389725, + "nauc_mrr_at_20_diff1": 0.537755, + "nauc_mrr_at_100_max": 0.246197, + "nauc_mrr_at_100_std": 0.387729, + "nauc_mrr_at_100_diff1": 0.538284, + "nauc_mrr_at_1000_max": 0.246197, + "nauc_mrr_at_1000_std": 0.387729, + "nauc_mrr_at_1000_diff1": 0.538284, + "hit_rate_at_1": 0.52711, + "hit_rate_at_3": 0.7741, + "hit_rate_at_5": 0.81627, + "hit_rate_at_10": 0.87952, + "hit_rate_at_20": 0.92169, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.56525, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.50301, + "ndcg_at_3": 0.51621, + "ndcg_at_5": 0.53197, + "ndcg_at_10": 0.565, + "ndcg_at_20": 0.60191, + "ndcg_at_100": 0.62933, + "ndcg_at_1000": 0.62933, + "map_at_1": 0.24121, + "map_at_3": 0.35618, + "map_at_5": 0.40491, + "map_at_10": 0.45021, + "map_at_20": 0.47888, + "map_at_100": 0.49104, + "map_at_1000": 0.49104, + "recall_at_1": 0.24121, + "recall_at_3": 0.41146, + "recall_at_5": 0.50767, + "recall_at_10": 0.62754, + "recall_at_20": 0.72997, + "recall_at_100": 0.80599, + "recall_at_1000": 0.80599, + "accuracy": 0.24121, + "precision_at_1": 0.5241, + "precision_at_3": 0.37349, + "precision_at_5": 0.30482, + "precision_at_10": 0.21747, + "precision_at_20": 0.13976, + "precision_at_100": 0.03337, + "precision_at_1000": 0.00334, + "mrr_at_1": 0.524096, + "mrr_at_3": 0.625502, + "mrr_at_5": 0.640261, + "mrr_at_10": 0.649574, + "mrr_at_20": 0.6523, + "mrr_at_100": 0.653026, + "mrr_at_1000": 0.653026, + "nauc_ndcg_at_1_max": 0.257775, + "nauc_ndcg_at_1_std": 0.304645, + "nauc_ndcg_at_1_diff1": 0.555733, + "nauc_ndcg_at_3_max": 0.285323, + "nauc_ndcg_at_3_std": 0.328533, + "nauc_ndcg_at_3_diff1": 0.491441, + "nauc_ndcg_at_5_max": 0.294733, + "nauc_ndcg_at_5_std": 0.33209, + "nauc_ndcg_at_5_diff1": 0.50534, + "nauc_ndcg_at_10_max": 0.295364, + "nauc_ndcg_at_10_std": 0.368805, + "nauc_ndcg_at_10_diff1": 0.534242, + "nauc_ndcg_at_20_max": 0.295602, + "nauc_ndcg_at_20_std": 0.412745, + "nauc_ndcg_at_20_diff1": 0.533182, + "nauc_ndcg_at_100_max": 0.29746, + "nauc_ndcg_at_100_std": 0.414372, + "nauc_ndcg_at_100_diff1": 0.52578, + "nauc_ndcg_at_1000_max": 0.29746, + "nauc_ndcg_at_1000_std": 0.414372, + "nauc_ndcg_at_1000_diff1": 0.52578, + "nauc_map_at_1_max": 0.26569, + "nauc_map_at_1_std": 0.135229, + "nauc_map_at_1_diff1": 0.582538, + "nauc_map_at_3_max": 0.291742, + "nauc_map_at_3_std": 0.227745, + "nauc_map_at_3_diff1": 0.559149, + "nauc_map_at_5_max": 0.290134, + "nauc_map_at_5_std": 0.258386, + "nauc_map_at_5_diff1": 0.533395, + "nauc_map_at_10_max": 0.289871, + "nauc_map_at_10_std": 0.301252, + "nauc_map_at_10_diff1": 0.528715, + "nauc_map_at_20_max": 0.280401, + "nauc_map_at_20_std": 0.328679, + "nauc_map_at_20_diff1": 0.523477, + "nauc_map_at_100_max": 0.279792, + "nauc_map_at_100_std": 0.334044, + "nauc_map_at_100_diff1": 0.519866, + "nauc_map_at_1000_max": 0.279792, + "nauc_map_at_1000_std": 0.334044, + "nauc_map_at_1000_diff1": 0.519866, + "nauc_recall_at_1_max": 0.26569, + "nauc_recall_at_1_std": 0.135229, + "nauc_recall_at_1_diff1": 0.582538, + "nauc_recall_at_3_max": 0.287183, + "nauc_recall_at_3_std": 0.239698, + "nauc_recall_at_3_diff1": 0.494633, + "nauc_recall_at_5_max": 0.2978, + "nauc_recall_at_5_std": 0.291605, + "nauc_recall_at_5_diff1": 0.463757, + "nauc_recall_at_10_max": 0.283618, + "nauc_recall_at_10_std": 0.392221, + "nauc_recall_at_10_diff1": 0.47172, + "nauc_recall_at_20_max": 0.274122, + "nauc_recall_at_20_std": 0.512735, + "nauc_recall_at_20_diff1": 0.45688, + "nauc_recall_at_100_max": 0.282707, + "nauc_recall_at_100_std": 0.539892, + "nauc_recall_at_100_diff1": 0.428154, + "nauc_recall_at_1000_max": 0.282707, + "nauc_recall_at_1000_std": 0.539892, + "nauc_recall_at_1000_diff1": 0.428154, + "nauc_precision_at_1_max": 0.259469, + "nauc_precision_at_1_std": 0.337328, + "nauc_precision_at_1_diff1": 0.556625, + "nauc_precision_at_3_max": 0.137883, + "nauc_precision_at_3_std": 0.341851, + "nauc_precision_at_3_diff1": 0.191565, + "nauc_precision_at_5_max": 0.06758, + "nauc_precision_at_5_std": 0.29121, + "nauc_precision_at_5_diff1": 0.055276, + "nauc_precision_at_10_max": -0.027425, + "nauc_precision_at_10_std": 0.25027, + "nauc_precision_at_10_diff1": -0.043061, + "nauc_precision_at_20_max": -0.074706, + "nauc_precision_at_20_std": 0.233124, + "nauc_precision_at_20_diff1": -0.111019, + "nauc_precision_at_100_max": -0.104433, + "nauc_precision_at_100_std": 0.14932, + "nauc_precision_at_100_diff1": -0.187008, + "nauc_precision_at_1000_max": -0.104433, + "nauc_precision_at_1000_std": 0.14932, + "nauc_precision_at_1000_diff1": -0.187008, + "nauc_mrr_at_1_max": 0.259469, + "nauc_mrr_at_1_std": 0.337328, + "nauc_mrr_at_1_diff1": 0.556625, + "nauc_mrr_at_3_max": 0.267572, + "nauc_mrr_at_3_std": 0.379753, + "nauc_mrr_at_3_diff1": 0.527721, + "nauc_mrr_at_5_max": 0.269226, + "nauc_mrr_at_5_std": 0.397271, + "nauc_mrr_at_5_diff1": 0.542511, + "nauc_mrr_at_10_max": 0.26543, + "nauc_mrr_at_10_std": 0.399137, + "nauc_mrr_at_10_diff1": 0.541762, + "nauc_mrr_at_20_max": 0.269366, + "nauc_mrr_at_20_std": 0.399429, + "nauc_mrr_at_20_diff1": 0.541168, + "nauc_mrr_at_100_max": 0.268789, + "nauc_mrr_at_100_std": 0.398073, + "nauc_mrr_at_100_diff1": 0.541648, + "nauc_mrr_at_1000_max": 0.268789, + "nauc_mrr_at_1000_std": 0.398073, + "nauc_mrr_at_1000_diff1": 0.541648, + "hit_rate_at_1": 0.5241, + "hit_rate_at_3": 0.75, + "hit_rate_at_5": 0.81325, + "hit_rate_at_10": 0.88554, + "hit_rate_at_20": 0.9247, + "hit_rate_at_100": 0.9488, + "hit_rate_at_1000": 0.9488, + "main_score": 0.565, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 20444.412023543962, + "kg_co2_emissions": null, + "date": 1792442747.853016, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 3.8277008570730686e-05, + "end": 11.584298671106808, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 11.584326192038134, + "end": 11.584330659010448, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries and documents", + "start": 11.589269364019856, + "end": 3410.754373177071, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 3410.754462901037, + "end": 3410.8066971990047, + "split": "test", + "subset": "english" + } + ] +} diff --git a/results/nvidia__Nemotron-3.5-Rerank-8B-BF16/708e844c3da0bbaa923e9dc310b07da4ffb6a6b5/Vidore3TelecomNemotron35Embed8BReranking.v2.json b/results/nvidia__Nemotron-3.5-Rerank-8B-BF16/708e844c3da0bbaa923e9dc310b07da4ffb6a6b5/Vidore3TelecomNemotron35Embed8BReranking.v2.json new file mode 100644 index 0000000000..5235ccc11f --- /dev/null +++ b/results/nvidia__Nemotron-3.5-Rerank-8B-BF16/708e844c3da0bbaa923e9dc310b07da4ffb6a6b5/Vidore3TelecomNemotron35Embed8BReranking.v2.json @@ -0,0 +1,978 @@ +{ + "dataset_revision": "47c9b703527792f3fbf0a082f4589ff40c1aa8cc", + "task_name": "Vidore3TelecomNemotron35Embed8BReranking.v2", + "mteb_version": "2.26.14", + "scores": { + "test": [ + { + "ndcg_at_1": 0.77874, + "ndcg_at_3": 0.74387, + "ndcg_at_5": 0.7464, + "ndcg_at_10": 0.76819, + "ndcg_at_20": 0.79837, + "ndcg_at_100": 0.81635, + "ndcg_at_1000": 0.81635, + "map_at_1": 0.35046, + "map_at_3": 0.54438, + "map_at_5": 0.60616, + "map_at_10": 0.66127, + "map_at_20": 0.69277, + "map_at_100": 0.70474, + "map_at_1000": 0.70474, + "recall_at_1": 0.35046, + "recall_at_3": 0.57825, + "recall_at_5": 0.67725, + "recall_at_10": 0.78824, + "recall_at_20": 0.88232, + "recall_at_100": 0.92961, + "recall_at_1000": 0.92961, + "accuracy": 0.35046, + "precision_at_1": 0.8046, + "precision_at_3": 0.55747, + "precision_at_5": 0.42874, + "precision_at_10": 0.28247, + "precision_at_20": 0.17399, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.804598, + "mrr_at_3": 0.854406, + "mrr_at_5": 0.863314, + "mrr_at_10": 0.866574, + "mrr_at_20": 0.867467, + "mrr_at_100": 0.867467, + "mrr_at_1000": 0.867467, + "nauc_ndcg_at_1_max": 0.392464, + "nauc_ndcg_at_1_std": 0.385054, + "nauc_ndcg_at_1_diff1": 0.565952, + "nauc_ndcg_at_3_max": 0.422868, + "nauc_ndcg_at_3_std": 0.38144, + "nauc_ndcg_at_3_diff1": 0.432019, + "nauc_ndcg_at_5_max": 0.43336, + "nauc_ndcg_at_5_std": 0.413378, + "nauc_ndcg_at_5_diff1": 0.43557, + "nauc_ndcg_at_10_max": 0.449833, + "nauc_ndcg_at_10_std": 0.43045, + "nauc_ndcg_at_10_diff1": 0.448395, + "nauc_ndcg_at_20_max": 0.464147, + "nauc_ndcg_at_20_std": 0.456696, + "nauc_ndcg_at_20_diff1": 0.472843, + "nauc_ndcg_at_100_max": 0.467408, + "nauc_ndcg_at_100_std": 0.460437, + "nauc_ndcg_at_100_diff1": 0.471422, + "nauc_ndcg_at_1000_max": 0.467408, + "nauc_ndcg_at_1000_std": 0.460437, + "nauc_ndcg_at_1000_diff1": 0.471422, + "nauc_map_at_1_max": 0.222475, + "nauc_map_at_1_std": 0.03514, + "nauc_map_at_1_diff1": 0.499011, + "nauc_map_at_3_max": 0.3403, + "nauc_map_at_3_std": 0.192032, + "nauc_map_at_3_diff1": 0.411125, + "nauc_map_at_5_max": 0.367481, + "nauc_map_at_5_std": 0.264987, + "nauc_map_at_5_diff1": 0.407903, + "nauc_map_at_10_max": 0.395522, + "nauc_map_at_10_std": 0.340364, + "nauc_map_at_10_diff1": 0.398103, + "nauc_map_at_20_max": 0.407047, + "nauc_map_at_20_std": 0.370775, + "nauc_map_at_20_diff1": 0.408616, + "nauc_map_at_100_max": 0.409599, + "nauc_map_at_100_std": 0.377918, + "nauc_map_at_100_diff1": 0.408577, + "nauc_map_at_1000_max": 0.409599, + "nauc_map_at_1000_std": 0.377918, + "nauc_map_at_1000_diff1": 0.408577, + "nauc_recall_at_1_max": 0.222475, + "nauc_recall_at_1_std": 0.03514, + "nauc_recall_at_1_diff1": 0.499011, + "nauc_recall_at_3_max": 0.337208, + "nauc_recall_at_3_std": 0.192736, + "nauc_recall_at_3_diff1": 0.370687, + "nauc_recall_at_5_max": 0.350291, + "nauc_recall_at_5_std": 0.297042, + "nauc_recall_at_5_diff1": 0.344032, + "nauc_recall_at_10_max": 0.416952, + "nauc_recall_at_10_std": 0.43018, + "nauc_recall_at_10_diff1": 0.327287, + "nauc_recall_at_20_max": 0.491729, + "nauc_recall_at_20_std": 0.541021, + "nauc_recall_at_20_diff1": 0.409032, + "nauc_recall_at_100_max": 0.593095, + "nauc_recall_at_100_std": 0.648278, + "nauc_recall_at_100_diff1": 0.386115, + "nauc_recall_at_1000_max": 0.593095, + "nauc_recall_at_1000_std": 0.648278, + "nauc_recall_at_1000_diff1": 0.386115, + "nauc_precision_at_1_max": 0.437481, + "nauc_precision_at_1_std": 0.379887, + "nauc_precision_at_1_diff1": 0.557019, + "nauc_precision_at_3_max": 0.199609, + "nauc_precision_at_3_std": 0.346988, + "nauc_precision_at_3_diff1": -0.048188, + "nauc_precision_at_5_max": 0.098089, + "nauc_precision_at_5_std": 0.313431, + "nauc_precision_at_5_diff1": -0.138452, + "nauc_precision_at_10_max": 0.011162, + "nauc_precision_at_10_std": 0.234666, + "nauc_precision_at_10_diff1": -0.197579, + "nauc_precision_at_20_max": -0.053973, + "nauc_precision_at_20_std": 0.14616, + "nauc_precision_at_20_diff1": -0.20915, + "nauc_precision_at_100_max": -0.091276, + "nauc_precision_at_100_std": 0.081217, + "nauc_precision_at_100_diff1": -0.223393, + "nauc_precision_at_1000_max": -0.091276, + "nauc_precision_at_1000_std": 0.081217, + "nauc_precision_at_1000_diff1": -0.223393, + "nauc_mrr_at_1_max": 0.437481, + "nauc_mrr_at_1_std": 0.379887, + "nauc_mrr_at_1_diff1": 0.557019, + "nauc_mrr_at_3_max": 0.48406, + "nauc_mrr_at_3_std": 0.419929, + "nauc_mrr_at_3_diff1": 0.549313, + "nauc_mrr_at_5_max": 0.47874, + "nauc_mrr_at_5_std": 0.42767, + "nauc_mrr_at_5_diff1": 0.556879, + "nauc_mrr_at_10_max": 0.480889, + "nauc_mrr_at_10_std": 0.423355, + "nauc_mrr_at_10_diff1": 0.557009, + "nauc_mrr_at_20_max": 0.477618, + "nauc_mrr_at_20_std": 0.419616, + "nauc_mrr_at_20_diff1": 0.556006, + "nauc_mrr_at_100_max": 0.477618, + "nauc_mrr_at_100_std": 0.419616, + "nauc_mrr_at_100_diff1": 0.556006, + "nauc_mrr_at_1000_max": 0.477618, + "nauc_mrr_at_1000_std": 0.419616, + "nauc_mrr_at_1000_diff1": 0.556006, + "hit_rate_at_1": 0.8046, + "hit_rate_at_3": 0.91954, + "hit_rate_at_5": 0.9569, + "hit_rate_at_10": 0.97989, + "hit_rate_at_20": 0.99425, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.76819, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.77299, + "ndcg_at_3": 0.73562, + "ndcg_at_5": 0.74045, + "ndcg_at_10": 0.75965, + "ndcg_at_20": 0.78919, + "ndcg_at_100": 0.80892, + "ndcg_at_1000": 0.80892, + "map_at_1": 0.34633, + "map_at_3": 0.53497, + "map_at_5": 0.59547, + "map_at_10": 0.64717, + "map_at_20": 0.67733, + "map_at_100": 0.69033, + "map_at_1000": 0.69033, + "recall_at_1": 0.34633, + "recall_at_3": 0.5714, + "recall_at_5": 0.6717, + "recall_at_10": 0.77603, + "recall_at_20": 0.87003, + "recall_at_100": 0.92187, + "recall_at_1000": 0.92187, + "accuracy": 0.34633, + "precision_at_1": 0.79023, + "precision_at_3": 0.54023, + "precision_at_5": 0.42069, + "precision_at_10": 0.27557, + "precision_at_20": 0.16968, + "precision_at_100": 0.03882, + "precision_at_1000": 0.00388, + "mrr_at_1": 0.79023, + "mrr_at_3": 0.850575, + "mrr_at_5": 0.859626, + "mrr_at_10": 0.862335, + "mrr_at_20": 0.862912, + "mrr_at_100": 0.862912, + "mrr_at_1000": 0.862912, + "nauc_ndcg_at_1_max": 0.370314, + "nauc_ndcg_at_1_std": 0.390856, + "nauc_ndcg_at_1_diff1": 0.598376, + "nauc_ndcg_at_3_max": 0.410094, + "nauc_ndcg_at_3_std": 0.329073, + "nauc_ndcg_at_3_diff1": 0.463961, + "nauc_ndcg_at_5_max": 0.424977, + "nauc_ndcg_at_5_std": 0.380272, + "nauc_ndcg_at_5_diff1": 0.48515, + "nauc_ndcg_at_10_max": 0.435996, + "nauc_ndcg_at_10_std": 0.402469, + "nauc_ndcg_at_10_diff1": 0.493408, + "nauc_ndcg_at_20_max": 0.456947, + "nauc_ndcg_at_20_std": 0.436414, + "nauc_ndcg_at_20_diff1": 0.518303, + "nauc_ndcg_at_100_max": 0.453766, + "nauc_ndcg_at_100_std": 0.428795, + "nauc_ndcg_at_100_diff1": 0.514263, + "nauc_ndcg_at_1000_max": 0.453766, + "nauc_ndcg_at_1000_std": 0.428795, + "nauc_ndcg_at_1000_diff1": 0.514263, + "nauc_map_at_1_max": 0.242882, + "nauc_map_at_1_std": 0.024677, + "nauc_map_at_1_diff1": 0.541997, + "nauc_map_at_3_max": 0.320879, + "nauc_map_at_3_std": 0.148167, + "nauc_map_at_3_diff1": 0.453029, + "nauc_map_at_5_max": 0.349313, + "nauc_map_at_5_std": 0.236935, + "nauc_map_at_5_diff1": 0.458139, + "nauc_map_at_10_max": 0.376017, + "nauc_map_at_10_std": 0.312269, + "nauc_map_at_10_diff1": 0.438055, + "nauc_map_at_20_max": 0.395146, + "nauc_map_at_20_std": 0.342841, + "nauc_map_at_20_diff1": 0.447911, + "nauc_map_at_100_max": 0.3949, + "nauc_map_at_100_std": 0.34391, + "nauc_map_at_100_diff1": 0.445326, + "nauc_map_at_1000_max": 0.3949, + "nauc_map_at_1000_std": 0.34391, + "nauc_map_at_1000_diff1": 0.445326, + "nauc_recall_at_1_max": 0.242882, + "nauc_recall_at_1_std": 0.024677, + "nauc_recall_at_1_diff1": 0.541997, + "nauc_recall_at_3_max": 0.320129, + "nauc_recall_at_3_std": 0.1361, + "nauc_recall_at_3_diff1": 0.413074, + "nauc_recall_at_5_max": 0.338228, + "nauc_recall_at_5_std": 0.264751, + "nauc_recall_at_5_diff1": 0.408623, + "nauc_recall_at_10_max": 0.406137, + "nauc_recall_at_10_std": 0.397025, + "nauc_recall_at_10_diff1": 0.385131, + "nauc_recall_at_20_max": 0.502548, + "nauc_recall_at_20_std": 0.513228, + "nauc_recall_at_20_diff1": 0.46861, + "nauc_recall_at_100_max": 0.554536, + "nauc_recall_at_100_std": 0.540452, + "nauc_recall_at_100_diff1": 0.468968, + "nauc_recall_at_1000_max": 0.554536, + "nauc_recall_at_1000_std": 0.540452, + "nauc_recall_at_1000_diff1": 0.468968, + "nauc_precision_at_1_max": 0.42428, + "nauc_precision_at_1_std": 0.395135, + "nauc_precision_at_1_diff1": 0.608137, + "nauc_precision_at_3_max": 0.18464, + "nauc_precision_at_3_std": 0.312244, + "nauc_precision_at_3_diff1": -0.041854, + "nauc_precision_at_5_max": 0.092124, + "nauc_precision_at_5_std": 0.333924, + "nauc_precision_at_5_diff1": -0.136396, + "nauc_precision_at_10_max": 0.017463, + "nauc_precision_at_10_std": 0.259891, + "nauc_precision_at_10_diff1": -0.224742, + "nauc_precision_at_20_max": -0.036301, + "nauc_precision_at_20_std": 0.170445, + "nauc_precision_at_20_diff1": -0.240749, + "nauc_precision_at_100_max": -0.086973, + "nauc_precision_at_100_std": 0.077836, + "nauc_precision_at_100_diff1": -0.25982, + "nauc_precision_at_1000_max": -0.086973, + "nauc_precision_at_1000_std": 0.077836, + "nauc_precision_at_1000_diff1": -0.25982, + "nauc_mrr_at_1_max": 0.42428, + "nauc_mrr_at_1_std": 0.395135, + "nauc_mrr_at_1_diff1": 0.608137, + "nauc_mrr_at_3_max": 0.466525, + "nauc_mrr_at_3_std": 0.415948, + "nauc_mrr_at_3_diff1": 0.586281, + "nauc_mrr_at_5_max": 0.462339, + "nauc_mrr_at_5_std": 0.430034, + "nauc_mrr_at_5_diff1": 0.594119, + "nauc_mrr_at_10_max": 0.465871, + "nauc_mrr_at_10_std": 0.428016, + "nauc_mrr_at_10_diff1": 0.596636, + "nauc_mrr_at_20_max": 0.463895, + "nauc_mrr_at_20_std": 0.425992, + "nauc_mrr_at_20_diff1": 0.596606, + "nauc_mrr_at_100_max": 0.463895, + "nauc_mrr_at_100_std": 0.425992, + "nauc_mrr_at_100_diff1": 0.596606, + "nauc_mrr_at_1000_max": 0.463895, + "nauc_mrr_at_1000_std": 0.425992, + "nauc_mrr_at_1000_diff1": 0.596606, + "hit_rate_at_1": 0.79023, + "hit_rate_at_3": 0.92241, + "hit_rate_at_5": 0.96264, + "hit_rate_at_10": 0.98276, + "hit_rate_at_20": 0.99138, + "hit_rate_at_100": 0.99138, + "hit_rate_at_1000": 0.99138, + "main_score": 0.75965, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.77443, + "ndcg_at_3": 0.7373, + "ndcg_at_5": 0.74192, + "ndcg_at_10": 0.76432, + "ndcg_at_20": 0.79229, + "ndcg_at_100": 0.81286, + "ndcg_at_1000": 0.81286, + "map_at_1": 0.34685, + "map_at_3": 0.53944, + "map_at_5": 0.60136, + "map_at_10": 0.65634, + "map_at_20": 0.68536, + "map_at_100": 0.69866, + "map_at_1000": 0.69866, + "recall_at_1": 0.34685, + "recall_at_3": 0.57342, + "recall_at_5": 0.67228, + "recall_at_10": 0.78544, + "recall_at_20": 0.87544, + "recall_at_100": 0.93008, + "recall_at_1000": 0.93008, + "accuracy": 0.34685, + "precision_at_1": 0.79598, + "precision_at_3": 0.54789, + "precision_at_5": 0.42529, + "precision_at_10": 0.28017, + "precision_at_20": 0.17098, + "precision_at_100": 0.03905, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.795977, + "mrr_at_3": 0.851054, + "mrr_at_5": 0.857519, + "mrr_at_10": 0.86224, + "mrr_at_20": 0.863042, + "mrr_at_100": 0.863239, + "mrr_at_1000": 0.863239, + "nauc_ndcg_at_1_max": 0.356675, + "nauc_ndcg_at_1_std": 0.42749, + "nauc_ndcg_at_1_diff1": 0.552602, + "nauc_ndcg_at_3_max": 0.40367, + "nauc_ndcg_at_3_std": 0.405365, + "nauc_ndcg_at_3_diff1": 0.451511, + "nauc_ndcg_at_5_max": 0.417799, + "nauc_ndcg_at_5_std": 0.42024, + "nauc_ndcg_at_5_diff1": 0.443903, + "nauc_ndcg_at_10_max": 0.412978, + "nauc_ndcg_at_10_std": 0.436187, + "nauc_ndcg_at_10_diff1": 0.450012, + "nauc_ndcg_at_20_max": 0.426723, + "nauc_ndcg_at_20_std": 0.457002, + "nauc_ndcg_at_20_diff1": 0.480969, + "nauc_ndcg_at_100_max": 0.427246, + "nauc_ndcg_at_100_std": 0.460019, + "nauc_ndcg_at_100_diff1": 0.478928, + "nauc_ndcg_at_1000_max": 0.427246, + "nauc_ndcg_at_1000_std": 0.460019, + "nauc_ndcg_at_1000_diff1": 0.478928, + "nauc_map_at_1_max": 0.240257, + "nauc_map_at_1_std": 0.025305, + "nauc_map_at_1_diff1": 0.523429, + "nauc_map_at_3_max": 0.313628, + "nauc_map_at_3_std": 0.169851, + "nauc_map_at_3_diff1": 0.444176, + "nauc_map_at_5_max": 0.341478, + "nauc_map_at_5_std": 0.241718, + "nauc_map_at_5_diff1": 0.430978, + "nauc_map_at_10_max": 0.357533, + "nauc_map_at_10_std": 0.330571, + "nauc_map_at_10_diff1": 0.407871, + "nauc_map_at_20_max": 0.371206, + "nauc_map_at_20_std": 0.358571, + "nauc_map_at_20_diff1": 0.420629, + "nauc_map_at_100_max": 0.372723, + "nauc_map_at_100_std": 0.365835, + "nauc_map_at_100_diff1": 0.418508, + "nauc_map_at_1000_max": 0.372723, + "nauc_map_at_1000_std": 0.365835, + "nauc_map_at_1000_diff1": 0.418508, + "nauc_recall_at_1_max": 0.240257, + "nauc_recall_at_1_std": 0.025305, + "nauc_recall_at_1_diff1": 0.523429, + "nauc_recall_at_3_max": 0.311539, + "nauc_recall_at_3_std": 0.162218, + "nauc_recall_at_3_diff1": 0.410405, + "nauc_recall_at_5_max": 0.33181, + "nauc_recall_at_5_std": 0.263146, + "nauc_recall_at_5_diff1": 0.368676, + "nauc_recall_at_10_max": 0.363939, + "nauc_recall_at_10_std": 0.404358, + "nauc_recall_at_10_diff1": 0.336137, + "nauc_recall_at_20_max": 0.435663, + "nauc_recall_at_20_std": 0.498924, + "nauc_recall_at_20_diff1": 0.430996, + "nauc_recall_at_100_max": 0.518619, + "nauc_recall_at_100_std": 0.5868, + "nauc_recall_at_100_diff1": 0.440714, + "nauc_recall_at_1000_max": 0.518619, + "nauc_recall_at_1000_std": 0.5868, + "nauc_recall_at_1000_diff1": 0.440714, + "nauc_precision_at_1_max": 0.408927, + "nauc_precision_at_1_std": 0.443009, + "nauc_precision_at_1_diff1": 0.548006, + "nauc_precision_at_3_max": 0.164088, + "nauc_precision_at_3_std": 0.373786, + "nauc_precision_at_3_diff1": -0.038452, + "nauc_precision_at_5_max": 0.088729, + "nauc_precision_at_5_std": 0.345921, + "nauc_precision_at_5_diff1": -0.161089, + "nauc_precision_at_10_max": -6.8e-05, + "nauc_precision_at_10_std": 0.284459, + "nauc_precision_at_10_diff1": -0.247908, + "nauc_precision_at_20_max": -0.053509, + "nauc_precision_at_20_std": 0.184904, + "nauc_precision_at_20_diff1": -0.240915, + "nauc_precision_at_100_max": -0.092996, + "nauc_precision_at_100_std": 0.106715, + "nauc_precision_at_100_diff1": -0.258979, + "nauc_precision_at_1000_max": -0.092996, + "nauc_precision_at_1000_std": 0.106715, + "nauc_precision_at_1000_diff1": -0.258979, + "nauc_mrr_at_1_max": 0.408927, + "nauc_mrr_at_1_std": 0.443009, + "nauc_mrr_at_1_diff1": 0.548006, + "nauc_mrr_at_3_max": 0.45535, + "nauc_mrr_at_3_std": 0.46778, + "nauc_mrr_at_3_diff1": 0.534915, + "nauc_mrr_at_5_max": 0.451062, + "nauc_mrr_at_5_std": 0.478951, + "nauc_mrr_at_5_diff1": 0.53178, + "nauc_mrr_at_10_max": 0.451953, + "nauc_mrr_at_10_std": 0.476137, + "nauc_mrr_at_10_diff1": 0.54049, + "nauc_mrr_at_20_max": 0.448994, + "nauc_mrr_at_20_std": 0.473461, + "nauc_mrr_at_20_diff1": 0.540398, + "nauc_mrr_at_100_max": 0.448155, + "nauc_mrr_at_100_std": 0.472659, + "nauc_mrr_at_100_diff1": 0.539945, + "nauc_mrr_at_1000_max": 0.448155, + "nauc_mrr_at_1000_std": 0.472659, + "nauc_mrr_at_1000_diff1": 0.539945, + "hit_rate_at_1": 0.79598, + "hit_rate_at_3": 0.91954, + "hit_rate_at_5": 0.94828, + "hit_rate_at_10": 0.97989, + "hit_rate_at_20": 0.99138, + "hit_rate_at_100": 0.99713, + "hit_rate_at_1000": 0.99713, + "main_score": 0.76432, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.77299, + "ndcg_at_3": 0.73599, + "ndcg_at_5": 0.73683, + "ndcg_at_10": 0.76029, + "ndcg_at_20": 0.78912, + "ndcg_at_100": 0.81055, + "ndcg_at_1000": 0.81055, + "map_at_1": 0.34068, + "map_at_3": 0.53477, + "map_at_5": 0.59437, + "map_at_10": 0.64938, + "map_at_20": 0.67816, + "map_at_100": 0.69181, + "map_at_1000": 0.69181, + "recall_at_1": 0.34068, + "recall_at_3": 0.57549, + "recall_at_5": 0.66869, + "recall_at_10": 0.7836, + "recall_at_20": 0.87434, + "recall_at_100": 0.93189, + "recall_at_1000": 0.93189, + "accuracy": 0.34068, + "precision_at_1": 0.7931, + "precision_at_3": 0.54693, + "precision_at_5": 0.42011, + "precision_at_10": 0.27874, + "precision_at_20": 0.17026, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.793103, + "mrr_at_3": 0.851533, + "mrr_at_5": 0.858716, + "mrr_at_10": 0.862761, + "mrr_at_20": 0.863357, + "mrr_at_100": 0.863435, + "mrr_at_1000": 0.863435, + "nauc_ndcg_at_1_max": 0.40445, + "nauc_ndcg_at_1_std": 0.390995, + "nauc_ndcg_at_1_diff1": 0.595987, + "nauc_ndcg_at_3_max": 0.426078, + "nauc_ndcg_at_3_std": 0.379806, + "nauc_ndcg_at_3_diff1": 0.461628, + "nauc_ndcg_at_5_max": 0.434171, + "nauc_ndcg_at_5_std": 0.424886, + "nauc_ndcg_at_5_diff1": 0.473034, + "nauc_ndcg_at_10_max": 0.433065, + "nauc_ndcg_at_10_std": 0.428606, + "nauc_ndcg_at_10_diff1": 0.472519, + "nauc_ndcg_at_20_max": 0.459357, + "nauc_ndcg_at_20_std": 0.457373, + "nauc_ndcg_at_20_diff1": 0.500971, + "nauc_ndcg_at_100_max": 0.455464, + "nauc_ndcg_at_100_std": 0.452025, + "nauc_ndcg_at_100_diff1": 0.499067, + "nauc_ndcg_at_1000_max": 0.455464, + "nauc_ndcg_at_1000_std": 0.452025, + "nauc_ndcg_at_1000_diff1": 0.499067, + "nauc_map_at_1_max": 0.234356, + "nauc_map_at_1_std": 0.039645, + "nauc_map_at_1_diff1": 0.549101, + "nauc_map_at_3_max": 0.332558, + "nauc_map_at_3_std": 0.168423, + "nauc_map_at_3_diff1": 0.456282, + "nauc_map_at_5_max": 0.35349, + "nauc_map_at_5_std": 0.254753, + "nauc_map_at_5_diff1": 0.450112, + "nauc_map_at_10_max": 0.373685, + "nauc_map_at_10_std": 0.330655, + "nauc_map_at_10_diff1": 0.427168, + "nauc_map_at_20_max": 0.395852, + "nauc_map_at_20_std": 0.364557, + "nauc_map_at_20_diff1": 0.439452, + "nauc_map_at_100_max": 0.395971, + "nauc_map_at_100_std": 0.368207, + "nauc_map_at_100_diff1": 0.438839, + "nauc_map_at_1000_max": 0.395971, + "nauc_map_at_1000_std": 0.368207, + "nauc_map_at_1000_diff1": 0.438839, + "nauc_recall_at_1_max": 0.234356, + "nauc_recall_at_1_std": 0.039645, + "nauc_recall_at_1_diff1": 0.549101, + "nauc_recall_at_3_max": 0.323083, + "nauc_recall_at_3_std": 0.154312, + "nauc_recall_at_3_diff1": 0.407669, + "nauc_recall_at_5_max": 0.331766, + "nauc_recall_at_5_std": 0.284565, + "nauc_recall_at_5_diff1": 0.384512, + "nauc_recall_at_10_max": 0.360185, + "nauc_recall_at_10_std": 0.392755, + "nauc_recall_at_10_diff1": 0.337812, + "nauc_recall_at_20_max": 0.474713, + "nauc_recall_at_20_std": 0.526764, + "nauc_recall_at_20_diff1": 0.437113, + "nauc_recall_at_100_max": 0.551129, + "nauc_recall_at_100_std": 0.596215, + "nauc_recall_at_100_diff1": 0.456158, + "nauc_recall_at_1000_max": 0.551129, + "nauc_recall_at_1000_std": 0.596215, + "nauc_recall_at_1000_diff1": 0.456158, + "nauc_precision_at_1_max": 0.438734, + "nauc_precision_at_1_std": 0.395494, + "nauc_precision_at_1_diff1": 0.597551, + "nauc_precision_at_3_max": 0.210337, + "nauc_precision_at_3_std": 0.349385, + "nauc_precision_at_3_diff1": -0.031955, + "nauc_precision_at_5_max": 0.118498, + "nauc_precision_at_5_std": 0.360362, + "nauc_precision_at_5_diff1": -0.132156, + "nauc_precision_at_10_max": 0.021879, + "nauc_precision_at_10_std": 0.276317, + "nauc_precision_at_10_diff1": -0.225066, + "nauc_precision_at_20_max": -0.024866, + "nauc_precision_at_20_std": 0.188129, + "nauc_precision_at_20_diff1": -0.230983, + "nauc_precision_at_100_max": -0.082842, + "nauc_precision_at_100_std": 0.093503, + "nauc_precision_at_100_diff1": -0.253132, + "nauc_precision_at_1000_max": -0.082842, + "nauc_precision_at_1000_std": 0.093503, + "nauc_precision_at_1000_diff1": -0.253132, + "nauc_mrr_at_1_max": 0.438734, + "nauc_mrr_at_1_std": 0.395494, + "nauc_mrr_at_1_diff1": 0.597551, + "nauc_mrr_at_3_max": 0.495794, + "nauc_mrr_at_3_std": 0.441832, + "nauc_mrr_at_3_diff1": 0.587135, + "nauc_mrr_at_5_max": 0.49311, + "nauc_mrr_at_5_std": 0.455357, + "nauc_mrr_at_5_diff1": 0.58312, + "nauc_mrr_at_10_max": 0.489599, + "nauc_mrr_at_10_std": 0.445621, + "nauc_mrr_at_10_diff1": 0.589825, + "nauc_mrr_at_20_max": 0.487636, + "nauc_mrr_at_20_std": 0.443655, + "nauc_mrr_at_20_diff1": 0.589932, + "nauc_mrr_at_100_max": 0.487328, + "nauc_mrr_at_100_std": 0.44332, + "nauc_mrr_at_100_diff1": 0.589685, + "nauc_mrr_at_1000_max": 0.487328, + "nauc_mrr_at_1000_std": 0.44332, + "nauc_mrr_at_1000_diff1": 0.589685, + "hit_rate_at_1": 0.7931, + "hit_rate_at_3": 0.92241, + "hit_rate_at_5": 0.95402, + "hit_rate_at_10": 0.98276, + "hit_rate_at_20": 0.99138, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.76029, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.76149, + "ndcg_at_3": 0.73465, + "ndcg_at_5": 0.7372, + "ndcg_at_10": 0.75976, + "ndcg_at_20": 0.7895, + "ndcg_at_100": 0.80916, + "ndcg_at_1000": 0.80916, + "map_at_1": 0.33627, + "map_at_3": 0.53496, + "map_at_5": 0.59384, + "map_at_10": 0.64873, + "map_at_20": 0.67865, + "map_at_100": 0.69141, + "map_at_1000": 0.69141, + "recall_at_1": 0.33627, + "recall_at_3": 0.57672, + "recall_at_5": 0.66953, + "recall_at_10": 0.7832, + "recall_at_20": 0.87738, + "recall_at_100": 0.93007, + "recall_at_1000": 0.93007, + "accuracy": 0.33627, + "precision_at_1": 0.78448, + "precision_at_3": 0.54789, + "precision_at_5": 0.42356, + "precision_at_10": 0.28046, + "precision_at_20": 0.17198, + "precision_at_100": 0.03914, + "precision_at_1000": 0.00391, + "mrr_at_1": 0.784483, + "mrr_at_3": 0.84818, + "mrr_at_5": 0.85522, + "mrr_at_10": 0.858179, + "mrr_at_20": 0.858989, + "mrr_at_100": 0.859126, + "mrr_at_1000": 0.859126, + "nauc_ndcg_at_1_max": 0.400354, + "nauc_ndcg_at_1_std": 0.365699, + "nauc_ndcg_at_1_diff1": 0.609221, + "nauc_ndcg_at_3_max": 0.431636, + "nauc_ndcg_at_3_std": 0.36412, + "nauc_ndcg_at_3_diff1": 0.489414, + "nauc_ndcg_at_5_max": 0.452331, + "nauc_ndcg_at_5_std": 0.404625, + "nauc_ndcg_at_5_diff1": 0.488716, + "nauc_ndcg_at_10_max": 0.436382, + "nauc_ndcg_at_10_std": 0.411519, + "nauc_ndcg_at_10_diff1": 0.4886, + "nauc_ndcg_at_20_max": 0.463667, + "nauc_ndcg_at_20_std": 0.438306, + "nauc_ndcg_at_20_diff1": 0.513196, + "nauc_ndcg_at_100_max": 0.46622, + "nauc_ndcg_at_100_std": 0.437886, + "nauc_ndcg_at_100_diff1": 0.510602, + "nauc_ndcg_at_1000_max": 0.46622, + "nauc_ndcg_at_1000_std": 0.437886, + "nauc_ndcg_at_1000_diff1": 0.510602, + "nauc_map_at_1_max": 0.233398, + "nauc_map_at_1_std": 0.00718, + "nauc_map_at_1_diff1": 0.574289, + "nauc_map_at_3_max": 0.333598, + "nauc_map_at_3_std": 0.166674, + "nauc_map_at_3_diff1": 0.479666, + "nauc_map_at_5_max": 0.370418, + "nauc_map_at_5_std": 0.249036, + "nauc_map_at_5_diff1": 0.471837, + "nauc_map_at_10_max": 0.38169, + "nauc_map_at_10_std": 0.322767, + "nauc_map_at_10_diff1": 0.445253, + "nauc_map_at_20_max": 0.405946, + "nauc_map_at_20_std": 0.352992, + "nauc_map_at_20_diff1": 0.455507, + "nauc_map_at_100_max": 0.408923, + "nauc_map_at_100_std": 0.35717, + "nauc_map_at_100_diff1": 0.453789, + "nauc_map_at_1000_max": 0.408923, + "nauc_map_at_1000_std": 0.35717, + "nauc_map_at_1000_diff1": 0.453789, + "nauc_recall_at_1_max": 0.233398, + "nauc_recall_at_1_std": 0.00718, + "nauc_recall_at_1_diff1": 0.574289, + "nauc_recall_at_3_max": 0.323591, + "nauc_recall_at_3_std": 0.15532, + "nauc_recall_at_3_diff1": 0.432021, + "nauc_recall_at_5_max": 0.360849, + "nauc_recall_at_5_std": 0.281561, + "nauc_recall_at_5_diff1": 0.399717, + "nauc_recall_at_10_max": 0.36467, + "nauc_recall_at_10_std": 0.393659, + "nauc_recall_at_10_diff1": 0.343615, + "nauc_recall_at_20_max": 0.476071, + "nauc_recall_at_20_std": 0.511519, + "nauc_recall_at_20_diff1": 0.427765, + "nauc_recall_at_100_max": 0.566632, + "nauc_recall_at_100_std": 0.596015, + "nauc_recall_at_100_diff1": 0.434994, + "nauc_recall_at_1000_max": 0.566632, + "nauc_recall_at_1000_std": 0.596015, + "nauc_recall_at_1000_diff1": 0.434994, + "nauc_precision_at_1_max": 0.446751, + "nauc_precision_at_1_std": 0.382249, + "nauc_precision_at_1_diff1": 0.617939, + "nauc_precision_at_3_max": 0.228679, + "nauc_precision_at_3_std": 0.368417, + "nauc_precision_at_3_diff1": -0.024087, + "nauc_precision_at_5_max": 0.148411, + "nauc_precision_at_5_std": 0.358315, + "nauc_precision_at_5_diff1": -0.135973, + "nauc_precision_at_10_max": 0.027614, + "nauc_precision_at_10_std": 0.275137, + "nauc_precision_at_10_diff1": -0.232609, + "nauc_precision_at_20_max": -0.021058, + "nauc_precision_at_20_std": 0.178137, + "nauc_precision_at_20_diff1": -0.246681, + "nauc_precision_at_100_max": -0.067061, + "nauc_precision_at_100_std": 0.095911, + "nauc_precision_at_100_diff1": -0.265859, + "nauc_precision_at_1000_max": -0.067061, + "nauc_precision_at_1000_std": 0.095911, + "nauc_precision_at_1000_diff1": -0.265859, + "nauc_mrr_at_1_max": 0.446751, + "nauc_mrr_at_1_std": 0.382249, + "nauc_mrr_at_1_diff1": 0.617939, + "nauc_mrr_at_3_max": 0.486545, + "nauc_mrr_at_3_std": 0.412371, + "nauc_mrr_at_3_diff1": 0.590907, + "nauc_mrr_at_5_max": 0.486812, + "nauc_mrr_at_5_std": 0.431659, + "nauc_mrr_at_5_diff1": 0.59838, + "nauc_mrr_at_10_max": 0.489708, + "nauc_mrr_at_10_std": 0.427738, + "nauc_mrr_at_10_diff1": 0.602127, + "nauc_mrr_at_20_max": 0.486988, + "nauc_mrr_at_20_std": 0.424836, + "nauc_mrr_at_20_diff1": 0.603352, + "nauc_mrr_at_100_max": 0.486456, + "nauc_mrr_at_100_std": 0.424239, + "nauc_mrr_at_100_diff1": 0.603076, + "nauc_mrr_at_1000_max": 0.486456, + "nauc_mrr_at_1000_std": 0.424239, + "nauc_mrr_at_1000_diff1": 0.603076, + "hit_rate_at_1": 0.78448, + "hit_rate_at_3": 0.92816, + "hit_rate_at_5": 0.95977, + "hit_rate_at_10": 0.97989, + "hit_rate_at_20": 0.99138, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.75976, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.14" + }, + { + "ndcg_at_1": 0.76149, + "ndcg_at_3": 0.7347, + "ndcg_at_5": 0.73951, + "ndcg_at_10": 0.76021, + "ndcg_at_20": 0.79061, + "ndcg_at_100": 0.81132, + "ndcg_at_1000": 0.81132, + "map_at_1": 0.33907, + "map_at_3": 0.53663, + "map_at_5": 0.59758, + "map_at_10": 0.65137, + "map_at_20": 0.68112, + "map_at_100": 0.69492, + "map_at_1000": 0.69492, + "recall_at_1": 0.33907, + "recall_at_3": 0.57632, + "recall_at_5": 0.67382, + "recall_at_10": 0.78354, + "recall_at_20": 0.87719, + "recall_at_100": 0.93268, + "recall_at_1000": 0.93268, + "accuracy": 0.33907, + "precision_at_1": 0.78161, + "precision_at_3": 0.54502, + "precision_at_5": 0.42471, + "precision_at_10": 0.27874, + "precision_at_20": 0.17141, + "precision_at_100": 0.03937, + "precision_at_1000": 0.00394, + "mrr_at_1": 0.781609, + "mrr_at_3": 0.84387, + "mrr_at_5": 0.851628, + "mrr_at_10": 0.855553, + "mrr_at_20": 0.856157, + "mrr_at_100": 0.856272, + "mrr_at_1000": 0.856272, + "nauc_ndcg_at_1_max": 0.367548, + "nauc_ndcg_at_1_std": 0.335696, + "nauc_ndcg_at_1_diff1": 0.595754, + "nauc_ndcg_at_3_max": 0.415169, + "nauc_ndcg_at_3_std": 0.358729, + "nauc_ndcg_at_3_diff1": 0.441207, + "nauc_ndcg_at_5_max": 0.424396, + "nauc_ndcg_at_5_std": 0.395828, + "nauc_ndcg_at_5_diff1": 0.46153, + "nauc_ndcg_at_10_max": 0.417812, + "nauc_ndcg_at_10_std": 0.409805, + "nauc_ndcg_at_10_diff1": 0.466616, + "nauc_ndcg_at_20_max": 0.433816, + "nauc_ndcg_at_20_std": 0.428554, + "nauc_ndcg_at_20_diff1": 0.490562, + "nauc_ndcg_at_100_max": 0.436893, + "nauc_ndcg_at_100_std": 0.433783, + "nauc_ndcg_at_100_diff1": 0.484267, + "nauc_ndcg_at_1000_max": 0.436893, + "nauc_ndcg_at_1000_std": 0.433783, + "nauc_ndcg_at_1000_diff1": 0.484267, + "nauc_map_at_1_max": 0.221824, + "nauc_map_at_1_std": 0.011217, + "nauc_map_at_1_diff1": 0.549658, + "nauc_map_at_3_max": 0.304666, + "nauc_map_at_3_std": 0.157466, + "nauc_map_at_3_diff1": 0.42619, + "nauc_map_at_5_max": 0.338799, + "nauc_map_at_5_std": 0.244567, + "nauc_map_at_5_diff1": 0.434076, + "nauc_map_at_10_max": 0.358649, + "nauc_map_at_10_std": 0.320219, + "nauc_map_at_10_diff1": 0.413782, + "nauc_map_at_20_max": 0.37624, + "nauc_map_at_20_std": 0.344217, + "nauc_map_at_20_diff1": 0.426198, + "nauc_map_at_100_max": 0.37887, + "nauc_map_at_100_std": 0.35104, + "nauc_map_at_100_diff1": 0.423091, + "nauc_map_at_1000_max": 0.37887, + "nauc_map_at_1000_std": 0.35104, + "nauc_map_at_1000_diff1": 0.423091, + "nauc_recall_at_1_max": 0.221824, + "nauc_recall_at_1_std": 0.011217, + "nauc_recall_at_1_diff1": 0.549658, + "nauc_recall_at_3_max": 0.292503, + "nauc_recall_at_3_std": 0.14818, + "nauc_recall_at_3_diff1": 0.373078, + "nauc_recall_at_5_max": 0.315813, + "nauc_recall_at_5_std": 0.280408, + "nauc_recall_at_5_diff1": 0.370251, + "nauc_recall_at_10_max": 0.35098, + "nauc_recall_at_10_std": 0.407101, + "nauc_recall_at_10_diff1": 0.334341, + "nauc_recall_at_20_max": 0.43126, + "nauc_recall_at_20_std": 0.501211, + "nauc_recall_at_20_diff1": 0.42177, + "nauc_recall_at_100_max": 0.520634, + "nauc_recall_at_100_std": 0.628984, + "nauc_recall_at_100_diff1": 0.402704, + "nauc_recall_at_1000_max": 0.520634, + "nauc_recall_at_1000_std": 0.628984, + "nauc_recall_at_1000_diff1": 0.402704, + "nauc_precision_at_1_max": 0.424871, + "nauc_precision_at_1_std": 0.33945, + "nauc_precision_at_1_diff1": 0.605831, + "nauc_precision_at_3_max": 0.209748, + "nauc_precision_at_3_std": 0.351725, + "nauc_precision_at_3_diff1": -0.066277, + "nauc_precision_at_5_max": 0.134582, + "nauc_precision_at_5_std": 0.349825, + "nauc_precision_at_5_diff1": -0.134913, + "nauc_precision_at_10_max": 0.043263, + "nauc_precision_at_10_std": 0.274734, + "nauc_precision_at_10_diff1": -0.206573, + "nauc_precision_at_20_max": -0.016603, + "nauc_precision_at_20_std": 0.171853, + "nauc_precision_at_20_diff1": -0.222004, + "nauc_precision_at_100_max": -0.063405, + "nauc_precision_at_100_std": 0.088413, + "nauc_precision_at_100_diff1": -0.242542, + "nauc_precision_at_1000_max": -0.063405, + "nauc_precision_at_1000_std": 0.088413, + "nauc_precision_at_1000_diff1": -0.242542, + "nauc_mrr_at_1_max": 0.424871, + "nauc_mrr_at_1_std": 0.33945, + "nauc_mrr_at_1_diff1": 0.605831, + "nauc_mrr_at_3_max": 0.479275, + "nauc_mrr_at_3_std": 0.394368, + "nauc_mrr_at_3_diff1": 0.581389, + "nauc_mrr_at_5_max": 0.470685, + "nauc_mrr_at_5_std": 0.404454, + "nauc_mrr_at_5_diff1": 0.589147, + "nauc_mrr_at_10_max": 0.473067, + "nauc_mrr_at_10_std": 0.398778, + "nauc_mrr_at_10_diff1": 0.593878, + "nauc_mrr_at_20_max": 0.471573, + "nauc_mrr_at_20_std": 0.39641, + "nauc_mrr_at_20_diff1": 0.594819, + "nauc_mrr_at_100_max": 0.471121, + "nauc_mrr_at_100_std": 0.395894, + "nauc_mrr_at_100_diff1": 0.594472, + "nauc_mrr_at_1000_max": 0.471121, + "nauc_mrr_at_1000_std": 0.395894, + "nauc_mrr_at_1000_diff1": 0.594472, + "hit_rate_at_1": 0.78161, + "hit_rate_at_3": 0.91954, + "hit_rate_at_5": 0.95402, + "hit_rate_at_10": 0.98276, + "hit_rate_at_20": 0.99138, + "hit_rate_at_100": 0.99425, + "hit_rate_at_1000": 0.99425, + "main_score": 0.76021, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.14" + } + ] + }, + "evaluation_time": 19865.009499204927, + "kg_co2_emissions": null, + "date": 1792442604.334624, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 9.563984349370003e-06, + "end": 11.090794609976001, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 11.09086055890657, + "end": 11.090864575933665, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries and documents", + "start": 11.097149818902835, + "end": 3277.948026307975, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 3277.948111656937, + "end": 3277.997848221916, + "split": "test", + "subset": "english" + } + ] +} diff --git a/results/nvidia__Nemotron-3.5-Rerank-8B-BF16/708e844c3da0bbaa923e9dc310b07da4ffb6a6b5/run_settings.jsonl b/results/nvidia__Nemotron-3.5-Rerank-8B-BF16/708e844c3da0bbaa923e9dc310b07da4ffb6a6b5/run_settings.jsonl index 1617b7a29a..4231ab1e1a 100644 --- a/results/nvidia__Nemotron-3.5-Rerank-8B-BF16/708e844c3da0bbaa923e9dc310b07da4ffb6a6b5/run_settings.jsonl +++ b/results/nvidia__Nemotron-3.5-Rerank-8B-BF16/708e844c3da0bbaa923e9dc310b07da4ffb6a6b5/run_settings.jsonl @@ -38,3 +38,15 @@ {"task": "Vidore3FinanceEnBM25sReranking.v2", "splits": ["test"], "version": {"mteb": "2.22.5", "torch": "2.14.0a0+b2c75dd062.nv26.9.68203377", "sentence-transformers": "6.1.0", "flash-attn": "2.8.3.post1+git91ffc2ac7c.63064690", "transformers": "5.17.0"}, "encode_kwargs": {"batch_size": 2}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.8.0+cu128", "sentence-transformers": "6.1.0", "flash-attn": "2.8.3.post2", "transformers": "5.16.1"}, "encode_kwargs": {}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.1", "torch": "2.8.0+cu128", "sentence-transformers": "6.1.0", "flash-attn": "2.8.3.post2", "transformers": "5.16.1"}, "encode_kwargs": {}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["english"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["french"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["german"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["italian"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["portuguese"]} +{"task": "Vidore3TelecomNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["spanish"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["english"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["french"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["german"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["italian"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["portuguese"]} +{"task": "Vidore3NuclearNemotron35Embed8BReranking.v2", "splits": ["test"], "version": {"mteb": "2.26.14", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.16.1"}, "encode_kwargs": {"batch_size": 8}, "subsets": ["spanish"]}