diff --git a/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3NuclearRetrieval.json b/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3NuclearRetrieval.json new file mode 100644 index 0000000000..55058486e1 --- /dev/null +++ b/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3NuclearRetrieval.json @@ -0,0 +1,1090 @@ +{ + "dataset_revision": "a463fc67fefc01152153101e88a32d5f9515e3e3", + "task_name": "Vidore3NuclearRetrieval", + "mteb_version": "2.26.10", + "scores": { + "test": [ + { + "ndcg_at_1": 0.18373, + "ndcg_at_3": 0.18521, + "ndcg_at_5": 0.19003, + "ndcg_at_10": 0.21219, + "ndcg_at_20": 0.24477, + "ndcg_at_100": 0.30801, + "ndcg_at_1000": 0.37683, + "map_at_1": 0.06814, + "map_at_3": 0.10909, + "map_at_5": 0.12357, + "map_at_10": 0.13952, + "map_at_20": 0.15371, + "map_at_100": 0.16824, + "map_at_1000": 0.17346, + "recall_at_1": 0.06814, + "recall_at_3": 0.14427, + "recall_at_5": 0.18108, + "recall_at_10": 0.25294, + "recall_at_20": 0.34359, + "recall_at_100": 0.57058, + "recall_at_1000": 0.92903, + "accuracy": 0.06814, + "precision_at_1": 0.19578, + "precision_at_3": 0.14859, + "precision_at_5": 0.11867, + "precision_at_10": 0.08946, + "precision_at_20": 0.06491, + "precision_at_100": 0.0238, + "precision_at_1000": 0.0043, + "mrr_at_1": 0.195783, + "mrr_at_3": 0.268072, + "mrr_at_5": 0.278012, + "mrr_at_10": 0.293951, + "mrr_at_20": 0.300797, + "mrr_at_100": 0.306466, + "mrr_at_1000": 0.307163, + "nauc_ndcg_at_1_max": 0.019913, + "nauc_ndcg_at_1_std": -0.154752, + "nauc_ndcg_at_1_diff1": 0.232328, + "nauc_ndcg_at_3_max": -0.013775, + "nauc_ndcg_at_3_std": -0.106531, + "nauc_ndcg_at_3_diff1": 0.169721, + "nauc_ndcg_at_5_max": -0.011675, + "nauc_ndcg_at_5_std": -0.09315, + "nauc_ndcg_at_5_diff1": 0.14041, + "nauc_ndcg_at_10_max": -0.025764, + "nauc_ndcg_at_10_std": -0.077636, + "nauc_ndcg_at_10_diff1": 0.135034, + "nauc_ndcg_at_20_max": -0.004377, + "nauc_ndcg_at_20_std": -0.047042, + "nauc_ndcg_at_20_diff1": 0.152527, + "nauc_ndcg_at_100_max": 0.00645, + "nauc_ndcg_at_100_std": -0.016771, + "nauc_ndcg_at_100_diff1": 0.161727, + "nauc_ndcg_at_1000_max": 0.003502, + "nauc_ndcg_at_1000_std": -0.035763, + "nauc_ndcg_at_1000_diff1": 0.164053, + "nauc_map_at_1_max": 0.038654, + "nauc_map_at_1_std": -0.149979, + "nauc_map_at_1_diff1": 0.176299, + "nauc_map_at_3_max": 0.001932, + "nauc_map_at_3_std": -0.11434, + "nauc_map_at_3_diff1": 0.149449, + "nauc_map_at_5_max": -0.005112, + "nauc_map_at_5_std": -0.098749, + "nauc_map_at_5_diff1": 0.12982, + "nauc_map_at_10_max": -0.010238, + "nauc_map_at_10_std": -0.087433, + "nauc_map_at_10_diff1": 0.128309, + "nauc_map_at_20_max": 0.00048, + "nauc_map_at_20_std": -0.071281, + "nauc_map_at_20_diff1": 0.13131, + "nauc_map_at_100_max": 0.008679, + "nauc_map_at_100_std": -0.054712, + "nauc_map_at_100_diff1": 0.130603, + "nauc_map_at_1000_max": 0.008227, + "nauc_map_at_1000_std": -0.054326, + "nauc_map_at_1000_diff1": 0.132998, + "nauc_recall_at_1_max": 0.038654, + "nauc_recall_at_1_std": -0.149979, + "nauc_recall_at_1_diff1": 0.176299, + "nauc_recall_at_3_max": -0.029993, + "nauc_recall_at_3_std": -0.090408, + "nauc_recall_at_3_diff1": 0.114044, + "nauc_recall_at_5_max": -0.025456, + "nauc_recall_at_5_std": -0.057758, + "nauc_recall_at_5_diff1": 0.090298, + "nauc_recall_at_10_max": -0.050259, + "nauc_recall_at_10_std": -0.003428, + "nauc_recall_at_10_diff1": 0.061626, + "nauc_recall_at_20_max": -0.000847, + "nauc_recall_at_20_std": 0.065066, + "nauc_recall_at_20_diff1": 0.097168, + "nauc_recall_at_100_max": 0.01217, + "nauc_recall_at_100_std": 0.134574, + "nauc_recall_at_100_diff1": 0.127697, + "nauc_recall_at_1000_max": -0.289285, + "nauc_recall_at_1000_std": 0.11321, + "nauc_recall_at_1000_diff1": -0.032969, + "nauc_precision_at_1_max": 0.035651, + "nauc_precision_at_1_std": -0.124158, + "nauc_precision_at_1_diff1": 0.214632, + "nauc_precision_at_3_max": -0.032707, + "nauc_precision_at_3_std": -0.057006, + "nauc_precision_at_3_diff1": 0.163017, + "nauc_precision_at_5_max": -0.037715, + "nauc_precision_at_5_std": -0.045147, + "nauc_precision_at_5_diff1": 0.10543, + "nauc_precision_at_10_max": -0.016325, + "nauc_precision_at_10_std": -0.010924, + "nauc_precision_at_10_diff1": 0.109447, + "nauc_precision_at_20_max": 0.03575, + "nauc_precision_at_20_std": 0.033096, + "nauc_precision_at_20_diff1": 0.12135, + "nauc_precision_at_100_max": 0.062297, + "nauc_precision_at_100_std": 0.099867, + "nauc_precision_at_100_diff1": 0.103369, + "nauc_precision_at_1000_max": 0.08563, + "nauc_precision_at_1000_std": 0.09262, + "nauc_precision_at_1000_diff1": 0.067166, + "nauc_mrr_at_1_max": 0.035651, + "nauc_mrr_at_1_std": -0.124158, + "nauc_mrr_at_1_diff1": 0.214632, + "nauc_mrr_at_3_max": -0.013923, + "nauc_mrr_at_3_std": -0.098515, + "nauc_mrr_at_3_diff1": 0.164903, + "nauc_mrr_at_5_max": -0.013629, + "nauc_mrr_at_5_std": -0.096918, + "nauc_mrr_at_5_diff1": 0.163495, + "nauc_mrr_at_10_max": -0.015574, + "nauc_mrr_at_10_std": -0.092823, + "nauc_mrr_at_10_diff1": 0.162483, + "nauc_mrr_at_20_max": -0.009564, + "nauc_mrr_at_20_std": -0.089615, + "nauc_mrr_at_20_diff1": 0.166869, + "nauc_mrr_at_100_max": -0.008278, + "nauc_mrr_at_100_std": -0.088791, + "nauc_mrr_at_100_diff1": 0.166431, + "nauc_mrr_at_1000_max": -0.00853, + "nauc_mrr_at_1000_std": -0.089287, + "nauc_mrr_at_1000_diff1": 0.166167, + "hit_rate_at_1": 0.19578, + "hit_rate_at_3": 0.35542, + "hit_rate_at_5": 0.39759, + "hit_rate_at_10": 0.51506, + "hit_rate_at_20": 0.61145, + "hit_rate_at_100": 0.81928, + "hit_rate_at_1000": 0.97892, + "main_score": 0.21219, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.15964, + "ndcg_at_3": 0.16053, + "ndcg_at_5": 0.16993, + "ndcg_at_10": 0.19878, + "ndcg_at_20": 0.23073, + "ndcg_at_100": 0.2928, + "ndcg_at_1000": 0.36355, + "map_at_1": 0.06176, + "map_at_3": 0.09283, + "map_at_5": 0.10666, + "map_at_10": 0.12478, + "map_at_20": 0.13788, + "map_at_100": 0.15231, + "map_at_1000": 0.15756, + "recall_at_1": 0.06176, + "recall_at_3": 0.1239, + "recall_at_5": 0.16413, + "recall_at_10": 0.24441, + "recall_at_20": 0.33797, + "recall_at_100": 0.5618, + "recall_at_1000": 0.93047, + "accuracy": 0.06176, + "precision_at_1": 0.16867, + "precision_at_3": 0.12751, + "precision_at_5": 0.10783, + "precision_at_10": 0.08795, + "precision_at_20": 0.06325, + "precision_at_100": 0.02316, + "precision_at_1000": 0.0043, + "mrr_at_1": 0.168675, + "mrr_at_3": 0.236948, + "mrr_at_5": 0.253966, + "mrr_at_10": 0.269674, + "mrr_at_20": 0.276159, + "mrr_at_100": 0.280787, + "mrr_at_1000": 0.281646, + "nauc_ndcg_at_1_max": -0.019418, + "nauc_ndcg_at_1_std": -0.075721, + "nauc_ndcg_at_1_diff1": 0.133021, + "nauc_ndcg_at_3_max": 0.007455, + "nauc_ndcg_at_3_std": -0.014139, + "nauc_ndcg_at_3_diff1": 0.148903, + "nauc_ndcg_at_5_max": 0.040813, + "nauc_ndcg_at_5_std": 0.000189, + "nauc_ndcg_at_5_diff1": 0.14519, + "nauc_ndcg_at_10_max": 0.031628, + "nauc_ndcg_at_10_std": 0.026361, + "nauc_ndcg_at_10_diff1": 0.127643, + "nauc_ndcg_at_20_max": 0.046715, + "nauc_ndcg_at_20_std": 0.042843, + "nauc_ndcg_at_20_diff1": 0.121896, + "nauc_ndcg_at_100_max": 0.058883, + "nauc_ndcg_at_100_std": 0.066872, + "nauc_ndcg_at_100_diff1": 0.126563, + "nauc_ndcg_at_1000_max": 0.044842, + "nauc_ndcg_at_1000_std": 0.039709, + "nauc_ndcg_at_1000_diff1": 0.132391, + "nauc_map_at_1_max": 0.078812, + "nauc_map_at_1_std": -0.071949, + "nauc_map_at_1_diff1": 0.23682, + "nauc_map_at_3_max": 0.057921, + "nauc_map_at_3_std": -0.020759, + "nauc_map_at_3_diff1": 0.181452, + "nauc_map_at_5_max": 0.058544, + "nauc_map_at_5_std": -0.01608, + "nauc_map_at_5_diff1": 0.165396, + "nauc_map_at_10_max": 0.052081, + "nauc_map_at_10_std": 0.008352, + "nauc_map_at_10_diff1": 0.153174, + "nauc_map_at_20_max": 0.057874, + "nauc_map_at_20_std": 0.020045, + "nauc_map_at_20_diff1": 0.146536, + "nauc_map_at_100_max": 0.065365, + "nauc_map_at_100_std": 0.032941, + "nauc_map_at_100_diff1": 0.141843, + "nauc_map_at_1000_max": 0.063766, + "nauc_map_at_1000_std": 0.031932, + "nauc_map_at_1000_diff1": 0.143434, + "nauc_recall_at_1_max": 0.078812, + "nauc_recall_at_1_std": -0.071949, + "nauc_recall_at_1_diff1": 0.23682, + "nauc_recall_at_3_max": 0.059238, + "nauc_recall_at_3_std": 0.032412, + "nauc_recall_at_3_diff1": 0.153852, + "nauc_recall_at_5_max": 0.057943, + "nauc_recall_at_5_std": 0.030934, + "nauc_recall_at_5_diff1": 0.108594, + "nauc_recall_at_10_max": 0.054198, + "nauc_recall_at_10_std": 0.087953, + "nauc_recall_at_10_diff1": 0.075216, + "nauc_recall_at_20_max": 0.065959, + "nauc_recall_at_20_std": 0.124456, + "nauc_recall_at_20_diff1": 0.076525, + "nauc_recall_at_100_max": 0.081959, + "nauc_recall_at_100_std": 0.1883, + "nauc_recall_at_100_diff1": 0.091134, + "nauc_recall_at_1000_max": -0.162753, + "nauc_recall_at_1000_std": 0.163724, + "nauc_recall_at_1000_diff1": 0.059241, + "nauc_precision_at_1_max": -0.022165, + "nauc_precision_at_1_std": -0.091018, + "nauc_precision_at_1_diff1": 0.122576, + "nauc_precision_at_3_max": -0.021962, + "nauc_precision_at_3_std": -0.008107, + "nauc_precision_at_3_diff1": 0.097146, + "nauc_precision_at_5_max": 0.040892, + "nauc_precision_at_5_std": 0.025081, + "nauc_precision_at_5_diff1": 0.076435, + "nauc_precision_at_10_max": 0.023309, + "nauc_precision_at_10_std": 0.074589, + "nauc_precision_at_10_diff1": 0.02907, + "nauc_precision_at_20_max": 0.074716, + "nauc_precision_at_20_std": 0.081304, + "nauc_precision_at_20_diff1": 0.016411, + "nauc_precision_at_100_max": 0.084783, + "nauc_precision_at_100_std": 0.078373, + "nauc_precision_at_100_diff1": 0.018718, + "nauc_precision_at_1000_max": 0.056502, + "nauc_precision_at_1000_std": 0.030083, + "nauc_precision_at_1000_diff1": 0.022986, + "nauc_mrr_at_1_max": -0.022165, + "nauc_mrr_at_1_std": -0.091018, + "nauc_mrr_at_1_diff1": 0.122576, + "nauc_mrr_at_3_max": -0.017083, + "nauc_mrr_at_3_std": -0.050282, + "nauc_mrr_at_3_diff1": 0.114803, + "nauc_mrr_at_5_max": -0.002657, + "nauc_mrr_at_5_std": -0.045989, + "nauc_mrr_at_5_diff1": 0.109999, + "nauc_mrr_at_10_max": 0.000188, + "nauc_mrr_at_10_std": -0.039411, + "nauc_mrr_at_10_diff1": 0.09962, + "nauc_mrr_at_20_max": -0.000642, + "nauc_mrr_at_20_std": -0.041258, + "nauc_mrr_at_20_diff1": 0.100863, + "nauc_mrr_at_100_max": -0.000406, + "nauc_mrr_at_100_std": -0.040705, + "nauc_mrr_at_100_diff1": 0.103267, + "nauc_mrr_at_1000_max": -0.000543, + "nauc_mrr_at_1000_std": -0.040888, + "nauc_mrr_at_1000_diff1": 0.103049, + "hit_rate_at_1": 0.16867, + "hit_rate_at_3": 0.33434, + "hit_rate_at_5": 0.40964, + "hit_rate_at_10": 0.52711, + "hit_rate_at_20": 0.62349, + "hit_rate_at_100": 0.8012, + "hit_rate_at_1000": 0.98795, + "main_score": 0.19878, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.26054, + "ndcg_at_3": 0.26729, + "ndcg_at_5": 0.27172, + "ndcg_at_10": 0.30275, + "ndcg_at_20": 0.34027, + "ndcg_at_100": 0.40569, + "ndcg_at_1000": 0.45682, + "map_at_1": 0.10287, + "map_at_3": 0.16323, + "map_at_5": 0.18225, + "map_at_10": 0.20645, + "map_at_20": 0.22484, + "map_at_100": 0.24289, + "map_at_1000": 0.24719, + "recall_at_1": 0.10287, + "recall_at_3": 0.2065, + "recall_at_5": 0.25753, + "recall_at_10": 0.362, + "recall_at_20": 0.471, + "recall_at_100": 0.70009, + "recall_at_1000": 0.95624, + "accuracy": 0.10287, + "precision_at_1": 0.2741, + "precision_at_3": 0.21386, + "precision_at_5": 0.16747, + "precision_at_10": 0.1241, + "precision_at_20": 0.08825, + "precision_at_100": 0.02943, + "precision_at_1000": 0.00444, + "mrr_at_1": 0.274096, + "mrr_at_3": 0.363956, + "mrr_at_5": 0.379468, + "mrr_at_10": 0.396801, + "mrr_at_20": 0.401988, + "mrr_at_100": 0.405787, + "mrr_at_1000": 0.406309, + "nauc_ndcg_at_1_max": 0.017921, + "nauc_ndcg_at_1_std": -0.170919, + "nauc_ndcg_at_1_diff1": 0.268783, + "nauc_ndcg_at_3_max": -0.024857, + "nauc_ndcg_at_3_std": -0.153981, + "nauc_ndcg_at_3_diff1": 0.23378, + "nauc_ndcg_at_5_max": -0.020596, + "nauc_ndcg_at_5_std": -0.127181, + "nauc_ndcg_at_5_diff1": 0.246799, + "nauc_ndcg_at_10_max": -0.062542, + "nauc_ndcg_at_10_std": -0.137319, + "nauc_ndcg_at_10_diff1": 0.230908, + "nauc_ndcg_at_20_max": -0.035868, + "nauc_ndcg_at_20_std": -0.124404, + "nauc_ndcg_at_20_diff1": 0.234402, + "nauc_ndcg_at_100_max": -0.016628, + "nauc_ndcg_at_100_std": -0.086418, + "nauc_ndcg_at_100_diff1": 0.228025, + "nauc_ndcg_at_1000_max": -0.003641, + "nauc_ndcg_at_1000_std": -0.101786, + "nauc_ndcg_at_1000_diff1": 0.232946, + "nauc_map_at_1_max": -0.0016, + "nauc_map_at_1_std": -0.106381, + "nauc_map_at_1_diff1": 0.222771, + "nauc_map_at_3_max": -0.053736, + "nauc_map_at_3_std": -0.144483, + "nauc_map_at_3_diff1": 0.19996, + "nauc_map_at_5_max": -0.048546, + "nauc_map_at_5_std": -0.134792, + "nauc_map_at_5_diff1": 0.227004, + "nauc_map_at_10_max": -0.063184, + "nauc_map_at_10_std": -0.135181, + "nauc_map_at_10_diff1": 0.216175, + "nauc_map_at_20_max": -0.050025, + "nauc_map_at_20_std": -0.128199, + "nauc_map_at_20_diff1": 0.217218, + "nauc_map_at_100_max": -0.038117, + "nauc_map_at_100_std": -0.107758, + "nauc_map_at_100_diff1": 0.213179, + "nauc_map_at_1000_max": -0.037871, + "nauc_map_at_1000_std": -0.106662, + "nauc_map_at_1000_diff1": 0.213291, + "nauc_recall_at_1_max": -0.0016, + "nauc_recall_at_1_std": -0.106381, + "nauc_recall_at_1_diff1": 0.222771, + "nauc_recall_at_3_max": -0.093718, + "nauc_recall_at_3_std": -0.129142, + "nauc_recall_at_3_diff1": 0.136717, + "nauc_recall_at_5_max": -0.066213, + "nauc_recall_at_5_std": -0.105455, + "nauc_recall_at_5_diff1": 0.180621, + "nauc_recall_at_10_max": -0.133085, + "nauc_recall_at_10_std": -0.089139, + "nauc_recall_at_10_diff1": 0.124689, + "nauc_recall_at_20_max": -0.05998, + "nauc_recall_at_20_std": -0.056848, + "nauc_recall_at_20_diff1": 0.141785, + "nauc_recall_at_100_max": -0.043679, + "nauc_recall_at_100_std": 0.059635, + "nauc_recall_at_100_diff1": 0.115071, + "nauc_recall_at_1000_max": -0.109291, + "nauc_recall_at_1000_std": -0.071904, + "nauc_recall_at_1000_diff1": 0.121776, + "nauc_precision_at_1_max": 0.033852, + "nauc_precision_at_1_std": -0.154755, + "nauc_precision_at_1_diff1": 0.265489, + "nauc_precision_at_3_max": 0.003465, + "nauc_precision_at_3_std": -0.132622, + "nauc_precision_at_3_diff1": 0.237556, + "nauc_precision_at_5_max": 0.011112, + "nauc_precision_at_5_std": -0.081578, + "nauc_precision_at_5_diff1": 0.246813, + "nauc_precision_at_10_max": -0.002361, + "nauc_precision_at_10_std": -0.065032, + "nauc_precision_at_10_diff1": 0.175614, + "nauc_precision_at_20_max": 0.053804, + "nauc_precision_at_20_std": -0.016913, + "nauc_precision_at_20_diff1": 0.12946, + "nauc_precision_at_100_max": 0.109843, + "nauc_precision_at_100_std": 0.104497, + "nauc_precision_at_100_diff1": 0.043788, + "nauc_precision_at_1000_max": 0.13706, + "nauc_precision_at_1000_std": 0.093346, + "nauc_precision_at_1000_diff1": -0.015154, + "nauc_mrr_at_1_max": 0.033852, + "nauc_mrr_at_1_std": -0.154755, + "nauc_mrr_at_1_diff1": 0.265489, + "nauc_mrr_at_3_max": 0.016033, + "nauc_mrr_at_3_std": -0.136974, + "nauc_mrr_at_3_diff1": 0.265964, + "nauc_mrr_at_5_max": 0.030648, + "nauc_mrr_at_5_std": -0.13145, + "nauc_mrr_at_5_diff1": 0.267024, + "nauc_mrr_at_10_max": 0.026474, + "nauc_mrr_at_10_std": -0.126816, + "nauc_mrr_at_10_diff1": 0.26344, + "nauc_mrr_at_20_max": 0.027754, + "nauc_mrr_at_20_std": -0.127848, + "nauc_mrr_at_20_diff1": 0.265573, + "nauc_mrr_at_100_max": 0.026162, + "nauc_mrr_at_100_std": -0.127877, + "nauc_mrr_at_100_diff1": 0.264119, + "nauc_mrr_at_1000_max": 0.026627, + "nauc_mrr_at_1000_std": -0.127876, + "nauc_mrr_at_1000_diff1": 0.264321, + "hit_rate_at_1": 0.2741, + "hit_rate_at_3": 0.47892, + "hit_rate_at_5": 0.54518, + "hit_rate_at_10": 0.67771, + "hit_rate_at_20": 0.75301, + "hit_rate_at_100": 0.88855, + "hit_rate_at_1000": 0.99398, + "main_score": 0.30275, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.14006, + "ndcg_at_3": 0.14535, + "ndcg_at_5": 0.15462, + "ndcg_at_10": 0.17591, + "ndcg_at_20": 0.20531, + "ndcg_at_100": 0.26651, + "ndcg_at_1000": 0.3393, + "map_at_1": 0.05097, + "map_at_3": 0.08318, + "map_at_5": 0.09815, + "map_at_10": 0.11298, + "map_at_20": 0.12589, + "map_at_100": 0.13928, + "map_at_1000": 0.14433, + "recall_at_1": 0.05097, + "recall_at_3": 0.11187, + "recall_at_5": 0.15537, + "recall_at_10": 0.22174, + "recall_at_20": 0.30192, + "recall_at_100": 0.51949, + "recall_at_1000": 0.90413, + "accuracy": 0.05097, + "precision_at_1": 0.1506, + "precision_at_3": 0.11948, + "precision_at_5": 0.0988, + "precision_at_10": 0.07831, + "precision_at_20": 0.05919, + "precision_at_100": 0.02262, + "precision_at_1000": 0.00419, + "mrr_at_1": 0.150602, + "mrr_at_3": 0.206827, + "mrr_at_5": 0.221737, + "mrr_at_10": 0.237158, + "mrr_at_20": 0.243777, + "mrr_at_100": 0.249145, + "mrr_at_1000": 0.250164, + "nauc_ndcg_at_1_max": 0.090523, + "nauc_ndcg_at_1_std": 0.048298, + "nauc_ndcg_at_1_diff1": 0.300746, + "nauc_ndcg_at_3_max": 0.06588, + "nauc_ndcg_at_3_std": 0.072732, + "nauc_ndcg_at_3_diff1": 0.204254, + "nauc_ndcg_at_5_max": 0.086923, + "nauc_ndcg_at_5_std": 0.090535, + "nauc_ndcg_at_5_diff1": 0.200561, + "nauc_ndcg_at_10_max": 0.089987, + "nauc_ndcg_at_10_std": 0.103771, + "nauc_ndcg_at_10_diff1": 0.168309, + "nauc_ndcg_at_20_max": 0.091772, + "nauc_ndcg_at_20_std": 0.114666, + "nauc_ndcg_at_20_diff1": 0.13638, + "nauc_ndcg_at_100_max": 0.124752, + "nauc_ndcg_at_100_std": 0.16489, + "nauc_ndcg_at_100_diff1": 0.10963, + "nauc_ndcg_at_1000_max": 0.103829, + "nauc_ndcg_at_1000_std": 0.13749, + "nauc_ndcg_at_1000_diff1": 0.122766, + "nauc_map_at_1_max": 0.161133, + "nauc_map_at_1_std": 0.041969, + "nauc_map_at_1_diff1": 0.317118, + "nauc_map_at_3_max": 0.101439, + "nauc_map_at_3_std": 0.065979, + "nauc_map_at_3_diff1": 0.224092, + "nauc_map_at_5_max": 0.103008, + "nauc_map_at_5_std": 0.079606, + "nauc_map_at_5_diff1": 0.221896, + "nauc_map_at_10_max": 0.101638, + "nauc_map_at_10_std": 0.090748, + "nauc_map_at_10_diff1": 0.198179, + "nauc_map_at_20_max": 0.106828, + "nauc_map_at_20_std": 0.101043, + "nauc_map_at_20_diff1": 0.177569, + "nauc_map_at_100_max": 0.1203, + "nauc_map_at_100_std": 0.124588, + "nauc_map_at_100_diff1": 0.161724, + "nauc_map_at_1000_max": 0.119386, + "nauc_map_at_1000_std": 0.124911, + "nauc_map_at_1000_diff1": 0.162898, + "nauc_recall_at_1_max": 0.161133, + "nauc_recall_at_1_std": 0.041969, + "nauc_recall_at_1_diff1": 0.317118, + "nauc_recall_at_3_max": 0.052495, + "nauc_recall_at_3_std": 0.08947, + "nauc_recall_at_3_diff1": 0.151781, + "nauc_recall_at_5_max": 0.079562, + "nauc_recall_at_5_std": 0.127694, + "nauc_recall_at_5_diff1": 0.152283, + "nauc_recall_at_10_max": 0.065388, + "nauc_recall_at_10_std": 0.132275, + "nauc_recall_at_10_diff1": 0.087689, + "nauc_recall_at_20_max": 0.060516, + "nauc_recall_at_20_std": 0.152933, + "nauc_recall_at_20_diff1": 0.040437, + "nauc_recall_at_100_max": 0.155841, + "nauc_recall_at_100_std": 0.276573, + "nauc_recall_at_100_diff1": -0.013017, + "nauc_recall_at_1000_max": -0.056872, + "nauc_recall_at_1000_std": 0.211122, + "nauc_recall_at_1000_diff1": -0.163777, + "nauc_precision_at_1_max": 0.109051, + "nauc_precision_at_1_std": 0.061605, + "nauc_precision_at_1_diff1": 0.308058, + "nauc_precision_at_3_max": 0.057387, + "nauc_precision_at_3_std": 0.093508, + "nauc_precision_at_3_diff1": 0.169324, + "nauc_precision_at_5_max": 0.063774, + "nauc_precision_at_5_std": 0.09757, + "nauc_precision_at_5_diff1": 0.144386, + "nauc_precision_at_10_max": 0.097224, + "nauc_precision_at_10_std": 0.130309, + "nauc_precision_at_10_diff1": 0.05329, + "nauc_precision_at_20_max": 0.107004, + "nauc_precision_at_20_std": 0.139322, + "nauc_precision_at_20_diff1": -0.015395, + "nauc_precision_at_100_max": 0.13373, + "nauc_precision_at_100_std": 0.190755, + "nauc_precision_at_100_diff1": -0.057924, + "nauc_precision_at_1000_max": 0.060971, + "nauc_precision_at_1000_std": 0.086456, + "nauc_precision_at_1000_diff1": -0.047967, + "nauc_mrr_at_1_max": 0.109051, + "nauc_mrr_at_1_std": 0.061605, + "nauc_mrr_at_1_diff1": 0.308058, + "nauc_mrr_at_3_max": 0.074839, + "nauc_mrr_at_3_std": 0.073061, + "nauc_mrr_at_3_diff1": 0.25048, + "nauc_mrr_at_5_max": 0.085493, + "nauc_mrr_at_5_std": 0.082644, + "nauc_mrr_at_5_diff1": 0.253197, + "nauc_mrr_at_10_max": 0.08089, + "nauc_mrr_at_10_std": 0.079415, + "nauc_mrr_at_10_diff1": 0.236776, + "nauc_mrr_at_20_max": 0.081593, + "nauc_mrr_at_20_std": 0.082248, + "nauc_mrr_at_20_diff1": 0.23198, + "nauc_mrr_at_100_max": 0.083463, + "nauc_mrr_at_100_std": 0.085019, + "nauc_mrr_at_100_diff1": 0.234047, + "nauc_mrr_at_1000_max": 0.083124, + "nauc_mrr_at_1000_std": 0.084419, + "nauc_mrr_at_1000_diff1": 0.234316, + "hit_rate_at_1": 0.1506, + "hit_rate_at_3": 0.28313, + "hit_rate_at_5": 0.34639, + "hit_rate_at_10": 0.46084, + "hit_rate_at_20": 0.55422, + "hit_rate_at_100": 0.76506, + "hit_rate_at_1000": 0.9759, + "main_score": 0.17591, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.16867, + "ndcg_at_3": 0.16478, + "ndcg_at_5": 0.18046, + "ndcg_at_10": 0.19329, + "ndcg_at_20": 0.23134, + "ndcg_at_100": 0.28955, + "ndcg_at_1000": 0.35908, + "map_at_1": 0.05774, + "map_at_3": 0.09073, + "map_at_5": 0.11196, + "map_at_10": 0.12385, + "map_at_20": 0.13841, + "map_at_100": 0.15209, + "map_at_1000": 0.15696, + "recall_at_1": 0.05774, + "recall_at_3": 0.12172, + "recall_at_5": 0.1828, + "recall_at_10": 0.23574, + "recall_at_20": 0.34627, + "recall_at_100": 0.55389, + "recall_at_1000": 0.92319, + "accuracy": 0.05774, + "precision_at_1": 0.18373, + "precision_at_3": 0.13655, + "precision_at_5": 0.12108, + "precision_at_10": 0.08494, + "precision_at_20": 0.06416, + "precision_at_100": 0.02298, + "precision_at_1000": 0.00423, + "mrr_at_1": 0.183735, + "mrr_at_3": 0.246486, + "mrr_at_5": 0.264859, + "mrr_at_10": 0.276335, + "mrr_at_20": 0.283977, + "mrr_at_100": 0.288592, + "mrr_at_1000": 0.289448, + "nauc_ndcg_at_1_max": -0.008132, + "nauc_ndcg_at_1_std": -0.049405, + "nauc_ndcg_at_1_diff1": 0.225399, + "nauc_ndcg_at_3_max": 0.020099, + "nauc_ndcg_at_3_std": -0.032022, + "nauc_ndcg_at_3_diff1": 0.156662, + "nauc_ndcg_at_5_max": 0.021169, + "nauc_ndcg_at_5_std": -0.029966, + "nauc_ndcg_at_5_diff1": 0.202475, + "nauc_ndcg_at_10_max": 0.029608, + "nauc_ndcg_at_10_std": 0.005315, + "nauc_ndcg_at_10_diff1": 0.186042, + "nauc_ndcg_at_20_max": 0.049685, + "nauc_ndcg_at_20_std": 0.022715, + "nauc_ndcg_at_20_diff1": 0.171504, + "nauc_ndcg_at_100_max": 0.064657, + "nauc_ndcg_at_100_std": 0.043771, + "nauc_ndcg_at_100_diff1": 0.144703, + "nauc_ndcg_at_1000_max": 0.034989, + "nauc_ndcg_at_1000_std": 0.015124, + "nauc_ndcg_at_1000_diff1": 0.160359, + "nauc_map_at_1_max": 0.051215, + "nauc_map_at_1_std": -0.018843, + "nauc_map_at_1_diff1": 0.251687, + "nauc_map_at_3_max": 0.070206, + "nauc_map_at_3_std": -0.004349, + "nauc_map_at_3_diff1": 0.183552, + "nauc_map_at_5_max": 0.050155, + "nauc_map_at_5_std": -0.013755, + "nauc_map_at_5_diff1": 0.213212, + "nauc_map_at_10_max": 0.052164, + "nauc_map_at_10_std": 0.004993, + "nauc_map_at_10_diff1": 0.187292, + "nauc_map_at_20_max": 0.060234, + "nauc_map_at_20_std": 0.014211, + "nauc_map_at_20_diff1": 0.18219, + "nauc_map_at_100_max": 0.069867, + "nauc_map_at_100_std": 0.027002, + "nauc_map_at_100_diff1": 0.171641, + "nauc_map_at_1000_max": 0.068993, + "nauc_map_at_1000_std": 0.026863, + "nauc_map_at_1000_diff1": 0.171209, + "nauc_recall_at_1_max": 0.051215, + "nauc_recall_at_1_std": -0.018843, + "nauc_recall_at_1_diff1": 0.251687, + "nauc_recall_at_3_max": 0.061291, + "nauc_recall_at_3_std": 0.00413, + "nauc_recall_at_3_diff1": 0.121256, + "nauc_recall_at_5_max": 0.006901, + "nauc_recall_at_5_std": -0.021474, + "nauc_recall_at_5_diff1": 0.180398, + "nauc_recall_at_10_max": 0.037028, + "nauc_recall_at_10_std": 0.060742, + "nauc_recall_at_10_diff1": 0.122057, + "nauc_recall_at_20_max": 0.064507, + "nauc_recall_at_20_std": 0.10291, + "nauc_recall_at_20_diff1": 0.085854, + "nauc_recall_at_100_max": 0.123426, + "nauc_recall_at_100_std": 0.163282, + "nauc_recall_at_100_diff1": 0.022127, + "nauc_recall_at_1000_max": -0.229688, + "nauc_recall_at_1000_std": -0.006956, + "nauc_recall_at_1000_diff1": 0.017202, + "nauc_precision_at_1_max": 0.022345, + "nauc_precision_at_1_std": -0.042539, + "nauc_precision_at_1_diff1": 0.176483, + "nauc_precision_at_3_max": 0.00968, + "nauc_precision_at_3_std": -0.025566, + "nauc_precision_at_3_diff1": 0.083569, + "nauc_precision_at_5_max": -0.00652, + "nauc_precision_at_5_std": -0.027355, + "nauc_precision_at_5_diff1": 0.144343, + "nauc_precision_at_10_max": 0.021651, + "nauc_precision_at_10_std": 0.020939, + "nauc_precision_at_10_diff1": 0.100469, + "nauc_precision_at_20_max": 0.041134, + "nauc_precision_at_20_std": 0.033088, + "nauc_precision_at_20_diff1": 0.062427, + "nauc_precision_at_100_max": 0.035086, + "nauc_precision_at_100_std": 0.047995, + "nauc_precision_at_100_diff1": -0.015969, + "nauc_precision_at_1000_max": -0.011317, + "nauc_precision_at_1000_std": 0.013426, + "nauc_precision_at_1000_diff1": -0.026683, + "nauc_mrr_at_1_max": 0.022345, + "nauc_mrr_at_1_std": -0.042539, + "nauc_mrr_at_1_diff1": 0.176483, + "nauc_mrr_at_3_max": 0.023816, + "nauc_mrr_at_3_std": -0.047315, + "nauc_mrr_at_3_diff1": 0.145705, + "nauc_mrr_at_5_max": 0.014219, + "nauc_mrr_at_5_std": -0.050205, + "nauc_mrr_at_5_diff1": 0.150446, + "nauc_mrr_at_10_max": 0.020889, + "nauc_mrr_at_10_std": -0.040869, + "nauc_mrr_at_10_diff1": 0.151535, + "nauc_mrr_at_20_max": 0.022591, + "nauc_mrr_at_20_std": -0.04097, + "nauc_mrr_at_20_diff1": 0.144044, + "nauc_mrr_at_100_max": 0.02434, + "nauc_mrr_at_100_std": -0.038663, + "nauc_mrr_at_100_diff1": 0.143491, + "nauc_mrr_at_1000_max": 0.024071, + "nauc_mrr_at_1000_std": -0.039077, + "nauc_mrr_at_1000_diff1": 0.144102, + "hit_rate_at_1": 0.18373, + "hit_rate_at_3": 0.33133, + "hit_rate_at_5": 0.41265, + "hit_rate_at_10": 0.50602, + "hit_rate_at_20": 0.62048, + "hit_rate_at_100": 0.79819, + "hit_rate_at_1000": 0.98494, + "main_score": 0.19329, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.15361, + "ndcg_at_3": 0.1631, + "ndcg_at_5": 0.17606, + "ndcg_at_10": 0.1976, + "ndcg_at_20": 0.2301, + "ndcg_at_100": 0.29322, + "ndcg_at_1000": 0.36262, + "map_at_1": 0.06547, + "map_at_3": 0.09865, + "map_at_5": 0.11339, + "map_at_10": 0.12842, + "map_at_20": 0.14081, + "map_at_100": 0.15534, + "map_at_1000": 0.16051, + "recall_at_1": 0.06547, + "recall_at_3": 0.12559, + "recall_at_5": 0.17003, + "recall_at_10": 0.24831, + "recall_at_20": 0.33672, + "recall_at_100": 0.56318, + "recall_at_1000": 0.92987, + "accuracy": 0.06547, + "precision_at_1": 0.16867, + "precision_at_3": 0.13554, + "precision_at_5": 0.11867, + "precision_at_10": 0.08614, + "precision_at_20": 0.06325, + "precision_at_100": 0.02364, + "precision_at_1000": 0.00428, + "mrr_at_1": 0.168675, + "mrr_at_3": 0.228916, + "mrr_at_5": 0.251958, + "mrr_at_10": 0.267958, + "mrr_at_20": 0.275168, + "mrr_at_100": 0.279375, + "mrr_at_1000": 0.280168, + "nauc_ndcg_at_1_max": -0.018651, + "nauc_ndcg_at_1_std": -0.033738, + "nauc_ndcg_at_1_diff1": 0.303083, + "nauc_ndcg_at_3_max": -0.011718, + "nauc_ndcg_at_3_std": -0.029307, + "nauc_ndcg_at_3_diff1": 0.237324, + "nauc_ndcg_at_5_max": 0.001649, + "nauc_ndcg_at_5_std": -0.016422, + "nauc_ndcg_at_5_diff1": 0.223645, + "nauc_ndcg_at_10_max": 0.006028, + "nauc_ndcg_at_10_std": 0.03406, + "nauc_ndcg_at_10_diff1": 0.196036, + "nauc_ndcg_at_20_max": 0.038853, + "nauc_ndcg_at_20_std": 0.050282, + "nauc_ndcg_at_20_diff1": 0.196393, + "nauc_ndcg_at_100_max": 0.033892, + "nauc_ndcg_at_100_std": 0.070277, + "nauc_ndcg_at_100_diff1": 0.198372, + "nauc_ndcg_at_1000_max": 0.026081, + "nauc_ndcg_at_1000_std": 0.04348, + "nauc_ndcg_at_1000_diff1": 0.210371, + "nauc_map_at_1_max": 0.04682, + "nauc_map_at_1_std": -0.011913, + "nauc_map_at_1_diff1": 0.323569, + "nauc_map_at_3_max": 0.02001, + "nauc_map_at_3_std": -0.017492, + "nauc_map_at_3_diff1": 0.249574, + "nauc_map_at_5_max": 0.013119, + "nauc_map_at_5_std": -0.017058, + "nauc_map_at_5_diff1": 0.233964, + "nauc_map_at_10_max": 0.013528, + "nauc_map_at_10_std": 0.017985, + "nauc_map_at_10_diff1": 0.213219, + "nauc_map_at_20_max": 0.026343, + "nauc_map_at_20_std": 0.031585, + "nauc_map_at_20_diff1": 0.206098, + "nauc_map_at_100_max": 0.029583, + "nauc_map_at_100_std": 0.043774, + "nauc_map_at_100_diff1": 0.198663, + "nauc_map_at_1000_max": 0.028521, + "nauc_map_at_1000_std": 0.042951, + "nauc_map_at_1000_diff1": 0.201526, + "nauc_recall_at_1_max": 0.04682, + "nauc_recall_at_1_std": -0.011913, + "nauc_recall_at_1_diff1": 0.323569, + "nauc_recall_at_3_max": 0.01959, + "nauc_recall_at_3_std": 0.008223, + "nauc_recall_at_3_diff1": 0.211247, + "nauc_recall_at_5_max": 0.010908, + "nauc_recall_at_5_std": -0.002301, + "nauc_recall_at_5_diff1": 0.170673, + "nauc_recall_at_10_max": 0.01756, + "nauc_recall_at_10_std": 0.114659, + "nauc_recall_at_10_diff1": 0.079073, + "nauc_recall_at_20_max": 0.077298, + "nauc_recall_at_20_std": 0.135826, + "nauc_recall_at_20_diff1": 0.112209, + "nauc_recall_at_100_max": 0.069315, + "nauc_recall_at_100_std": 0.198334, + "nauc_recall_at_100_diff1": 0.118775, + "nauc_recall_at_1000_max": -0.139171, + "nauc_recall_at_1000_std": 0.091897, + "nauc_recall_at_1000_diff1": 0.032941, + "nauc_precision_at_1_max": -0.009467, + "nauc_precision_at_1_std": -0.040555, + "nauc_precision_at_1_diff1": 0.267316, + "nauc_precision_at_3_max": -0.037472, + "nauc_precision_at_3_std": -0.042463, + "nauc_precision_at_3_diff1": 0.160913, + "nauc_precision_at_5_max": -0.018746, + "nauc_precision_at_5_std": -0.014158, + "nauc_precision_at_5_diff1": 0.11445, + "nauc_precision_at_10_max": -0.019829, + "nauc_precision_at_10_std": 0.06228, + "nauc_precision_at_10_diff1": 0.062875, + "nauc_precision_at_20_max": 0.045089, + "nauc_precision_at_20_std": 0.108018, + "nauc_precision_at_20_diff1": 0.024481, + "nauc_precision_at_100_max": 0.006425, + "nauc_precision_at_100_std": 0.090028, + "nauc_precision_at_100_diff1": 0.027546, + "nauc_precision_at_1000_max": 0.031416, + "nauc_precision_at_1000_std": 0.052575, + "nauc_precision_at_1000_diff1": 0.008222, + "nauc_mrr_at_1_max": -0.009467, + "nauc_mrr_at_1_std": -0.040555, + "nauc_mrr_at_1_diff1": 0.267316, + "nauc_mrr_at_3_max": -0.010602, + "nauc_mrr_at_3_std": -0.036177, + "nauc_mrr_at_3_diff1": 0.217539, + "nauc_mrr_at_5_max": -0.001583, + "nauc_mrr_at_5_std": -0.040995, + "nauc_mrr_at_5_diff1": 0.21188, + "nauc_mrr_at_10_max": -0.002393, + "nauc_mrr_at_10_std": -0.028873, + "nauc_mrr_at_10_diff1": 0.203304, + "nauc_mrr_at_20_max": 0.005405, + "nauc_mrr_at_20_std": -0.026623, + "nauc_mrr_at_20_diff1": 0.2079, + "nauc_mrr_at_100_max": 0.004549, + "nauc_mrr_at_100_std": -0.025972, + "nauc_mrr_at_100_diff1": 0.209855, + "nauc_mrr_at_1000_max": 0.004115, + "nauc_mrr_at_1000_std": -0.026372, + "nauc_mrr_at_1000_diff1": 0.20964, + "hit_rate_at_1": 0.16867, + "hit_rate_at_3": 0.30723, + "hit_rate_at_5": 0.40663, + "hit_rate_at_10": 0.52711, + "hit_rate_at_20": 0.63253, + "hit_rate_at_100": 0.8012, + "hit_rate_at_1000": 0.98795, + "main_score": 0.1976, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.10" + } + ] + }, + "evaluation_time": 1673.7006321059307, + "kg_co2_emissions": null, + "date": 1792351619.732035, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 8.232076652348042e-06, + "end": 25.69836485409178, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 25.698451164062135, + "end": 25.698455711011775, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries", + "start": 25.736358478083275, + "end": 31.189771877019666, + "split": "test", + "subset": "french" + }, + { + "name": "Searching corpus", + "start": 31.192601107992232, + "end": 267.39158782199956, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 267.45104464003816, + "end": 267.7055856770603, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 267.7553299689898, + "end": 268.1952574410243, + "split": "test", + "subset": "spanish" + }, + { + "name": "Searching corpus", + "start": 268.1977245489834, + "end": 517.1357855020324, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 517.1816764440155, + "end": 517.4124362070579, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 517.4670203829883, + "end": 518.362105126027, + "split": "test", + "subset": "english" + }, + { + "name": "Searching corpus", + "start": 518.3640286270529, + "end": 786.9867734450381, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 787.0439486349933, + "end": 787.2852207149845, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 787.3302335779881, + "end": 788.8229114100104, + "split": "test", + "subset": "german" + }, + { + "name": "Searching corpus", + "start": 788.8277909730095, + "end": 1107.327730191988, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 1107.377682417049, + "end": 1107.6032516279956, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 1107.6498098029988, + "end": 1108.028156768065, + "split": "test", + "subset": "italian" + }, + { + "name": "Searching corpus", + "start": 1108.032235878054, + "end": 1376.9409373860108, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 1376.9868914630497, + "end": 1377.219151037978, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 1377.2698482429842, + "end": 1377.75251914002, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Searching corpus", + "start": 1377.7544081690721, + "end": 1673.3656608869787, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 1673.4266327969963, + "end": 1673.6876030369895, + "split": "test", + "subset": "portuguese" + } + ] +} \ No newline at end of file diff --git a/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3NuclearRetrieval.v2.json b/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3NuclearRetrieval.v2.json new file mode 100644 index 0000000000..a67cb71ead --- /dev/null +++ b/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3NuclearRetrieval.v2.json @@ -0,0 +1,1090 @@ +{ + "dataset_revision": "826e3bc829e01f5a088d0435114200bce6058af8", + "task_name": "Vidore3NuclearRetrieval.v2", + "mteb_version": "2.26.10", + "scores": { + "test": [ + { + "ndcg_at_1": 0.2244, + "ndcg_at_3": 0.21969, + "ndcg_at_5": 0.23402, + "ndcg_at_10": 0.25364, + "ndcg_at_20": 0.28608, + "ndcg_at_100": 0.34677, + "ndcg_at_1000": 0.40837, + "map_at_1": 0.08696, + "map_at_3": 0.13499, + "map_at_5": 0.15497, + "map_at_10": 0.17307, + "map_at_20": 0.18828, + "map_at_100": 0.20267, + "map_at_1000": 0.20728, + "recall_at_1": 0.08696, + "recall_at_3": 0.17047, + "recall_at_5": 0.22533, + "recall_at_10": 0.29407, + "recall_at_20": 0.39112, + "recall_at_100": 0.60861, + "recall_at_1000": 0.93046, + "accuracy": 0.08696, + "precision_at_1": 0.24398, + "precision_at_3": 0.17369, + "precision_at_5": 0.1494, + "precision_at_10": 0.10482, + "precision_at_20": 0.07184, + "precision_at_100": 0.02518, + "precision_at_1000": 0.00429, + "mrr_at_1": 0.243976, + "mrr_at_3": 0.319277, + "mrr_at_5": 0.337199, + "mrr_at_10": 0.348526, + "mrr_at_20": 0.354369, + "mrr_at_100": 0.358342, + "mrr_at_1000": 0.358996, + "nauc_ndcg_at_1_max": -0.015628, + "nauc_ndcg_at_1_std": -0.120502, + "nauc_ndcg_at_1_diff1": 0.276909, + "nauc_ndcg_at_3_max": -0.02276, + "nauc_ndcg_at_3_std": -0.110741, + "nauc_ndcg_at_3_diff1": 0.170958, + "nauc_ndcg_at_5_max": -0.032275, + "nauc_ndcg_at_5_std": -0.108781, + "nauc_ndcg_at_5_diff1": 0.160954, + "nauc_ndcg_at_10_max": -0.017244, + "nauc_ndcg_at_10_std": -0.090282, + "nauc_ndcg_at_10_diff1": 0.152237, + "nauc_ndcg_at_20_max": -0.013771, + "nauc_ndcg_at_20_std": -0.06777, + "nauc_ndcg_at_20_diff1": 0.159932, + "nauc_ndcg_at_100_max": -0.025446, + "nauc_ndcg_at_100_std": -0.072674, + "nauc_ndcg_at_100_diff1": 0.165696, + "nauc_ndcg_at_1000_max": -0.019047, + "nauc_ndcg_at_1000_std": -0.083977, + "nauc_ndcg_at_1000_diff1": 0.163531, + "nauc_map_at_1_max": 0.008036, + "nauc_map_at_1_std": -0.092029, + "nauc_map_at_1_diff1": 0.208103, + "nauc_map_at_3_max": 0.012459, + "nauc_map_at_3_std": -0.082663, + "nauc_map_at_3_diff1": 0.151164, + "nauc_map_at_5_max": -0.00587, + "nauc_map_at_5_std": -0.083185, + "nauc_map_at_5_diff1": 0.150551, + "nauc_map_at_10_max": 0.001908, + "nauc_map_at_10_std": -0.080104, + "nauc_map_at_10_diff1": 0.150305, + "nauc_map_at_20_max": 0.0079, + "nauc_map_at_20_std": -0.064491, + "nauc_map_at_20_diff1": 0.149877, + "nauc_map_at_100_max": 0.004367, + "nauc_map_at_100_std": -0.061996, + "nauc_map_at_100_diff1": 0.151496, + "nauc_map_at_1000_max": 0.004098, + "nauc_map_at_1000_std": -0.06163, + "nauc_map_at_1000_diff1": 0.15056, + "nauc_recall_at_1_max": 0.008036, + "nauc_recall_at_1_std": -0.092029, + "nauc_recall_at_1_diff1": 0.208103, + "nauc_recall_at_3_max": 0.00016, + "nauc_recall_at_3_std": -0.071167, + "nauc_recall_at_3_diff1": 0.104928, + "nauc_recall_at_5_max": 0.00276, + "nauc_recall_at_5_std": -0.037902, + "nauc_recall_at_5_diff1": 0.112188, + "nauc_recall_at_10_max": 0.005458, + "nauc_recall_at_10_std": -0.03418, + "nauc_recall_at_10_diff1": 0.092601, + "nauc_recall_at_20_max": 0.008119, + "nauc_recall_at_20_std": 0.006159, + "nauc_recall_at_20_diff1": 0.104659, + "nauc_recall_at_100_max": -0.025492, + "nauc_recall_at_100_std": 0.004347, + "nauc_recall_at_100_diff1": 0.102962, + "nauc_recall_at_1000_max": -0.203128, + "nauc_recall_at_1000_std": -0.155195, + "nauc_recall_at_1000_diff1": 0.114588, + "nauc_precision_at_1_max": 0.00298, + "nauc_precision_at_1_std": -0.111563, + "nauc_precision_at_1_diff1": 0.272521, + "nauc_precision_at_3_max": -0.042193, + "nauc_precision_at_3_std": -0.130779, + "nauc_precision_at_3_diff1": 0.169751, + "nauc_precision_at_5_max": -0.096702, + "nauc_precision_at_5_std": -0.149925, + "nauc_precision_at_5_diff1": 0.135388, + "nauc_precision_at_10_max": -0.016155, + "nauc_precision_at_10_std": -0.065257, + "nauc_precision_at_10_diff1": 0.090518, + "nauc_precision_at_20_max": 0.009548, + "nauc_precision_at_20_std": 0.003491, + "nauc_precision_at_20_diff1": 0.074479, + "nauc_precision_at_100_max": -0.016471, + "nauc_precision_at_100_std": 0.002603, + "nauc_precision_at_100_diff1": 0.023831, + "nauc_precision_at_1000_max": 0.054836, + "nauc_precision_at_1000_std": 0.038851, + "nauc_precision_at_1000_diff1": -0.040543, + "nauc_mrr_at_1_max": 0.00298, + "nauc_mrr_at_1_std": -0.111563, + "nauc_mrr_at_1_diff1": 0.272521, + "nauc_mrr_at_3_max": -0.029114, + "nauc_mrr_at_3_std": -0.109612, + "nauc_mrr_at_3_diff1": 0.233401, + "nauc_mrr_at_5_max": -0.026564, + "nauc_mrr_at_5_std": -0.113362, + "nauc_mrr_at_5_diff1": 0.233428, + "nauc_mrr_at_10_max": -0.021826, + "nauc_mrr_at_10_std": -0.105471, + "nauc_mrr_at_10_diff1": 0.224494, + "nauc_mrr_at_20_max": -0.022535, + "nauc_mrr_at_20_std": -0.104294, + "nauc_mrr_at_20_diff1": 0.22744, + "nauc_mrr_at_100_max": -0.022739, + "nauc_mrr_at_100_std": -0.10558, + "nauc_mrr_at_100_diff1": 0.228197, + "nauc_mrr_at_1000_max": -0.022854, + "nauc_mrr_at_1000_std": -0.105837, + "nauc_mrr_at_1000_diff1": 0.228273, + "hit_rate_at_1": 0.24398, + "hit_rate_at_3": 0.40964, + "hit_rate_at_5": 0.48795, + "hit_rate_at_10": 0.57229, + "hit_rate_at_20": 0.65663, + "hit_rate_at_100": 0.8253, + "hit_rate_at_1000": 0.98494, + "main_score": 0.25364, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.20482, + "ndcg_at_3": 0.20095, + "ndcg_at_5": 0.21275, + "ndcg_at_10": 0.23946, + "ndcg_at_20": 0.26619, + "ndcg_at_100": 0.32957, + "ndcg_at_1000": 0.394, + "map_at_1": 0.0772, + "map_at_3": 0.11851, + "map_at_5": 0.137, + "map_at_10": 0.15773, + "map_at_20": 0.17103, + "map_at_100": 0.18537, + "map_at_1000": 0.19025, + "recall_at_1": 0.0772, + "recall_at_3": 0.15443, + "recall_at_5": 0.20546, + "recall_at_10": 0.29316, + "recall_at_20": 0.36634, + "recall_at_100": 0.59904, + "recall_at_1000": 0.93254, + "accuracy": 0.0772, + "precision_at_1": 0.21386, + "precision_at_3": 0.16064, + "precision_at_5": 0.13494, + "precision_at_10": 0.10271, + "precision_at_20": 0.07078, + "precision_at_100": 0.02467, + "precision_at_1000": 0.00432, + "mrr_at_1": 0.213855, + "mrr_at_3": 0.288655, + "mrr_at_5": 0.308233, + "mrr_at_10": 0.321891, + "mrr_at_20": 0.326252, + "mrr_at_100": 0.331279, + "mrr_at_1000": 0.331993, + "nauc_ndcg_at_1_max": 0.08484, + "nauc_ndcg_at_1_std": -0.054692, + "nauc_ndcg_at_1_diff1": 0.244821, + "nauc_ndcg_at_3_max": 0.083089, + "nauc_ndcg_at_3_std": -0.045838, + "nauc_ndcg_at_3_diff1": 0.172317, + "nauc_ndcg_at_5_max": 0.075867, + "nauc_ndcg_at_5_std": -0.025548, + "nauc_ndcg_at_5_diff1": 0.121656, + "nauc_ndcg_at_10_max": 0.066156, + "nauc_ndcg_at_10_std": -0.037144, + "nauc_ndcg_at_10_diff1": 0.128005, + "nauc_ndcg_at_20_max": 0.081324, + "nauc_ndcg_at_20_std": -0.02471, + "nauc_ndcg_at_20_diff1": 0.132693, + "nauc_ndcg_at_100_max": 0.068138, + "nauc_ndcg_at_100_std": -0.024404, + "nauc_ndcg_at_100_diff1": 0.143229, + "nauc_ndcg_at_1000_max": 0.075205, + "nauc_ndcg_at_1000_std": -0.040001, + "nauc_ndcg_at_1000_diff1": 0.152599, + "nauc_map_at_1_max": 0.194234, + "nauc_map_at_1_std": 0.010639, + "nauc_map_at_1_diff1": 0.209688, + "nauc_map_at_3_max": 0.135777, + "nauc_map_at_3_std": 0.003088, + "nauc_map_at_3_diff1": 0.147521, + "nauc_map_at_5_max": 0.120856, + "nauc_map_at_5_std": 0.014744, + "nauc_map_at_5_diff1": 0.110436, + "nauc_map_at_10_max": 0.111805, + "nauc_map_at_10_std": 0.000117, + "nauc_map_at_10_diff1": 0.12824, + "nauc_map_at_20_max": 0.121932, + "nauc_map_at_20_std": 0.005421, + "nauc_map_at_20_diff1": 0.127623, + "nauc_map_at_100_max": 0.119383, + "nauc_map_at_100_std": 0.007382, + "nauc_map_at_100_diff1": 0.129654, + "nauc_map_at_1000_max": 0.118341, + "nauc_map_at_1000_std": 0.006019, + "nauc_map_at_1000_diff1": 0.130385, + "nauc_recall_at_1_max": 0.194234, + "nauc_recall_at_1_std": 0.010639, + "nauc_recall_at_1_diff1": 0.209688, + "nauc_recall_at_3_max": 0.095311, + "nauc_recall_at_3_std": 0.006859, + "nauc_recall_at_3_diff1": 0.084844, + "nauc_recall_at_5_max": 0.066649, + "nauc_recall_at_5_std": 0.03047, + "nauc_recall_at_5_diff1": 0.013415, + "nauc_recall_at_10_max": 0.051722, + "nauc_recall_at_10_std": 0.000815, + "nauc_recall_at_10_diff1": 0.062555, + "nauc_recall_at_20_max": 0.09255, + "nauc_recall_at_20_std": 0.022311, + "nauc_recall_at_20_diff1": 0.081785, + "nauc_recall_at_100_max": 0.019116, + "nauc_recall_at_100_std": 0.037519, + "nauc_recall_at_100_diff1": 0.075557, + "nauc_recall_at_1000_max": -0.131498, + "nauc_recall_at_1000_std": -0.13818, + "nauc_recall_at_1000_diff1": 0.101005, + "nauc_precision_at_1_max": 0.096495, + "nauc_precision_at_1_std": -0.047369, + "nauc_precision_at_1_diff1": 0.248716, + "nauc_precision_at_3_max": 0.057306, + "nauc_precision_at_3_std": -0.068652, + "nauc_precision_at_3_diff1": 0.172574, + "nauc_precision_at_5_max": 0.019497, + "nauc_precision_at_5_std": -0.054058, + "nauc_precision_at_5_diff1": 0.091523, + "nauc_precision_at_10_max": 0.012456, + "nauc_precision_at_10_std": -0.084508, + "nauc_precision_at_10_diff1": 0.106862, + "nauc_precision_at_20_max": 0.040711, + "nauc_precision_at_20_std": -0.049487, + "nauc_precision_at_20_diff1": 0.103901, + "nauc_precision_at_100_max": 0.009034, + "nauc_precision_at_100_std": -0.04984, + "nauc_precision_at_100_diff1": 0.103042, + "nauc_precision_at_1000_max": 0.048383, + "nauc_precision_at_1000_std": -0.006177, + "nauc_precision_at_1000_diff1": 0.051346, + "nauc_mrr_at_1_max": 0.096495, + "nauc_mrr_at_1_std": -0.047369, + "nauc_mrr_at_1_diff1": 0.248716, + "nauc_mrr_at_3_max": 0.063856, + "nauc_mrr_at_3_std": -0.086856, + "nauc_mrr_at_3_diff1": 0.210411, + "nauc_mrr_at_5_max": 0.05609, + "nauc_mrr_at_5_std": -0.086234, + "nauc_mrr_at_5_diff1": 0.206138, + "nauc_mrr_at_10_max": 0.057392, + "nauc_mrr_at_10_std": -0.087161, + "nauc_mrr_at_10_diff1": 0.208164, + "nauc_mrr_at_20_max": 0.057958, + "nauc_mrr_at_20_std": -0.084048, + "nauc_mrr_at_20_diff1": 0.210322, + "nauc_mrr_at_100_max": 0.058279, + "nauc_mrr_at_100_std": -0.08229, + "nauc_mrr_at_100_diff1": 0.210473, + "nauc_mrr_at_1000_max": 0.058031, + "nauc_mrr_at_1000_std": -0.082731, + "nauc_mrr_at_1000_diff1": 0.210491, + "hit_rate_at_1": 0.21386, + "hit_rate_at_3": 0.38554, + "hit_rate_at_5": 0.46988, + "hit_rate_at_10": 0.56928, + "hit_rate_at_20": 0.62952, + "hit_rate_at_100": 0.82831, + "hit_rate_at_1000": 0.99096, + "main_score": 0.23946, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.28614, + "ndcg_at_3": 0.3047, + "ndcg_at_5": 0.30791, + "ndcg_at_10": 0.33574, + "ndcg_at_20": 0.37328, + "ndcg_at_100": 0.43781, + "ndcg_at_1000": 0.4844, + "map_at_1": 0.11834, + "map_at_3": 0.19051, + "map_at_5": 0.21076, + "map_at_10": 0.23801, + "map_at_20": 0.25656, + "map_at_100": 0.27618, + "map_at_1000": 0.28031, + "recall_at_1": 0.11834, + "recall_at_3": 0.24587, + "recall_at_5": 0.29828, + "recall_at_10": 0.39217, + "recall_at_20": 0.50613, + "recall_at_100": 0.72434, + "recall_at_1000": 0.95279, + "accuracy": 0.11834, + "precision_at_1": 0.30422, + "precision_at_3": 0.24197, + "precision_at_5": 0.18735, + "precision_at_10": 0.13705, + "precision_at_20": 0.09337, + "precision_at_100": 0.0306, + "precision_at_1000": 0.00445, + "mrr_at_1": 0.304217, + "mrr_at_3": 0.402108, + "mrr_at_5": 0.41988, + "mrr_at_10": 0.430903, + "mrr_at_20": 0.437679, + "mrr_at_100": 0.441124, + "mrr_at_1000": 0.441393, + "nauc_ndcg_at_1_max": 0.053952, + "nauc_ndcg_at_1_std": -0.03784, + "nauc_ndcg_at_1_diff1": 0.291565, + "nauc_ndcg_at_3_max": -0.003988, + "nauc_ndcg_at_3_std": -0.060897, + "nauc_ndcg_at_3_diff1": 0.254418, + "nauc_ndcg_at_5_max": -0.006224, + "nauc_ndcg_at_5_std": -0.07007, + "nauc_ndcg_at_5_diff1": 0.242767, + "nauc_ndcg_at_10_max": -0.027824, + "nauc_ndcg_at_10_std": -0.087802, + "nauc_ndcg_at_10_diff1": 0.270568, + "nauc_ndcg_at_20_max": -0.021318, + "nauc_ndcg_at_20_std": -0.082198, + "nauc_ndcg_at_20_diff1": 0.271776, + "nauc_ndcg_at_100_max": -0.023835, + "nauc_ndcg_at_100_std": -0.074209, + "nauc_ndcg_at_100_diff1": 0.275013, + "nauc_ndcg_at_1000_max": -0.000581, + "nauc_ndcg_at_1000_std": -0.06815, + "nauc_ndcg_at_1000_diff1": 0.277525, + "nauc_map_at_1_max": -0.021647, + "nauc_map_at_1_std": -0.004724, + "nauc_map_at_1_diff1": 0.220357, + "nauc_map_at_3_max": -0.05195, + "nauc_map_at_3_std": -0.059054, + "nauc_map_at_3_diff1": 0.241032, + "nauc_map_at_5_max": -0.05448, + "nauc_map_at_5_std": -0.072576, + "nauc_map_at_5_diff1": 0.230228, + "nauc_map_at_10_max": -0.051307, + "nauc_map_at_10_std": -0.085095, + "nauc_map_at_10_diff1": 0.248241, + "nauc_map_at_20_max": -0.04307, + "nauc_map_at_20_std": -0.080344, + "nauc_map_at_20_diff1": 0.250443, + "nauc_map_at_100_max": -0.0387, + "nauc_map_at_100_std": -0.073742, + "nauc_map_at_100_diff1": 0.253054, + "nauc_map_at_1000_max": -0.03796, + "nauc_map_at_1000_std": -0.072603, + "nauc_map_at_1000_diff1": 0.253271, + "nauc_recall_at_1_max": -0.021647, + "nauc_recall_at_1_std": -0.004724, + "nauc_recall_at_1_diff1": 0.220357, + "nauc_recall_at_3_max": -0.059864, + "nauc_recall_at_3_std": -0.063124, + "nauc_recall_at_3_diff1": 0.194485, + "nauc_recall_at_5_max": -0.049383, + "nauc_recall_at_5_std": -0.068702, + "nauc_recall_at_5_diff1": 0.154012, + "nauc_recall_at_10_max": -0.047055, + "nauc_recall_at_10_std": -0.081138, + "nauc_recall_at_10_diff1": 0.210368, + "nauc_recall_at_20_max": -0.032299, + "nauc_recall_at_20_std": -0.055281, + "nauc_recall_at_20_diff1": 0.184283, + "nauc_recall_at_100_max": -0.097249, + "nauc_recall_at_100_std": -0.046293, + "nauc_recall_at_100_diff1": 0.197719, + "nauc_recall_at_1000_max": -0.058027, + "nauc_recall_at_1000_std": 0.069535, + "nauc_recall_at_1000_diff1": 0.172387, + "nauc_precision_at_1_max": 0.078695, + "nauc_precision_at_1_std": -0.026008, + "nauc_precision_at_1_diff1": 0.291883, + "nauc_precision_at_3_max": 0.021285, + "nauc_precision_at_3_std": -0.086366, + "nauc_precision_at_3_diff1": 0.261421, + "nauc_precision_at_5_max": 0.028668, + "nauc_precision_at_5_std": -0.093324, + "nauc_precision_at_5_diff1": 0.221693, + "nauc_precision_at_10_max": 0.021929, + "nauc_precision_at_10_std": -0.108622, + "nauc_precision_at_10_diff1": 0.217324, + "nauc_precision_at_20_max": 0.041938, + "nauc_precision_at_20_std": -0.059321, + "nauc_precision_at_20_diff1": 0.166193, + "nauc_precision_at_100_max": 0.055631, + "nauc_precision_at_100_std": 0.023155, + "nauc_precision_at_100_diff1": 0.066271, + "nauc_precision_at_1000_max": 0.118921, + "nauc_precision_at_1000_std": 0.06124, + "nauc_precision_at_1000_diff1": -0.002559, + "nauc_mrr_at_1_max": 0.078695, + "nauc_mrr_at_1_std": -0.026008, + "nauc_mrr_at_1_diff1": 0.291883, + "nauc_mrr_at_3_max": 0.070212, + "nauc_mrr_at_3_std": -0.03861, + "nauc_mrr_at_3_diff1": 0.297897, + "nauc_mrr_at_5_max": 0.074438, + "nauc_mrr_at_5_std": -0.041741, + "nauc_mrr_at_5_diff1": 0.301362, + "nauc_mrr_at_10_max": 0.073862, + "nauc_mrr_at_10_std": -0.042294, + "nauc_mrr_at_10_diff1": 0.304077, + "nauc_mrr_at_20_max": 0.070885, + "nauc_mrr_at_20_std": -0.042857, + "nauc_mrr_at_20_diff1": 0.303936, + "nauc_mrr_at_100_max": 0.069957, + "nauc_mrr_at_100_std": -0.043543, + "nauc_mrr_at_100_diff1": 0.30471, + "nauc_mrr_at_1000_max": 0.070285, + "nauc_mrr_at_1000_std": -0.043272, + "nauc_mrr_at_1000_diff1": 0.304571, + "hit_rate_at_1": 0.30422, + "hit_rate_at_3": 0.53012, + "hit_rate_at_5": 0.60843, + "hit_rate_at_10": 0.68976, + "hit_rate_at_20": 0.78614, + "hit_rate_at_100": 0.91867, + "hit_rate_at_1000": 0.98494, + "main_score": 0.33574, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.18072, + "ndcg_at_3": 0.17908, + "ndcg_at_5": 0.18066, + "ndcg_at_10": 0.20004, + "ndcg_at_20": 0.23466, + "ndcg_at_100": 0.29379, + "ndcg_at_1000": 0.36346, + "map_at_1": 0.06373, + "map_at_3": 0.10452, + "map_at_5": 0.11777, + "map_at_10": 0.13278, + "map_at_20": 0.14833, + "map_at_100": 0.16178, + "map_at_1000": 0.16677, + "recall_at_1": 0.06373, + "recall_at_3": 0.13997, + "recall_at_5": 0.17689, + "recall_at_10": 0.23917, + "recall_at_20": 0.34174, + "recall_at_100": 0.54987, + "recall_at_1000": 0.91772, + "accuracy": 0.06373, + "precision_at_1": 0.19578, + "precision_at_3": 0.14458, + "precision_at_5": 0.11084, + "precision_at_10": 0.08434, + "precision_at_20": 0.06446, + "precision_at_100": 0.02343, + "precision_at_1000": 0.00423, + "mrr_at_1": 0.195783, + "mrr_at_3": 0.258534, + "mrr_at_5": 0.270131, + "mrr_at_10": 0.283866, + "mrr_at_20": 0.290646, + "mrr_at_100": 0.294853, + "mrr_at_1000": 0.295859, + "nauc_ndcg_at_1_max": 0.083163, + "nauc_ndcg_at_1_std": 0.046248, + "nauc_ndcg_at_1_diff1": 0.212175, + "nauc_ndcg_at_3_max": 0.052097, + "nauc_ndcg_at_3_std": 0.052165, + "nauc_ndcg_at_3_diff1": 0.163835, + "nauc_ndcg_at_5_max": 0.047396, + "nauc_ndcg_at_5_std": 0.056851, + "nauc_ndcg_at_5_diff1": 0.14862, + "nauc_ndcg_at_10_max": 0.064734, + "nauc_ndcg_at_10_std": 0.065002, + "nauc_ndcg_at_10_diff1": 0.134364, + "nauc_ndcg_at_20_max": 0.081245, + "nauc_ndcg_at_20_std": 0.078832, + "nauc_ndcg_at_20_diff1": 0.136485, + "nauc_ndcg_at_100_max": 0.089345, + "nauc_ndcg_at_100_std": 0.096566, + "nauc_ndcg_at_100_diff1": 0.14246, + "nauc_ndcg_at_1000_max": 0.071671, + "nauc_ndcg_at_1000_std": 0.067695, + "nauc_ndcg_at_1000_diff1": 0.150163, + "nauc_map_at_1_max": 0.056309, + "nauc_map_at_1_std": 0.044599, + "nauc_map_at_1_diff1": 0.195639, + "nauc_map_at_3_max": 0.050339, + "nauc_map_at_3_std": 0.06539, + "nauc_map_at_3_diff1": 0.168033, + "nauc_map_at_5_max": 0.049005, + "nauc_map_at_5_std": 0.066567, + "nauc_map_at_5_diff1": 0.16349, + "nauc_map_at_10_max": 0.063538, + "nauc_map_at_10_std": 0.072082, + "nauc_map_at_10_diff1": 0.161826, + "nauc_map_at_20_max": 0.077421, + "nauc_map_at_20_std": 0.086218, + "nauc_map_at_20_diff1": 0.161121, + "nauc_map_at_100_max": 0.085729, + "nauc_map_at_100_std": 0.094798, + "nauc_map_at_100_diff1": 0.159097, + "nauc_map_at_1000_max": 0.084722, + "nauc_map_at_1000_std": 0.094186, + "nauc_map_at_1000_diff1": 0.158826, + "nauc_recall_at_1_max": 0.056309, + "nauc_recall_at_1_std": 0.044599, + "nauc_recall_at_1_diff1": 0.195639, + "nauc_recall_at_3_max": 0.037801, + "nauc_recall_at_3_std": 0.073854, + "nauc_recall_at_3_diff1": 0.124987, + "nauc_recall_at_5_max": 0.040085, + "nauc_recall_at_5_std": 0.08148, + "nauc_recall_at_5_diff1": 0.092928, + "nauc_recall_at_10_max": 0.08962, + "nauc_recall_at_10_std": 0.102981, + "nauc_recall_at_10_diff1": 0.068608, + "nauc_recall_at_20_max": 0.107243, + "nauc_recall_at_20_std": 0.108573, + "nauc_recall_at_20_diff1": 0.062364, + "nauc_recall_at_100_max": 0.123079, + "nauc_recall_at_100_std": 0.155048, + "nauc_recall_at_100_diff1": 0.088368, + "nauc_recall_at_1000_max": -0.035604, + "nauc_recall_at_1000_std": -0.034128, + "nauc_recall_at_1000_diff1": 0.179142, + "nauc_precision_at_1_max": 0.098651, + "nauc_precision_at_1_std": 0.058238, + "nauc_precision_at_1_diff1": 0.226719, + "nauc_precision_at_3_max": 0.050136, + "nauc_precision_at_3_std": 0.025251, + "nauc_precision_at_3_diff1": 0.164408, + "nauc_precision_at_5_max": 0.044902, + "nauc_precision_at_5_std": 0.029141, + "nauc_precision_at_5_diff1": 0.153453, + "nauc_precision_at_10_max": 0.090421, + "nauc_precision_at_10_std": 0.043938, + "nauc_precision_at_10_diff1": 0.121663, + "nauc_precision_at_20_max": 0.11557, + "nauc_precision_at_20_std": 0.092662, + "nauc_precision_at_20_diff1": 0.097019, + "nauc_precision_at_100_max": 0.113324, + "nauc_precision_at_100_std": 0.096038, + "nauc_precision_at_100_diff1": 0.078269, + "nauc_precision_at_1000_max": 0.042741, + "nauc_precision_at_1000_std": 0.019213, + "nauc_precision_at_1000_diff1": 0.023133, + "nauc_mrr_at_1_max": 0.098651, + "nauc_mrr_at_1_std": 0.058238, + "nauc_mrr_at_1_diff1": 0.226719, + "nauc_mrr_at_3_max": 0.06892, + "nauc_mrr_at_3_std": 0.051307, + "nauc_mrr_at_3_diff1": 0.218384, + "nauc_mrr_at_5_max": 0.064606, + "nauc_mrr_at_5_std": 0.044634, + "nauc_mrr_at_5_diff1": 0.210696, + "nauc_mrr_at_10_max": 0.074235, + "nauc_mrr_at_10_std": 0.046838, + "nauc_mrr_at_10_diff1": 0.208039, + "nauc_mrr_at_20_max": 0.074724, + "nauc_mrr_at_20_std": 0.046947, + "nauc_mrr_at_20_diff1": 0.20724, + "nauc_mrr_at_100_max": 0.07498, + "nauc_mrr_at_100_std": 0.047413, + "nauc_mrr_at_100_diff1": 0.207899, + "nauc_mrr_at_1000_max": 0.074974, + "nauc_mrr_at_1000_std": 0.047315, + "nauc_mrr_at_1000_diff1": 0.20804, + "hit_rate_at_1": 0.19578, + "hit_rate_at_3": 0.34036, + "hit_rate_at_5": 0.39157, + "hit_rate_at_10": 0.49096, + "hit_rate_at_20": 0.59036, + "hit_rate_at_100": 0.77108, + "hit_rate_at_1000": 0.97892, + "main_score": 0.20004, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.21235, + "ndcg_at_3": 0.1924, + "ndcg_at_5": 0.19932, + "ndcg_at_10": 0.22528, + "ndcg_at_20": 0.25367, + "ndcg_at_100": 0.3155, + "ndcg_at_1000": 0.38216, + "map_at_1": 0.0736, + "map_at_3": 0.11349, + "map_at_5": 0.12938, + "map_at_10": 0.14944, + "map_at_20": 0.1622, + "map_at_100": 0.17645, + "map_at_1000": 0.1812, + "recall_at_1": 0.0736, + "recall_at_3": 0.14543, + "recall_at_5": 0.18696, + "recall_at_10": 0.27214, + "recall_at_20": 0.35471, + "recall_at_100": 0.57992, + "recall_at_1000": 0.92802, + "accuracy": 0.0736, + "precision_at_1": 0.22289, + "precision_at_3": 0.15261, + "precision_at_5": 0.12831, + "precision_at_10": 0.09639, + "precision_at_20": 0.06642, + "precision_at_100": 0.02377, + "precision_at_1000": 0.00427, + "mrr_at_1": 0.222892, + "mrr_at_3": 0.283635, + "mrr_at_5": 0.298544, + "mrr_at_10": 0.311053, + "mrr_at_20": 0.316885, + "mrr_at_100": 0.321954, + "mrr_at_1000": 0.32281, + "nauc_ndcg_at_1_max": 0.049997, + "nauc_ndcg_at_1_std": -0.033132, + "nauc_ndcg_at_1_diff1": 0.067474, + "nauc_ndcg_at_3_max": 0.020702, + "nauc_ndcg_at_3_std": -0.063012, + "nauc_ndcg_at_3_diff1": 0.041506, + "nauc_ndcg_at_5_max": 0.025087, + "nauc_ndcg_at_5_std": -0.054355, + "nauc_ndcg_at_5_diff1": 0.019661, + "nauc_ndcg_at_10_max": 0.025263, + "nauc_ndcg_at_10_std": -0.042161, + "nauc_ndcg_at_10_diff1": 0.014946, + "nauc_ndcg_at_20_max": 0.038967, + "nauc_ndcg_at_20_std": -0.023083, + "nauc_ndcg_at_20_diff1": 0.022226, + "nauc_ndcg_at_100_max": 0.031576, + "nauc_ndcg_at_100_std": -0.036054, + "nauc_ndcg_at_100_diff1": 0.026626, + "nauc_ndcg_at_1000_max": 0.013487, + "nauc_ndcg_at_1000_std": -0.046799, + "nauc_ndcg_at_1000_diff1": 0.004888, + "nauc_map_at_1_max": 0.037901, + "nauc_map_at_1_std": -0.00056, + "nauc_map_at_1_diff1": 0.032388, + "nauc_map_at_3_max": 0.044651, + "nauc_map_at_3_std": -0.020302, + "nauc_map_at_3_diff1": 0.026442, + "nauc_map_at_5_max": 0.030066, + "nauc_map_at_5_std": -0.023195, + "nauc_map_at_5_diff1": 0.018184, + "nauc_map_at_10_max": 0.039853, + "nauc_map_at_10_std": -0.022218, + "nauc_map_at_10_diff1": 0.013944, + "nauc_map_at_20_max": 0.050398, + "nauc_map_at_20_std": -0.012735, + "nauc_map_at_20_diff1": 0.024653, + "nauc_map_at_100_max": 0.05071, + "nauc_map_at_100_std": -0.013555, + "nauc_map_at_100_diff1": 0.016883, + "nauc_map_at_1000_max": 0.048396, + "nauc_map_at_1000_std": -0.01416, + "nauc_map_at_1000_diff1": 0.013891, + "nauc_recall_at_1_max": 0.037901, + "nauc_recall_at_1_std": -0.00056, + "nauc_recall_at_1_diff1": 0.032388, + "nauc_recall_at_3_max": 0.070533, + "nauc_recall_at_3_std": -0.014564, + "nauc_recall_at_3_diff1": 0.017891, + "nauc_recall_at_5_max": 0.044697, + "nauc_recall_at_5_std": -0.018199, + "nauc_recall_at_5_diff1": -0.008601, + "nauc_recall_at_10_max": 0.035286, + "nauc_recall_at_10_std": -0.013888, + "nauc_recall_at_10_diff1": -0.00806, + "nauc_recall_at_20_max": 0.059783, + "nauc_recall_at_20_std": 0.024805, + "nauc_recall_at_20_diff1": 0.003635, + "nauc_recall_at_100_max": 0.046968, + "nauc_recall_at_100_std": -0.006112, + "nauc_recall_at_100_diff1": 0.053924, + "nauc_recall_at_1000_max": -0.227907, + "nauc_recall_at_1000_std": -0.152361, + "nauc_recall_at_1000_diff1": -0.063893, + "nauc_precision_at_1_max": 0.055631, + "nauc_precision_at_1_std": -0.035084, + "nauc_precision_at_1_diff1": 0.052521, + "nauc_precision_at_3_max": -0.013282, + "nauc_precision_at_3_std": -0.115098, + "nauc_precision_at_3_diff1": 0.057674, + "nauc_precision_at_5_max": -0.033799, + "nauc_precision_at_5_std": -0.106433, + "nauc_precision_at_5_diff1": -0.001211, + "nauc_precision_at_10_max": 0.00739, + "nauc_precision_at_10_std": -0.065951, + "nauc_precision_at_10_diff1": -0.029525, + "nauc_precision_at_20_max": 0.030611, + "nauc_precision_at_20_std": -0.019022, + "nauc_precision_at_20_diff1": -0.019264, + "nauc_precision_at_100_max": -0.01464, + "nauc_precision_at_100_std": -0.04358, + "nauc_precision_at_100_diff1": -0.062502, + "nauc_precision_at_1000_max": -0.032637, + "nauc_precision_at_1000_std": -0.009472, + "nauc_precision_at_1000_diff1": -0.112456, + "nauc_mrr_at_1_max": 0.055631, + "nauc_mrr_at_1_std": -0.035084, + "nauc_mrr_at_1_diff1": 0.052521, + "nauc_mrr_at_3_max": 0.041753, + "nauc_mrr_at_3_std": -0.07052, + "nauc_mrr_at_3_diff1": 0.066186, + "nauc_mrr_at_5_max": 0.033492, + "nauc_mrr_at_5_std": -0.075348, + "nauc_mrr_at_5_diff1": 0.046922, + "nauc_mrr_at_10_max": 0.027682, + "nauc_mrr_at_10_std": -0.077032, + "nauc_mrr_at_10_diff1": 0.045726, + "nauc_mrr_at_20_max": 0.028494, + "nauc_mrr_at_20_std": -0.07319, + "nauc_mrr_at_20_diff1": 0.046925, + "nauc_mrr_at_100_max": 0.029773, + "nauc_mrr_at_100_std": -0.072819, + "nauc_mrr_at_100_diff1": 0.050099, + "nauc_mrr_at_1000_max": 0.029695, + "nauc_mrr_at_1000_std": -0.073049, + "nauc_mrr_at_1000_diff1": 0.049925, + "hit_rate_at_1": 0.22289, + "hit_rate_at_3": 0.36747, + "hit_rate_at_5": 0.43373, + "hit_rate_at_10": 0.53012, + "hit_rate_at_20": 0.61446, + "hit_rate_at_100": 0.8012, + "hit_rate_at_1000": 0.98494, + "main_score": 0.22528, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.20181, + "ndcg_at_3": 0.19634, + "ndcg_at_5": 0.20738, + "ndcg_at_10": 0.2337, + "ndcg_at_20": 0.26124, + "ndcg_at_100": 0.32703, + "ndcg_at_1000": 0.38972, + "map_at_1": 0.07338, + "map_at_3": 0.11487, + "map_at_5": 0.13426, + "map_at_10": 0.15388, + "map_at_20": 0.16671, + "map_at_100": 0.18227, + "map_at_1000": 0.18693, + "recall_at_1": 0.07338, + "recall_at_3": 0.14927, + "recall_at_5": 0.20084, + "recall_at_10": 0.28415, + "recall_at_20": 0.36409, + "recall_at_100": 0.60518, + "recall_at_1000": 0.93015, + "accuracy": 0.07338, + "precision_at_1": 0.21988, + "precision_at_3": 0.15863, + "precision_at_5": 0.13373, + "precision_at_10": 0.10271, + "precision_at_20": 0.07063, + "precision_at_100": 0.02512, + "precision_at_1000": 0.0043, + "mrr_at_1": 0.21988, + "mrr_at_3": 0.290161, + "mrr_at_5": 0.306275, + "mrr_at_10": 0.321841, + "mrr_at_20": 0.327135, + "mrr_at_100": 0.331732, + "mrr_at_1000": 0.332452, + "nauc_ndcg_at_1_max": 0.028958, + "nauc_ndcg_at_1_std": -0.05948, + "nauc_ndcg_at_1_diff1": 0.232326, + "nauc_ndcg_at_3_max": -0.002836, + "nauc_ndcg_at_3_std": -0.076152, + "nauc_ndcg_at_3_diff1": 0.224239, + "nauc_ndcg_at_5_max": 0.010696, + "nauc_ndcg_at_5_std": -0.066345, + "nauc_ndcg_at_5_diff1": 0.220768, + "nauc_ndcg_at_10_max": 0.018452, + "nauc_ndcg_at_10_std": -0.032807, + "nauc_ndcg_at_10_diff1": 0.178224, + "nauc_ndcg_at_20_max": 0.020992, + "nauc_ndcg_at_20_std": -0.042759, + "nauc_ndcg_at_20_diff1": 0.186614, + "nauc_ndcg_at_100_max": 0.000558, + "nauc_ndcg_at_100_std": -0.047794, + "nauc_ndcg_at_100_diff1": 0.180028, + "nauc_ndcg_at_1000_max": 0.00927, + "nauc_ndcg_at_1000_std": -0.053376, + "nauc_ndcg_at_1000_diff1": 0.199467, + "nauc_map_at_1_max": 0.101237, + "nauc_map_at_1_std": -0.011792, + "nauc_map_at_1_diff1": 0.211083, + "nauc_map_at_3_max": 0.044137, + "nauc_map_at_3_std": -0.035092, + "nauc_map_at_3_diff1": 0.177102, + "nauc_map_at_5_max": 0.031369, + "nauc_map_at_5_std": -0.034367, + "nauc_map_at_5_diff1": 0.17025, + "nauc_map_at_10_max": 0.034032, + "nauc_map_at_10_std": -0.023279, + "nauc_map_at_10_diff1": 0.170743, + "nauc_map_at_20_max": 0.03906, + "nauc_map_at_20_std": -0.021932, + "nauc_map_at_20_diff1": 0.174904, + "nauc_map_at_100_max": 0.036416, + "nauc_map_at_100_std": -0.020143, + "nauc_map_at_100_diff1": 0.171812, + "nauc_map_at_1000_max": 0.035701, + "nauc_map_at_1000_std": -0.02108, + "nauc_map_at_1000_diff1": 0.172963, + "nauc_recall_at_1_max": 0.101237, + "nauc_recall_at_1_std": -0.011792, + "nauc_recall_at_1_diff1": 0.211083, + "nauc_recall_at_3_max": 0.031831, + "nauc_recall_at_3_std": -0.031737, + "nauc_recall_at_3_diff1": 0.178008, + "nauc_recall_at_5_max": 0.026607, + "nauc_recall_at_5_std": -0.02925, + "nauc_recall_at_5_diff1": 0.152662, + "nauc_recall_at_10_max": 0.052626, + "nauc_recall_at_10_std": 0.038632, + "nauc_recall_at_10_diff1": 0.091056, + "nauc_recall_at_20_max": 0.057911, + "nauc_recall_at_20_std": -0.004755, + "nauc_recall_at_20_diff1": 0.115221, + "nauc_recall_at_100_max": -0.018102, + "nauc_recall_at_100_std": -0.005925, + "nauc_recall_at_100_diff1": 0.068361, + "nauc_recall_at_1000_max": -0.083256, + "nauc_recall_at_1000_std": -0.063231, + "nauc_recall_at_1000_diff1": 0.174143, + "nauc_precision_at_1_max": 0.030487, + "nauc_precision_at_1_std": -0.058888, + "nauc_precision_at_1_diff1": 0.218118, + "nauc_precision_at_3_max": -0.048733, + "nauc_precision_at_3_std": -0.114804, + "nauc_precision_at_3_diff1": 0.199982, + "nauc_precision_at_5_max": -0.049835, + "nauc_precision_at_5_std": -0.102015, + "nauc_precision_at_5_diff1": 0.221432, + "nauc_precision_at_10_max": -0.025786, + "nauc_precision_at_10_std": -0.065345, + "nauc_precision_at_10_diff1": 0.145729, + "nauc_precision_at_20_max": -0.020909, + "nauc_precision_at_20_std": -0.053959, + "nauc_precision_at_20_diff1": 0.147345, + "nauc_precision_at_100_max": -0.057075, + "nauc_precision_at_100_std": -0.058399, + "nauc_precision_at_100_diff1": 0.102935, + "nauc_precision_at_1000_max": 0.010766, + "nauc_precision_at_1000_std": -0.007946, + "nauc_precision_at_1000_diff1": 0.048348, + "nauc_mrr_at_1_max": 0.030487, + "nauc_mrr_at_1_std": -0.058888, + "nauc_mrr_at_1_diff1": 0.218118, + "nauc_mrr_at_3_max": 0.005212, + "nauc_mrr_at_3_std": -0.088919, + "nauc_mrr_at_3_diff1": 0.23814, + "nauc_mrr_at_5_max": 0.010022, + "nauc_mrr_at_5_std": -0.089172, + "nauc_mrr_at_5_diff1": 0.243489, + "nauc_mrr_at_10_max": 0.014265, + "nauc_mrr_at_10_std": -0.07703, + "nauc_mrr_at_10_diff1": 0.227687, + "nauc_mrr_at_20_max": 0.01292, + "nauc_mrr_at_20_std": -0.082092, + "nauc_mrr_at_20_diff1": 0.230441, + "nauc_mrr_at_100_max": 0.011962, + "nauc_mrr_at_100_std": -0.081089, + "nauc_mrr_at_100_diff1": 0.229722, + "nauc_mrr_at_1000_max": 0.012122, + "nauc_mrr_at_1000_std": -0.081025, + "nauc_mrr_at_1000_diff1": 0.229926, + "hit_rate_at_1": 0.21988, + "hit_rate_at_3": 0.37651, + "hit_rate_at_5": 0.44578, + "hit_rate_at_10": 0.55723, + "hit_rate_at_20": 0.63554, + "hit_rate_at_100": 0.81928, + "hit_rate_at_1000": 0.98494, + "main_score": 0.2337, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.10" + } + ] + }, + "evaluation_time": 1698.2292834989494, + "kg_co2_emissions": null, + "date": 1792351674.916691, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 1.4551100321114063e-05, + "end": 25.007833581999876, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 25.00787647604011, + "end": 25.007881424040534, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries", + "start": 25.033121976070106, + "end": 26.864456848008558, + "split": "test", + "subset": "french" + }, + { + "name": "Searching corpus", + "start": 26.866583296097815, + "end": 294.66711337002926, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 294.7101648780517, + "end": 294.9146883180365, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 294.9493856030749, + "end": 295.4103723861044, + "split": "test", + "subset": "spanish" + }, + { + "name": "Searching corpus", + "start": 295.41226710705087, + "end": 570.9157997360453, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 570.9594172060024, + "end": 571.1766668290365, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 571.2183055820642, + "end": 571.8868133170763, + "split": "test", + "subset": "english" + }, + { + "name": "Searching corpus", + "start": 571.8892289680662, + "end": 835.1322023960529, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 835.1779615971027, + "end": 835.3949254130712, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 835.4332444100874, + "end": 835.7448901090538, + "split": "test", + "subset": "german" + }, + { + "name": "Searching corpus", + "start": 835.74667327106, + "end": 1123.9970308491029, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 1124.0438165710075, + "end": 1124.2612541459966, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 1124.3011594620766, + "end": 1124.6634682590375, + "split": "test", + "subset": "italian" + }, + { + "name": "Searching corpus", + "start": 1124.665445423103, + "end": 1410.9623042860767, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 1411.0174697760958, + "end": 1411.221301409998, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 1411.2593341920292, + "end": 1411.5434246880468, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Searching corpus", + "start": 1411.545218798099, + "end": 1697.9712389570195, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 1698.0149511540076, + "end": 1698.2154415460536, + "split": "test", + "subset": "portuguese" + } + ] +} \ No newline at end of file diff --git a/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3TelecomRetrieval.json b/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3TelecomRetrieval.json new file mode 100644 index 0000000000..fb7093e92d --- /dev/null +++ b/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3TelecomRetrieval.json @@ -0,0 +1,1090 @@ +{ + "dataset_revision": "a54635a274ef2835721b7cbe3eb27483b9ec964b", + "task_name": "Vidore3TelecomRetrieval", + "mteb_version": "2.26.10", + "scores": { + "test": [ + { + "ndcg_at_1": 0.38218, + "ndcg_at_3": 0.37262, + "ndcg_at_5": 0.38456, + "ndcg_at_10": 0.41747, + "ndcg_at_20": 0.45513, + "ndcg_at_100": 0.51558, + "ndcg_at_1000": 0.54656, + "map_at_1": 0.15211, + "map_at_3": 0.24296, + "map_at_5": 0.27418, + "map_at_10": 0.3058, + "map_at_20": 0.32864, + "map_at_100": 0.34739, + "map_at_1000": 0.35059, + "recall_at_1": 0.15211, + "recall_at_3": 0.30076, + "recall_at_5": 0.37681, + "recall_at_10": 0.48286, + "recall_at_20": 0.59809, + "recall_at_100": 0.80837, + "recall_at_1000": 0.95583, + "accuracy": 0.15211, + "precision_at_1": 0.4023, + "precision_at_3": 0.28831, + "precision_at_5": 0.22989, + "precision_at_10": 0.16552, + "precision_at_20": 0.11063, + "precision_at_100": 0.03348, + "precision_at_1000": 0.00427, + "mrr_at_1": 0.402299, + "mrr_at_3": 0.493295, + "mrr_at_5": 0.510824, + "mrr_at_10": 0.524155, + "mrr_at_20": 0.529429, + "mrr_at_100": 0.532205, + "mrr_at_1000": 0.532417, + "nauc_ndcg_at_1_max": 0.225103, + "nauc_ndcg_at_1_std": 0.078337, + "nauc_ndcg_at_1_diff1": 0.403308, + "nauc_ndcg_at_3_max": 0.217585, + "nauc_ndcg_at_3_std": 0.127796, + "nauc_ndcg_at_3_diff1": 0.340632, + "nauc_ndcg_at_5_max": 0.21746, + "nauc_ndcg_at_5_std": 0.122035, + "nauc_ndcg_at_5_diff1": 0.325516, + "nauc_ndcg_at_10_max": 0.225105, + "nauc_ndcg_at_10_std": 0.128208, + "nauc_ndcg_at_10_diff1": 0.329436, + "nauc_ndcg_at_20_max": 0.250001, + "nauc_ndcg_at_20_std": 0.156131, + "nauc_ndcg_at_20_diff1": 0.321219, + "nauc_ndcg_at_100_max": 0.270634, + "nauc_ndcg_at_100_std": 0.204798, + "nauc_ndcg_at_100_diff1": 0.306254, + "nauc_ndcg_at_1000_max": 0.261126, + "nauc_ndcg_at_1000_std": 0.183189, + "nauc_ndcg_at_1000_diff1": 0.310496, + "nauc_map_at_1_max": 0.153949, + "nauc_map_at_1_std": -0.010126, + "nauc_map_at_1_diff1": 0.39918, + "nauc_map_at_3_max": 0.156981, + "nauc_map_at_3_std": 0.041338, + "nauc_map_at_3_diff1": 0.333757, + "nauc_map_at_5_max": 0.174423, + "nauc_map_at_5_std": 0.054667, + "nauc_map_at_5_diff1": 0.32112, + "nauc_map_at_10_max": 0.190171, + "nauc_map_at_10_std": 0.06734, + "nauc_map_at_10_diff1": 0.323652, + "nauc_map_at_20_max": 0.211928, + "nauc_map_at_20_std": 0.094532, + "nauc_map_at_20_diff1": 0.314707, + "nauc_map_at_100_max": 0.225549, + "nauc_map_at_100_std": 0.117417, + "nauc_map_at_100_diff1": 0.309108, + "nauc_map_at_1000_max": 0.225061, + "nauc_map_at_1000_std": 0.116469, + "nauc_map_at_1000_diff1": 0.309496, + "nauc_recall_at_1_max": 0.153949, + "nauc_recall_at_1_std": -0.010126, + "nauc_recall_at_1_diff1": 0.39918, + "nauc_recall_at_3_max": 0.131607, + "nauc_recall_at_3_std": 0.081154, + "nauc_recall_at_3_diff1": 0.259071, + "nauc_recall_at_5_max": 0.134641, + "nauc_recall_at_5_std": 0.078972, + "nauc_recall_at_5_diff1": 0.219899, + "nauc_recall_at_10_max": 0.170736, + "nauc_recall_at_10_std": 0.100877, + "nauc_recall_at_10_diff1": 0.23755, + "nauc_recall_at_20_max": 0.234446, + "nauc_recall_at_20_std": 0.176654, + "nauc_recall_at_20_diff1": 0.214609, + "nauc_recall_at_100_max": 0.319512, + "nauc_recall_at_100_std": 0.452193, + "nauc_recall_at_100_diff1": 0.131685, + "nauc_recall_at_1000_max": 0.444833, + "nauc_recall_at_1000_std": 0.703083, + "nauc_recall_at_1000_diff1": -0.051465, + "nauc_precision_at_1_max": 0.215721, + "nauc_precision_at_1_std": 0.086682, + "nauc_precision_at_1_diff1": 0.406859, + "nauc_precision_at_3_max": 0.217624, + "nauc_precision_at_3_std": 0.173739, + "nauc_precision_at_3_diff1": 0.202493, + "nauc_precision_at_5_max": 0.221158, + "nauc_precision_at_5_std": 0.203271, + "nauc_precision_at_5_diff1": 0.128978, + "nauc_precision_at_10_max": 0.21339, + "nauc_precision_at_10_std": 0.217244, + "nauc_precision_at_10_diff1": 0.067978, + "nauc_precision_at_20_max": 0.230967, + "nauc_precision_at_20_std": 0.25416, + "nauc_precision_at_20_diff1": -0.002426, + "nauc_precision_at_100_max": 0.187513, + "nauc_precision_at_100_std": 0.252091, + "nauc_precision_at_100_diff1": -0.064792, + "nauc_precision_at_1000_max": 0.079716, + "nauc_precision_at_1000_std": 0.125394, + "nauc_precision_at_1000_diff1": -0.075105, + "nauc_mrr_at_1_max": 0.215721, + "nauc_mrr_at_1_std": 0.086682, + "nauc_mrr_at_1_diff1": 0.406859, + "nauc_mrr_at_3_max": 0.208689, + "nauc_mrr_at_3_std": 0.127655, + "nauc_mrr_at_3_diff1": 0.360254, + "nauc_mrr_at_5_max": 0.213966, + "nauc_mrr_at_5_std": 0.12899, + "nauc_mrr_at_5_diff1": 0.35729, + "nauc_mrr_at_10_max": 0.219236, + "nauc_mrr_at_10_std": 0.136756, + "nauc_mrr_at_10_diff1": 0.361259, + "nauc_mrr_at_20_max": 0.219675, + "nauc_mrr_at_20_std": 0.132858, + "nauc_mrr_at_20_diff1": 0.362356, + "nauc_mrr_at_100_max": 0.219102, + "nauc_mrr_at_100_std": 0.133899, + "nauc_mrr_at_100_diff1": 0.361859, + "nauc_mrr_at_1000_max": 0.219113, + "nauc_mrr_at_1000_std": 0.133525, + "nauc_mrr_at_1000_diff1": 0.362024, + "hit_rate_at_1": 0.4023, + "hit_rate_at_3": 0.61207, + "hit_rate_at_5": 0.68966, + "hit_rate_at_10": 0.78161, + "hit_rate_at_20": 0.85632, + "hit_rate_at_100": 0.94828, + "hit_rate_at_1000": 0.99138, + "main_score": 0.41747, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.35776, + "ndcg_at_3": 0.35269, + "ndcg_at_5": 0.367, + "ndcg_at_10": 0.40095, + "ndcg_at_20": 0.43871, + "ndcg_at_100": 0.49986, + "ndcg_at_1000": 0.53271, + "map_at_1": 0.14691, + "map_at_3": 0.22787, + "map_at_5": 0.2598, + "map_at_10": 0.29301, + "map_at_20": 0.31432, + "map_at_100": 0.33293, + "map_at_1000": 0.33612, + "recall_at_1": 0.14691, + "recall_at_3": 0.273, + "recall_at_5": 0.35284, + "recall_at_10": 0.46533, + "recall_at_20": 0.58409, + "recall_at_100": 0.7978, + "recall_at_1000": 0.95916, + "accuracy": 0.14691, + "precision_at_1": 0.38218, + "precision_at_3": 0.27586, + "precision_at_5": 0.22701, + "precision_at_10": 0.16092, + "precision_at_20": 0.10675, + "precision_at_100": 0.03307, + "precision_at_1000": 0.00427, + "mrr_at_1": 0.382184, + "mrr_at_3": 0.470785, + "mrr_at_5": 0.491762, + "mrr_at_10": 0.505955, + "mrr_at_20": 0.511249, + "mrr_at_100": 0.513936, + "mrr_at_1000": 0.514168, + "nauc_ndcg_at_1_max": 0.181277, + "nauc_ndcg_at_1_std": 0.06426, + "nauc_ndcg_at_1_diff1": 0.406651, + "nauc_ndcg_at_3_max": 0.18945, + "nauc_ndcg_at_3_std": 0.062416, + "nauc_ndcg_at_3_diff1": 0.347949, + "nauc_ndcg_at_5_max": 0.188436, + "nauc_ndcg_at_5_std": 0.059503, + "nauc_ndcg_at_5_diff1": 0.348788, + "nauc_ndcg_at_10_max": 0.185003, + "nauc_ndcg_at_10_std": 0.080019, + "nauc_ndcg_at_10_diff1": 0.321732, + "nauc_ndcg_at_20_max": 0.18572, + "nauc_ndcg_at_20_std": 0.107034, + "nauc_ndcg_at_20_diff1": 0.312933, + "nauc_ndcg_at_100_max": 0.239067, + "nauc_ndcg_at_100_std": 0.168664, + "nauc_ndcg_at_100_diff1": 0.31694, + "nauc_ndcg_at_1000_max": 0.228446, + "nauc_ndcg_at_1000_std": 0.147418, + "nauc_ndcg_at_1000_diff1": 0.322461, + "nauc_map_at_1_max": 0.076531, + "nauc_map_at_1_std": -0.035729, + "nauc_map_at_1_diff1": 0.416313, + "nauc_map_at_3_max": 0.104978, + "nauc_map_at_3_std": -0.014267, + "nauc_map_at_3_diff1": 0.343576, + "nauc_map_at_5_max": 0.120119, + "nauc_map_at_5_std": -0.007277, + "nauc_map_at_5_diff1": 0.333509, + "nauc_map_at_10_max": 0.140921, + "nauc_map_at_10_std": 0.017819, + "nauc_map_at_10_diff1": 0.311189, + "nauc_map_at_20_max": 0.151999, + "nauc_map_at_20_std": 0.041242, + "nauc_map_at_20_diff1": 0.3101, + "nauc_map_at_100_max": 0.170501, + "nauc_map_at_100_std": 0.066701, + "nauc_map_at_100_diff1": 0.308892, + "nauc_map_at_1000_max": 0.170884, + "nauc_map_at_1000_std": 0.065892, + "nauc_map_at_1000_diff1": 0.309454, + "nauc_recall_at_1_max": 0.076531, + "nauc_recall_at_1_std": -0.035729, + "nauc_recall_at_1_diff1": 0.416313, + "nauc_recall_at_3_max": 0.106172, + "nauc_recall_at_3_std": -0.000154, + "nauc_recall_at_3_diff1": 0.29792, + "nauc_recall_at_5_max": 0.101572, + "nauc_recall_at_5_std": -0.00395, + "nauc_recall_at_5_diff1": 0.248363, + "nauc_recall_at_10_max": 0.128712, + "nauc_recall_at_10_std": 0.053649, + "nauc_recall_at_10_diff1": 0.205621, + "nauc_recall_at_20_max": 0.132838, + "nauc_recall_at_20_std": 0.127801, + "nauc_recall_at_20_diff1": 0.173294, + "nauc_recall_at_100_max": 0.388761, + "nauc_recall_at_100_std": 0.432229, + "nauc_recall_at_100_diff1": 0.184201, + "nauc_recall_at_1000_max": 0.594477, + "nauc_recall_at_1000_std": 0.77165, + "nauc_recall_at_1000_diff1": 0.230348, + "nauc_precision_at_1_max": 0.188689, + "nauc_precision_at_1_std": 0.053364, + "nauc_precision_at_1_diff1": 0.397353, + "nauc_precision_at_3_max": 0.213329, + "nauc_precision_at_3_std": 0.101814, + "nauc_precision_at_3_diff1": 0.18937, + "nauc_precision_at_5_max": 0.234787, + "nauc_precision_at_5_std": 0.139608, + "nauc_precision_at_5_diff1": 0.147217, + "nauc_precision_at_10_max": 0.259507, + "nauc_precision_at_10_std": 0.209911, + "nauc_precision_at_10_diff1": 0.065297, + "nauc_precision_at_20_max": 0.236089, + "nauc_precision_at_20_std": 0.24426, + "nauc_precision_at_20_diff1": 0.031496, + "nauc_precision_at_100_max": 0.219148, + "nauc_precision_at_100_std": 0.271716, + "nauc_precision_at_100_diff1": -0.016952, + "nauc_precision_at_1000_max": 0.1336, + "nauc_precision_at_1000_std": 0.143158, + "nauc_precision_at_1000_diff1": -0.046516, + "nauc_mrr_at_1_max": 0.188689, + "nauc_mrr_at_1_std": 0.053364, + "nauc_mrr_at_1_diff1": 0.397353, + "nauc_mrr_at_3_max": 0.220919, + "nauc_mrr_at_3_std": 0.091016, + "nauc_mrr_at_3_diff1": 0.37093, + "nauc_mrr_at_5_max": 0.219163, + "nauc_mrr_at_5_std": 0.102228, + "nauc_mrr_at_5_diff1": 0.3549, + "nauc_mrr_at_10_max": 0.219626, + "nauc_mrr_at_10_std": 0.108289, + "nauc_mrr_at_10_diff1": 0.358241, + "nauc_mrr_at_20_max": 0.218906, + "nauc_mrr_at_20_std": 0.107793, + "nauc_mrr_at_20_diff1": 0.358211, + "nauc_mrr_at_100_max": 0.220747, + "nauc_mrr_at_100_std": 0.107594, + "nauc_mrr_at_100_diff1": 0.358911, + "nauc_mrr_at_1000_max": 0.220517, + "nauc_mrr_at_1000_std": 0.107231, + "nauc_mrr_at_1000_diff1": 0.359088, + "hit_rate_at_1": 0.38218, + "hit_rate_at_3": 0.57184, + "hit_rate_at_5": 0.66667, + "hit_rate_at_10": 0.77299, + "hit_rate_at_20": 0.8477, + "hit_rate_at_100": 0.94253, + "hit_rate_at_1000": 0.99425, + "main_score": 0.40095, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.43678, + "ndcg_at_3": 0.43093, + "ndcg_at_5": 0.45426, + "ndcg_at_10": 0.48616, + "ndcg_at_20": 0.52181, + "ndcg_at_100": 0.57541, + "ndcg_at_1000": 0.59826, + "map_at_1": 0.16783, + "map_at_3": 0.27773, + "map_at_5": 0.32029, + "map_at_10": 0.35892, + "map_at_20": 0.38312, + "map_at_100": 0.40207, + "map_at_1000": 0.40455, + "recall_at_1": 0.16783, + "recall_at_3": 0.34506, + "recall_at_5": 0.44992, + "recall_at_10": 0.56942, + "recall_at_20": 0.67915, + "recall_at_100": 0.85624, + "recall_at_1000": 0.96595, + "accuracy": 0.16783, + "precision_at_1": 0.46264, + "precision_at_3": 0.33908, + "precision_at_5": 0.28103, + "precision_at_10": 0.19425, + "precision_at_20": 0.12601, + "precision_at_100": 0.03592, + "precision_at_1000": 0.0043, + "mrr_at_1": 0.462644, + "mrr_at_3": 0.565134, + "mrr_at_5": 0.584962, + "mrr_at_10": 0.595577, + "mrr_at_20": 0.598275, + "mrr_at_100": 0.600251, + "mrr_at_1000": 0.600379, + "nauc_ndcg_at_1_max": 0.270569, + "nauc_ndcg_at_1_std": -0.052818, + "nauc_ndcg_at_1_diff1": 0.442632, + "nauc_ndcg_at_3_max": 0.238824, + "nauc_ndcg_at_3_std": 0.006638, + "nauc_ndcg_at_3_diff1": 0.348682, + "nauc_ndcg_at_5_max": 0.247091, + "nauc_ndcg_at_5_std": 0.019397, + "nauc_ndcg_at_5_diff1": 0.367217, + "nauc_ndcg_at_10_max": 0.251139, + "nauc_ndcg_at_10_std": 0.025701, + "nauc_ndcg_at_10_diff1": 0.377117, + "nauc_ndcg_at_20_max": 0.262042, + "nauc_ndcg_at_20_std": 0.064832, + "nauc_ndcg_at_20_diff1": 0.36769, + "nauc_ndcg_at_100_max": 0.296769, + "nauc_ndcg_at_100_std": 0.104437, + "nauc_ndcg_at_100_diff1": 0.352836, + "nauc_ndcg_at_1000_max": 0.29404, + "nauc_ndcg_at_1000_std": 0.077244, + "nauc_ndcg_at_1000_diff1": 0.359991, + "nauc_map_at_1_max": 0.174719, + "nauc_map_at_1_std": -0.067205, + "nauc_map_at_1_diff1": 0.471943, + "nauc_map_at_3_max": 0.148508, + "nauc_map_at_3_std": -0.040084, + "nauc_map_at_3_diff1": 0.367088, + "nauc_map_at_5_max": 0.168838, + "nauc_map_at_5_std": -0.031157, + "nauc_map_at_5_diff1": 0.366699, + "nauc_map_at_10_max": 0.202521, + "nauc_map_at_10_std": -0.022334, + "nauc_map_at_10_diff1": 0.356936, + "nauc_map_at_20_max": 0.223067, + "nauc_map_at_20_std": 0.004308, + "nauc_map_at_20_diff1": 0.348895, + "nauc_map_at_100_max": 0.241175, + "nauc_map_at_100_std": 0.024671, + "nauc_map_at_100_diff1": 0.339673, + "nauc_map_at_1000_max": 0.241815, + "nauc_map_at_1000_std": 0.023703, + "nauc_map_at_1000_diff1": 0.340871, + "nauc_recall_at_1_max": 0.174719, + "nauc_recall_at_1_std": -0.067205, + "nauc_recall_at_1_diff1": 0.471943, + "nauc_recall_at_3_max": 0.098657, + "nauc_recall_at_3_std": -0.009416, + "nauc_recall_at_3_diff1": 0.283446, + "nauc_recall_at_5_max": 0.099798, + "nauc_recall_at_5_std": -0.016367, + "nauc_recall_at_5_diff1": 0.286516, + "nauc_recall_at_10_max": 0.15331, + "nauc_recall_at_10_std": 0.012176, + "nauc_recall_at_10_diff1": 0.277752, + "nauc_recall_at_20_max": 0.185303, + "nauc_recall_at_20_std": 0.150577, + "nauc_recall_at_20_diff1": 0.242938, + "nauc_recall_at_100_max": 0.309725, + "nauc_recall_at_100_std": 0.434303, + "nauc_recall_at_100_diff1": 0.185132, + "nauc_recall_at_1000_max": 0.456891, + "nauc_recall_at_1000_std": 0.544436, + "nauc_recall_at_1000_diff1": 0.193711, + "nauc_precision_at_1_max": 0.251721, + "nauc_precision_at_1_std": -0.055597, + "nauc_precision_at_1_diff1": 0.45498, + "nauc_precision_at_3_max": 0.22205, + "nauc_precision_at_3_std": 0.05612, + "nauc_precision_at_3_diff1": 0.170445, + "nauc_precision_at_5_max": 0.247367, + "nauc_precision_at_5_std": 0.106451, + "nauc_precision_at_5_diff1": 0.087404, + "nauc_precision_at_10_max": 0.272319, + "nauc_precision_at_10_std": 0.123674, + "nauc_precision_at_10_diff1": 0.019545, + "nauc_precision_at_20_max": 0.267913, + "nauc_precision_at_20_std": 0.183209, + "nauc_precision_at_20_diff1": -0.042014, + "nauc_precision_at_100_max": 0.220531, + "nauc_precision_at_100_std": 0.198256, + "nauc_precision_at_100_diff1": -0.127695, + "nauc_precision_at_1000_max": 0.142467, + "nauc_precision_at_1000_std": 0.099914, + "nauc_precision_at_1000_diff1": -0.116605, + "nauc_mrr_at_1_max": 0.251721, + "nauc_mrr_at_1_std": -0.055597, + "nauc_mrr_at_1_diff1": 0.45498, + "nauc_mrr_at_3_max": 0.277743, + "nauc_mrr_at_3_std": 0.015058, + "nauc_mrr_at_3_diff1": 0.412213, + "nauc_mrr_at_5_max": 0.279012, + "nauc_mrr_at_5_std": 0.01534, + "nauc_mrr_at_5_diff1": 0.421407, + "nauc_mrr_at_10_max": 0.279952, + "nauc_mrr_at_10_std": 0.017491, + "nauc_mrr_at_10_diff1": 0.422998, + "nauc_mrr_at_20_max": 0.279888, + "nauc_mrr_at_20_std": 0.018011, + "nauc_mrr_at_20_diff1": 0.422361, + "nauc_mrr_at_100_max": 0.280208, + "nauc_mrr_at_100_std": 0.016411, + "nauc_mrr_at_100_diff1": 0.423777, + "nauc_mrr_at_1000_max": 0.280072, + "nauc_mrr_at_1000_std": 0.016015, + "nauc_mrr_at_1000_diff1": 0.423754, + "hit_rate_at_1": 0.46264, + "hit_rate_at_3": 0.70115, + "hit_rate_at_5": 0.79023, + "hit_rate_at_10": 0.86782, + "hit_rate_at_20": 0.90517, + "hit_rate_at_100": 0.96552, + "hit_rate_at_1000": 0.99425, + "main_score": 0.48616, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.36638, + "ndcg_at_3": 0.3524, + "ndcg_at_5": 0.37792, + "ndcg_at_10": 0.41007, + "ndcg_at_20": 0.45112, + "ndcg_at_100": 0.50641, + "ndcg_at_1000": 0.53694, + "map_at_1": 0.14352, + "map_at_3": 0.22887, + "map_at_5": 0.26662, + "map_at_10": 0.30079, + "map_at_20": 0.32555, + "map_at_100": 0.3424, + "map_at_1000": 0.3455, + "recall_at_1": 0.14352, + "recall_at_3": 0.27822, + "recall_at_5": 0.36843, + "recall_at_10": 0.48401, + "recall_at_20": 0.6093, + "recall_at_100": 0.80409, + "recall_at_1000": 0.94872, + "accuracy": 0.14352, + "precision_at_1": 0.38218, + "precision_at_3": 0.27299, + "precision_at_5": 0.23448, + "precision_at_10": 0.16523, + "precision_at_20": 0.11221, + "precision_at_100": 0.0331, + "precision_at_1000": 0.00422, + "mrr_at_1": 0.382184, + "mrr_at_3": 0.463123, + "mrr_at_5": 0.485967, + "mrr_at_10": 0.499571, + "mrr_at_20": 0.504868, + "mrr_at_100": 0.507116, + "mrr_at_1000": 0.507345, + "nauc_ndcg_at_1_max": 0.217194, + "nauc_ndcg_at_1_std": 0.083907, + "nauc_ndcg_at_1_diff1": 0.412766, + "nauc_ndcg_at_3_max": 0.199045, + "nauc_ndcg_at_3_std": 0.086038, + "nauc_ndcg_at_3_diff1": 0.36905, + "nauc_ndcg_at_5_max": 0.204733, + "nauc_ndcg_at_5_std": 0.100119, + "nauc_ndcg_at_5_diff1": 0.366939, + "nauc_ndcg_at_10_max": 0.223038, + "nauc_ndcg_at_10_std": 0.121737, + "nauc_ndcg_at_10_diff1": 0.373897, + "nauc_ndcg_at_20_max": 0.23033, + "nauc_ndcg_at_20_std": 0.145058, + "nauc_ndcg_at_20_diff1": 0.375139, + "nauc_ndcg_at_100_max": 0.265341, + "nauc_ndcg_at_100_std": 0.192784, + "nauc_ndcg_at_100_diff1": 0.376532, + "nauc_ndcg_at_1000_max": 0.257652, + "nauc_ndcg_at_1000_std": 0.174919, + "nauc_ndcg_at_1000_diff1": 0.367438, + "nauc_map_at_1_max": 0.148185, + "nauc_map_at_1_std": 0.01732, + "nauc_map_at_1_diff1": 0.416796, + "nauc_map_at_3_max": 0.123443, + "nauc_map_at_3_std": 0.012745, + "nauc_map_at_3_diff1": 0.364989, + "nauc_map_at_5_max": 0.136764, + "nauc_map_at_5_std": 0.028155, + "nauc_map_at_5_diff1": 0.358099, + "nauc_map_at_10_max": 0.167557, + "nauc_map_at_10_std": 0.056711, + "nauc_map_at_10_diff1": 0.358589, + "nauc_map_at_20_max": 0.176655, + "nauc_map_at_20_std": 0.075603, + "nauc_map_at_20_diff1": 0.358588, + "nauc_map_at_100_max": 0.193058, + "nauc_map_at_100_std": 0.102268, + "nauc_map_at_100_diff1": 0.35225, + "nauc_map_at_1000_max": 0.193612, + "nauc_map_at_1000_std": 0.102632, + "nauc_map_at_1000_diff1": 0.351835, + "nauc_recall_at_1_max": 0.148185, + "nauc_recall_at_1_std": 0.01732, + "nauc_recall_at_1_diff1": 0.416796, + "nauc_recall_at_3_max": 0.119377, + "nauc_recall_at_3_std": 0.030767, + "nauc_recall_at_3_diff1": 0.323367, + "nauc_recall_at_5_max": 0.113305, + "nauc_recall_at_5_std": 0.044134, + "nauc_recall_at_5_diff1": 0.281833, + "nauc_recall_at_10_max": 0.16536, + "nauc_recall_at_10_std": 0.111363, + "nauc_recall_at_10_diff1": 0.290224, + "nauc_recall_at_20_max": 0.216346, + "nauc_recall_at_20_std": 0.19544, + "nauc_recall_at_20_diff1": 0.281835, + "nauc_recall_at_100_max": 0.372454, + "nauc_recall_at_100_std": 0.432194, + "nauc_recall_at_100_diff1": 0.330806, + "nauc_recall_at_1000_max": 0.555235, + "nauc_recall_at_1000_std": 0.631652, + "nauc_recall_at_1000_diff1": 0.208365, + "nauc_precision_at_1_max": 0.212131, + "nauc_precision_at_1_std": 0.082213, + "nauc_precision_at_1_diff1": 0.414674, + "nauc_precision_at_3_max": 0.203344, + "nauc_precision_at_3_std": 0.14601, + "nauc_precision_at_3_diff1": 0.264322, + "nauc_precision_at_5_max": 0.196968, + "nauc_precision_at_5_std": 0.177834, + "nauc_precision_at_5_diff1": 0.203778, + "nauc_precision_at_10_max": 0.235663, + "nauc_precision_at_10_std": 0.217889, + "nauc_precision_at_10_diff1": 0.144541, + "nauc_precision_at_20_max": 0.219876, + "nauc_precision_at_20_std": 0.25334, + "nauc_precision_at_20_diff1": 0.069986, + "nauc_precision_at_100_max": 0.196102, + "nauc_precision_at_100_std": 0.270597, + "nauc_precision_at_100_diff1": -0.023095, + "nauc_precision_at_1000_max": 0.107762, + "nauc_precision_at_1000_std": 0.164423, + "nauc_precision_at_1000_diff1": -0.07533, + "nauc_mrr_at_1_max": 0.212131, + "nauc_mrr_at_1_std": 0.082213, + "nauc_mrr_at_1_diff1": 0.414674, + "nauc_mrr_at_3_max": 0.250577, + "nauc_mrr_at_3_std": 0.142967, + "nauc_mrr_at_3_diff1": 0.391291, + "nauc_mrr_at_5_max": 0.243955, + "nauc_mrr_at_5_std": 0.140803, + "nauc_mrr_at_5_diff1": 0.38502, + "nauc_mrr_at_10_max": 0.240853, + "nauc_mrr_at_10_std": 0.142855, + "nauc_mrr_at_10_diff1": 0.384012, + "nauc_mrr_at_20_max": 0.246779, + "nauc_mrr_at_20_std": 0.146176, + "nauc_mrr_at_20_diff1": 0.382394, + "nauc_mrr_at_100_max": 0.247396, + "nauc_mrr_at_100_std": 0.145208, + "nauc_mrr_at_100_diff1": 0.384523, + "nauc_mrr_at_1000_max": 0.247149, + "nauc_mrr_at_1000_std": 0.144867, + "nauc_mrr_at_1000_diff1": 0.384285, + "hit_rate_at_1": 0.38218, + "hit_rate_at_3": 0.56897, + "hit_rate_at_5": 0.66667, + "hit_rate_at_10": 0.76724, + "hit_rate_at_20": 0.84195, + "hit_rate_at_100": 0.93966, + "hit_rate_at_1000": 0.99138, + "main_score": 0.41007, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.38793, + "ndcg_at_3": 0.36841, + "ndcg_at_5": 0.37687, + "ndcg_at_10": 0.40703, + "ndcg_at_20": 0.44847, + "ndcg_at_100": 0.50975, + "ndcg_at_1000": 0.54054, + "map_at_1": 0.15125, + "map_at_3": 0.24209, + "map_at_5": 0.27242, + "map_at_10": 0.30046, + "map_at_20": 0.32271, + "map_at_100": 0.3415, + "map_at_1000": 0.34454, + "recall_at_1": 0.15125, + "recall_at_3": 0.29158, + "recall_at_5": 0.36143, + "recall_at_10": 0.46514, + "recall_at_20": 0.59301, + "recall_at_100": 0.80448, + "recall_at_1000": 0.95317, + "accuracy": 0.15125, + "precision_at_1": 0.40517, + "precision_at_3": 0.28352, + "precision_at_5": 0.22126, + "precision_at_10": 0.15546, + "precision_at_20": 0.10474, + "precision_at_100": 0.03313, + "precision_at_1000": 0.00422, + "mrr_at_1": 0.405172, + "mrr_at_3": 0.481801, + "mrr_at_5": 0.496312, + "mrr_at_10": 0.513931, + "mrr_at_20": 0.519668, + "mrr_at_100": 0.522526, + "mrr_at_1000": 0.522667, + "nauc_ndcg_at_1_max": 0.201892, + "nauc_ndcg_at_1_std": 0.061003, + "nauc_ndcg_at_1_diff1": 0.354188, + "nauc_ndcg_at_3_max": 0.19439, + "nauc_ndcg_at_3_std": 0.061528, + "nauc_ndcg_at_3_diff1": 0.345589, + "nauc_ndcg_at_5_max": 0.207894, + "nauc_ndcg_at_5_std": 0.074465, + "nauc_ndcg_at_5_diff1": 0.339457, + "nauc_ndcg_at_10_max": 0.222571, + "nauc_ndcg_at_10_std": 0.098387, + "nauc_ndcg_at_10_diff1": 0.351024, + "nauc_ndcg_at_20_max": 0.236571, + "nauc_ndcg_at_20_std": 0.127865, + "nauc_ndcg_at_20_diff1": 0.366733, + "nauc_ndcg_at_100_max": 0.253233, + "nauc_ndcg_at_100_std": 0.179732, + "nauc_ndcg_at_100_diff1": 0.352683, + "nauc_ndcg_at_1000_max": 0.243688, + "nauc_ndcg_at_1000_std": 0.148541, + "nauc_ndcg_at_1000_diff1": 0.353048, + "nauc_map_at_1_max": 0.144967, + "nauc_map_at_1_std": -0.042616, + "nauc_map_at_1_diff1": 0.392125, + "nauc_map_at_3_max": 0.165505, + "nauc_map_at_3_std": -0.011192, + "nauc_map_at_3_diff1": 0.360605, + "nauc_map_at_5_max": 0.185099, + "nauc_map_at_5_std": 0.011466, + "nauc_map_at_5_diff1": 0.345253, + "nauc_map_at_10_max": 0.202025, + "nauc_map_at_10_std": 0.034172, + "nauc_map_at_10_diff1": 0.338702, + "nauc_map_at_20_max": 0.209835, + "nauc_map_at_20_std": 0.055496, + "nauc_map_at_20_diff1": 0.338552, + "nauc_map_at_100_max": 0.217032, + "nauc_map_at_100_std": 0.081731, + "nauc_map_at_100_diff1": 0.334675, + "nauc_map_at_1000_max": 0.216661, + "nauc_map_at_1000_std": 0.079782, + "nauc_map_at_1000_diff1": 0.334578, + "nauc_recall_at_1_max": 0.144967, + "nauc_recall_at_1_std": -0.042616, + "nauc_recall_at_1_diff1": 0.392125, + "nauc_recall_at_3_max": 0.153523, + "nauc_recall_at_3_std": 0.029861, + "nauc_recall_at_3_diff1": 0.326158, + "nauc_recall_at_5_max": 0.165165, + "nauc_recall_at_5_std": 0.067238, + "nauc_recall_at_5_diff1": 0.271752, + "nauc_recall_at_10_max": 0.20277, + "nauc_recall_at_10_std": 0.129952, + "nauc_recall_at_10_diff1": 0.264684, + "nauc_recall_at_20_max": 0.218192, + "nauc_recall_at_20_std": 0.190125, + "nauc_recall_at_20_diff1": 0.285404, + "nauc_recall_at_100_max": 0.342636, + "nauc_recall_at_100_std": 0.501755, + "nauc_recall_at_100_diff1": 0.255166, + "nauc_recall_at_1000_max": 0.532212, + "nauc_recall_at_1000_std": 0.727825, + "nauc_recall_at_1000_diff1": 0.338947, + "nauc_precision_at_1_max": 0.201608, + "nauc_precision_at_1_std": 0.064742, + "nauc_precision_at_1_diff1": 0.3398, + "nauc_precision_at_3_max": 0.198218, + "nauc_precision_at_3_std": 0.101193, + "nauc_precision_at_3_diff1": 0.216996, + "nauc_precision_at_5_max": 0.208157, + "nauc_precision_at_5_std": 0.140078, + "nauc_precision_at_5_diff1": 0.135356, + "nauc_precision_at_10_max": 0.198604, + "nauc_precision_at_10_std": 0.202151, + "nauc_precision_at_10_diff1": 0.078988, + "nauc_precision_at_20_max": 0.16683, + "nauc_precision_at_20_std": 0.243509, + "nauc_precision_at_20_diff1": 0.039568, + "nauc_precision_at_100_max": 0.098461, + "nauc_precision_at_100_std": 0.245867, + "nauc_precision_at_100_diff1": -0.072619, + "nauc_precision_at_1000_max": 0.007304, + "nauc_precision_at_1000_std": 0.089952, + "nauc_precision_at_1000_diff1": -0.119321, + "nauc_mrr_at_1_max": 0.201608, + "nauc_mrr_at_1_std": 0.064742, + "nauc_mrr_at_1_diff1": 0.3398, + "nauc_mrr_at_3_max": 0.206722, + "nauc_mrr_at_3_std": 0.094626, + "nauc_mrr_at_3_diff1": 0.341239, + "nauc_mrr_at_5_max": 0.202161, + "nauc_mrr_at_5_std": 0.099497, + "nauc_mrr_at_5_diff1": 0.327639, + "nauc_mrr_at_10_max": 0.208771, + "nauc_mrr_at_10_std": 0.113402, + "nauc_mrr_at_10_diff1": 0.335221, + "nauc_mrr_at_20_max": 0.208775, + "nauc_mrr_at_20_std": 0.111313, + "nauc_mrr_at_20_diff1": 0.338784, + "nauc_mrr_at_100_max": 0.20809, + "nauc_mrr_at_100_std": 0.110336, + "nauc_mrr_at_100_diff1": 0.335985, + "nauc_mrr_at_1000_max": 0.207996, + "nauc_mrr_at_1000_std": 0.110096, + "nauc_mrr_at_1000_diff1": 0.336002, + "hit_rate_at_1": 0.40517, + "hit_rate_at_3": 0.58046, + "hit_rate_at_5": 0.6408, + "hit_rate_at_10": 0.77011, + "hit_rate_at_20": 0.85345, + "hit_rate_at_100": 0.95115, + "hit_rate_at_1000": 0.98563, + "main_score": 0.40703, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.36925, + "ndcg_at_3": 0.36163, + "ndcg_at_5": 0.37253, + "ndcg_at_10": 0.404, + "ndcg_at_20": 0.44009, + "ndcg_at_100": 0.50149, + "ndcg_at_1000": 0.53474, + "map_at_1": 0.14574, + "map_at_3": 0.23029, + "map_at_5": 0.26176, + "map_at_10": 0.29517, + "map_at_20": 0.31542, + "map_at_100": 0.33482, + "map_at_1000": 0.33795, + "recall_at_1": 0.14574, + "recall_at_3": 0.28491, + "recall_at_5": 0.36026, + "recall_at_10": 0.47092, + "recall_at_20": 0.58077, + "recall_at_100": 0.79494, + "recall_at_1000": 0.95834, + "accuracy": 0.14574, + "precision_at_1": 0.3908, + "precision_at_3": 0.28352, + "precision_at_5": 0.22529, + "precision_at_10": 0.16006, + "precision_at_20": 0.10603, + "precision_at_100": 0.03293, + "precision_at_1000": 0.00425, + "mrr_at_1": 0.390805, + "mrr_at_3": 0.480364, + "mrr_at_5": 0.498755, + "mrr_at_10": 0.510142, + "mrr_at_20": 0.516268, + "mrr_at_100": 0.518452, + "mrr_at_1000": 0.518684, + "nauc_ndcg_at_1_max": 0.260326, + "nauc_ndcg_at_1_std": 0.106994, + "nauc_ndcg_at_1_diff1": 0.351456, + "nauc_ndcg_at_3_max": 0.284279, + "nauc_ndcg_at_3_std": 0.084782, + "nauc_ndcg_at_3_diff1": 0.320627, + "nauc_ndcg_at_5_max": 0.271627, + "nauc_ndcg_at_5_std": 0.065705, + "nauc_ndcg_at_5_diff1": 0.327573, + "nauc_ndcg_at_10_max": 0.277538, + "nauc_ndcg_at_10_std": 0.0956, + "nauc_ndcg_at_10_diff1": 0.302983, + "nauc_ndcg_at_20_max": 0.290252, + "nauc_ndcg_at_20_std": 0.129224, + "nauc_ndcg_at_20_diff1": 0.290825, + "nauc_ndcg_at_100_max": 0.329203, + "nauc_ndcg_at_100_std": 0.180986, + "nauc_ndcg_at_100_diff1": 0.308174, + "nauc_ndcg_at_1000_max": 0.321623, + "nauc_ndcg_at_1000_std": 0.160734, + "nauc_ndcg_at_1000_diff1": 0.315084, + "nauc_map_at_1_max": 0.17878, + "nauc_map_at_1_std": -0.013375, + "nauc_map_at_1_diff1": 0.373441, + "nauc_map_at_3_max": 0.207613, + "nauc_map_at_3_std": -0.015949, + "nauc_map_at_3_diff1": 0.355859, + "nauc_map_at_5_max": 0.217584, + "nauc_map_at_5_std": -0.009511, + "nauc_map_at_5_diff1": 0.343694, + "nauc_map_at_10_max": 0.247869, + "nauc_map_at_10_std": 0.031592, + "nauc_map_at_10_diff1": 0.312185, + "nauc_map_at_20_max": 0.263405, + "nauc_map_at_20_std": 0.053901, + "nauc_map_at_20_diff1": 0.302699, + "nauc_map_at_100_max": 0.280951, + "nauc_map_at_100_std": 0.079928, + "nauc_map_at_100_diff1": 0.304372, + "nauc_map_at_1000_max": 0.281674, + "nauc_map_at_1000_std": 0.079424, + "nauc_map_at_1000_diff1": 0.305516, + "nauc_recall_at_1_max": 0.17878, + "nauc_recall_at_1_std": -0.013375, + "nauc_recall_at_1_diff1": 0.373441, + "nauc_recall_at_3_max": 0.186409, + "nauc_recall_at_3_std": -0.009684, + "nauc_recall_at_3_diff1": 0.281876, + "nauc_recall_at_5_max": 0.166089, + "nauc_recall_at_5_std": -0.017974, + "nauc_recall_at_5_diff1": 0.256283, + "nauc_recall_at_10_max": 0.200124, + "nauc_recall_at_10_std": 0.066269, + "nauc_recall_at_10_diff1": 0.195369, + "nauc_recall_at_20_max": 0.242791, + "nauc_recall_at_20_std": 0.166927, + "nauc_recall_at_20_diff1": 0.143351, + "nauc_recall_at_100_max": 0.402131, + "nauc_recall_at_100_std": 0.39692, + "nauc_recall_at_100_diff1": 0.226236, + "nauc_recall_at_1000_max": 0.601582, + "nauc_recall_at_1000_std": 0.65688, + "nauc_recall_at_1000_diff1": 0.405975, + "nauc_precision_at_1_max": 0.258673, + "nauc_precision_at_1_std": 0.081913, + "nauc_precision_at_1_diff1": 0.345894, + "nauc_precision_at_3_max": 0.291081, + "nauc_precision_at_3_std": 0.113881, + "nauc_precision_at_3_diff1": 0.196008, + "nauc_precision_at_5_max": 0.282727, + "nauc_precision_at_5_std": 0.141892, + "nauc_precision_at_5_diff1": 0.149883, + "nauc_precision_at_10_max": 0.2942, + "nauc_precision_at_10_std": 0.230375, + "nauc_precision_at_10_diff1": 0.037975, + "nauc_precision_at_20_max": 0.275395, + "nauc_precision_at_20_std": 0.26666, + "nauc_precision_at_20_diff1": -0.023672, + "nauc_precision_at_100_max": 0.223124, + "nauc_precision_at_100_std": 0.262549, + "nauc_precision_at_100_diff1": -0.050297, + "nauc_precision_at_1000_max": 0.116837, + "nauc_precision_at_1000_std": 0.13178, + "nauc_precision_at_1000_diff1": -0.067989, + "nauc_mrr_at_1_max": 0.258673, + "nauc_mrr_at_1_std": 0.081913, + "nauc_mrr_at_1_diff1": 0.345894, + "nauc_mrr_at_3_max": 0.277096, + "nauc_mrr_at_3_std": 0.114802, + "nauc_mrr_at_3_diff1": 0.301657, + "nauc_mrr_at_5_max": 0.275553, + "nauc_mrr_at_5_std": 0.121641, + "nauc_mrr_at_5_diff1": 0.305558, + "nauc_mrr_at_10_max": 0.279722, + "nauc_mrr_at_10_std": 0.12688, + "nauc_mrr_at_10_diff1": 0.307225, + "nauc_mrr_at_20_max": 0.280437, + "nauc_mrr_at_20_std": 0.129729, + "nauc_mrr_at_20_diff1": 0.305906, + "nauc_mrr_at_100_max": 0.280196, + "nauc_mrr_at_100_std": 0.127919, + "nauc_mrr_at_100_diff1": 0.30764, + "nauc_mrr_at_1000_max": 0.28013, + "nauc_mrr_at_1000_std": 0.127518, + "nauc_mrr_at_1000_diff1": 0.307554, + "hit_rate_at_1": 0.3908, + "hit_rate_at_3": 0.5977, + "hit_rate_at_5": 0.68103, + "hit_rate_at_10": 0.76724, + "hit_rate_at_20": 0.85632, + "hit_rate_at_100": 0.94253, + "hit_rate_at_1000": 0.99425, + "main_score": 0.404, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.10" + } + ] + }, + "evaluation_time": 2969.72516350809, + "kg_co2_emissions": null, + "date": 1792352913.028473, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 1.0204967111349106e-05, + "end": 26.195120678981766, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 26.19516056892462, + "end": 26.19516315299552, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries", + "start": 26.22251154994592, + "end": 32.230278505943716, + "split": "test", + "subset": "french" + }, + { + "name": "Searching corpus", + "start": 32.232474825927056, + "end": 484.1996223980095, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 484.25454123690724, + "end": 484.52562347392086, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 484.5757337169489, + "end": 485.1991443539737, + "split": "test", + "subset": "spanish" + }, + { + "name": "Searching corpus", + "start": 485.201189477928, + "end": 1017.7219080779469, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 1017.7851741999621, + "end": 1018.0877344179899, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 1018.1258947229944, + "end": 1018.7085352470167, + "split": "test", + "subset": "english" + }, + { + "name": "Searching corpus", + "start": 1018.7105163249653, + "end": 1541.8214359809645, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 1541.8847507219762, + "end": 1542.1415630939882, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 1542.1984334709123, + "end": 1542.7541990589816, + "split": "test", + "subset": "german" + }, + { + "name": "Searching corpus", + "start": 1542.7592190019786, + "end": 2027.9279412149917, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 2027.9884269819595, + "end": 2028.2439303289866, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 2028.298081186018, + "end": 2028.5957644699374, + "split": "test", + "subset": "italian" + }, + { + "name": "Searching corpus", + "start": 2028.5977799189277, + "end": 2495.8387306729564, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 2495.8985046399757, + "end": 2496.1741338359425, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 2496.21529866301, + "end": 2496.543590687914, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Searching corpus", + "start": 2496.5455966729205, + "end": 2969.3864243419375, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 2969.43779139698, + "end": 2969.710834000958, + "split": "test", + "subset": "portuguese" + } + ] +} \ No newline at end of file diff --git a/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3TelecomRetrieval.v2.json b/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3TelecomRetrieval.v2.json new file mode 100644 index 0000000000..a8f9218f7f --- /dev/null +++ b/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/Vidore3TelecomRetrieval.v2.json @@ -0,0 +1,1090 @@ +{ + "dataset_revision": "057fe83342b446ede7928cb51bc6c7c3223da3d0", + "task_name": "Vidore3TelecomRetrieval.v2", + "mteb_version": "2.26.10", + "scores": { + "test": [ + { + "ndcg_at_1": 0.44971, + "ndcg_at_3": 0.42313, + "ndcg_at_5": 0.43572, + "ndcg_at_10": 0.47069, + "ndcg_at_20": 0.5061, + "ndcg_at_100": 0.56553, + "ndcg_at_1000": 0.59263, + "map_at_1": 0.18431, + "map_at_3": 0.27843, + "map_at_5": 0.31388, + "map_at_10": 0.35279, + "map_at_20": 0.3767, + "map_at_100": 0.39646, + "map_at_1000": 0.39927, + "recall_at_1": 0.18431, + "recall_at_3": 0.32756, + "recall_at_5": 0.40696, + "recall_at_10": 0.53182, + "recall_at_20": 0.64513, + "recall_at_100": 0.84496, + "recall_at_1000": 0.98028, + "accuracy": 0.18431, + "precision_at_1": 0.47126, + "precision_at_3": 0.32088, + "precision_at_5": 0.25805, + "precision_at_10": 0.18477, + "precision_at_20": 0.12126, + "precision_at_100": 0.03586, + "precision_at_1000": 0.00436, + "mrr_at_1": 0.471264, + "mrr_at_3": 0.550287, + "mrr_at_5": 0.567385, + "mrr_at_10": 0.579229, + "mrr_at_20": 0.584416, + "mrr_at_100": 0.586883, + "mrr_at_1000": 0.587, + "nauc_ndcg_at_1_max": 0.2273, + "nauc_ndcg_at_1_std": 0.112759, + "nauc_ndcg_at_1_diff1": 0.370897, + "nauc_ndcg_at_3_max": 0.21778, + "nauc_ndcg_at_3_std": 0.126444, + "nauc_ndcg_at_3_diff1": 0.271506, + "nauc_ndcg_at_5_max": 0.220292, + "nauc_ndcg_at_5_std": 0.115724, + "nauc_ndcg_at_5_diff1": 0.285444, + "nauc_ndcg_at_10_max": 0.23042, + "nauc_ndcg_at_10_std": 0.113233, + "nauc_ndcg_at_10_diff1": 0.300911, + "nauc_ndcg_at_20_max": 0.262699, + "nauc_ndcg_at_20_std": 0.150937, + "nauc_ndcg_at_20_diff1": 0.303055, + "nauc_ndcg_at_100_max": 0.28147, + "nauc_ndcg_at_100_std": 0.194961, + "nauc_ndcg_at_100_diff1": 0.283368, + "nauc_ndcg_at_1000_max": 0.271055, + "nauc_ndcg_at_1000_std": 0.172049, + "nauc_ndcg_at_1000_diff1": 0.298852, + "nauc_map_at_1_max": 0.158327, + "nauc_map_at_1_std": 0.033286, + "nauc_map_at_1_diff1": 0.300649, + "nauc_map_at_3_max": 0.169076, + "nauc_map_at_3_std": 0.044517, + "nauc_map_at_3_diff1": 0.284145, + "nauc_map_at_5_max": 0.186482, + "nauc_map_at_5_std": 0.061922, + "nauc_map_at_5_diff1": 0.286766, + "nauc_map_at_10_max": 0.208088, + "nauc_map_at_10_std": 0.074274, + "nauc_map_at_10_diff1": 0.29655, + "nauc_map_at_20_max": 0.233422, + "nauc_map_at_20_std": 0.104433, + "nauc_map_at_20_diff1": 0.296228, + "nauc_map_at_100_max": 0.243611, + "nauc_map_at_100_std": 0.123755, + "nauc_map_at_100_diff1": 0.28417, + "nauc_map_at_1000_max": 0.243562, + "nauc_map_at_1000_std": 0.122836, + "nauc_map_at_1000_diff1": 0.285152, + "nauc_recall_at_1_max": 0.158327, + "nauc_recall_at_1_std": 0.033286, + "nauc_recall_at_1_diff1": 0.300649, + "nauc_recall_at_3_max": 0.158133, + "nauc_recall_at_3_std": 0.056353, + "nauc_recall_at_3_diff1": 0.221556, + "nauc_recall_at_5_max": 0.157204, + "nauc_recall_at_5_std": 0.069903, + "nauc_recall_at_5_diff1": 0.214447, + "nauc_recall_at_10_max": 0.183978, + "nauc_recall_at_10_std": 0.069006, + "nauc_recall_at_10_diff1": 0.232957, + "nauc_recall_at_20_max": 0.272575, + "nauc_recall_at_20_std": 0.184077, + "nauc_recall_at_20_diff1": 0.216075, + "nauc_recall_at_100_max": 0.382722, + "nauc_recall_at_100_std": 0.472444, + "nauc_recall_at_100_diff1": 0.109775, + "nauc_recall_at_1000_max": 0.608087, + "nauc_recall_at_1000_std": 0.763896, + "nauc_recall_at_1000_diff1": 0.235227, + "nauc_precision_at_1_max": 0.248453, + "nauc_precision_at_1_std": 0.136911, + "nauc_precision_at_1_diff1": 0.36231, + "nauc_precision_at_3_max": 0.198077, + "nauc_precision_at_3_std": 0.157625, + "nauc_precision_at_3_diff1": 0.155689, + "nauc_precision_at_5_max": 0.195287, + "nauc_precision_at_5_std": 0.159192, + "nauc_precision_at_5_diff1": 0.132535, + "nauc_precision_at_10_max": 0.188633, + "nauc_precision_at_10_std": 0.157984, + "nauc_precision_at_10_diff1": 0.096696, + "nauc_precision_at_20_max": 0.204158, + "nauc_precision_at_20_std": 0.214834, + "nauc_precision_at_20_diff1": 0.02842, + "nauc_precision_at_100_max": 0.14184, + "nauc_precision_at_100_std": 0.211466, + "nauc_precision_at_100_diff1": -0.076041, + "nauc_precision_at_1000_max": 0.052306, + "nauc_precision_at_1000_std": 0.114432, + "nauc_precision_at_1000_diff1": -0.076037, + "nauc_mrr_at_1_max": 0.248453, + "nauc_mrr_at_1_std": 0.136911, + "nauc_mrr_at_1_diff1": 0.36231, + "nauc_mrr_at_3_max": 0.260025, + "nauc_mrr_at_3_std": 0.181101, + "nauc_mrr_at_3_diff1": 0.317862, + "nauc_mrr_at_5_max": 0.25216, + "nauc_mrr_at_5_std": 0.175657, + "nauc_mrr_at_5_diff1": 0.324034, + "nauc_mrr_at_10_max": 0.251566, + "nauc_mrr_at_10_std": 0.171502, + "nauc_mrr_at_10_diff1": 0.327313, + "nauc_mrr_at_20_max": 0.254883, + "nauc_mrr_at_20_std": 0.174399, + "nauc_mrr_at_20_diff1": 0.326637, + "nauc_mrr_at_100_max": 0.256426, + "nauc_mrr_at_100_std": 0.175638, + "nauc_mrr_at_100_diff1": 0.326516, + "nauc_mrr_at_1000_max": 0.256253, + "nauc_mrr_at_1000_std": 0.17539, + "nauc_mrr_at_1000_diff1": 0.326618, + "hit_rate_at_1": 0.47126, + "hit_rate_at_3": 0.64655, + "hit_rate_at_5": 0.71839, + "hit_rate_at_10": 0.8046, + "hit_rate_at_20": 0.87931, + "hit_rate_at_100": 0.96552, + "hit_rate_at_1000": 0.99713, + "main_score": 0.47069, + "hf_subset": "french", + "languages": [ + "fra-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.43391, + "ndcg_at_3": 0.41721, + "ndcg_at_5": 0.41874, + "ndcg_at_10": 0.44696, + "ndcg_at_20": 0.48518, + "ndcg_at_100": 0.54812, + "ndcg_at_1000": 0.57695, + "map_at_1": 0.17422, + "map_at_3": 0.27116, + "map_at_5": 0.30078, + "map_at_10": 0.33695, + "map_at_20": 0.35904, + "map_at_100": 0.37966, + "map_at_1000": 0.38271, + "recall_at_1": 0.17422, + "recall_at_3": 0.31724, + "recall_at_5": 0.38139, + "recall_at_10": 0.49507, + "recall_at_20": 0.61784, + "recall_at_100": 0.83454, + "recall_at_1000": 0.97539, + "accuracy": 0.17422, + "precision_at_1": 0.45977, + "precision_at_3": 0.32567, + "precision_at_5": 0.2523, + "precision_at_10": 0.17529, + "precision_at_20": 0.11595, + "precision_at_100": 0.03514, + "precision_at_1000": 0.00433, + "mrr_at_1": 0.45977, + "mrr_at_3": 0.54454, + "mrr_at_5": 0.55704, + "mrr_at_10": 0.566821, + "mrr_at_20": 0.572952, + "mrr_at_100": 0.575796, + "mrr_at_1000": 0.575998, + "nauc_ndcg_at_1_max": 0.206853, + "nauc_ndcg_at_1_std": 0.113107, + "nauc_ndcg_at_1_diff1": 0.353842, + "nauc_ndcg_at_3_max": 0.193765, + "nauc_ndcg_at_3_std": 0.089527, + "nauc_ndcg_at_3_diff1": 0.285378, + "nauc_ndcg_at_5_max": 0.194575, + "nauc_ndcg_at_5_std": 0.072141, + "nauc_ndcg_at_5_diff1": 0.313685, + "nauc_ndcg_at_10_max": 0.207991, + "nauc_ndcg_at_10_std": 0.074405, + "nauc_ndcg_at_10_diff1": 0.308457, + "nauc_ndcg_at_20_max": 0.236917, + "nauc_ndcg_at_20_std": 0.106933, + "nauc_ndcg_at_20_diff1": 0.316319, + "nauc_ndcg_at_100_max": 0.271522, + "nauc_ndcg_at_100_std": 0.172434, + "nauc_ndcg_at_100_diff1": 0.318411, + "nauc_ndcg_at_1000_max": 0.249524, + "nauc_ndcg_at_1000_std": 0.145118, + "nauc_ndcg_at_1000_diff1": 0.31419, + "nauc_map_at_1_max": 0.099725, + "nauc_map_at_1_std": -0.008292, + "nauc_map_at_1_diff1": 0.323182, + "nauc_map_at_3_max": 0.122988, + "nauc_map_at_3_std": 0.005369, + "nauc_map_at_3_diff1": 0.29348, + "nauc_map_at_5_max": 0.13701, + "nauc_map_at_5_std": 0.010077, + "nauc_map_at_5_diff1": 0.296768, + "nauc_map_at_10_max": 0.166105, + "nauc_map_at_10_std": 0.025756, + "nauc_map_at_10_diff1": 0.291886, + "nauc_map_at_20_max": 0.191625, + "nauc_map_at_20_std": 0.050389, + "nauc_map_at_20_diff1": 0.301064, + "nauc_map_at_100_max": 0.206116, + "nauc_map_at_100_std": 0.075143, + "nauc_map_at_100_diff1": 0.295783, + "nauc_map_at_1000_max": 0.205021, + "nauc_map_at_1000_std": 0.074118, + "nauc_map_at_1000_diff1": 0.295695, + "nauc_recall_at_1_max": 0.099725, + "nauc_recall_at_1_std": -0.008292, + "nauc_recall_at_1_diff1": 0.323182, + "nauc_recall_at_3_max": 0.123301, + "nauc_recall_at_3_std": 0.023332, + "nauc_recall_at_3_diff1": 0.268057, + "nauc_recall_at_5_max": 0.119251, + "nauc_recall_at_5_std": 0.011563, + "nauc_recall_at_5_diff1": 0.261092, + "nauc_recall_at_10_max": 0.176554, + "nauc_recall_at_10_std": 0.030277, + "nauc_recall_at_10_diff1": 0.225784, + "nauc_recall_at_20_max": 0.248395, + "nauc_recall_at_20_std": 0.123975, + "nauc_recall_at_20_diff1": 0.239489, + "nauc_recall_at_100_max": 0.483128, + "nauc_recall_at_100_std": 0.490258, + "nauc_recall_at_100_diff1": 0.315162, + "nauc_recall_at_1000_max": 0.739664, + "nauc_recall_at_1000_std": 0.810361, + "nauc_recall_at_1000_diff1": 0.548309, + "nauc_precision_at_1_max": 0.220453, + "nauc_precision_at_1_std": 0.129685, + "nauc_precision_at_1_diff1": 0.358779, + "nauc_precision_at_3_max": 0.223299, + "nauc_precision_at_3_std": 0.133283, + "nauc_precision_at_3_diff1": 0.193217, + "nauc_precision_at_5_max": 0.241123, + "nauc_precision_at_5_std": 0.136931, + "nauc_precision_at_5_diff1": 0.173335, + "nauc_precision_at_10_max": 0.241466, + "nauc_precision_at_10_std": 0.157919, + "nauc_precision_at_10_diff1": 0.117825, + "nauc_precision_at_20_max": 0.253238, + "nauc_precision_at_20_std": 0.220593, + "nauc_precision_at_20_diff1": 0.077102, + "nauc_precision_at_100_max": 0.196392, + "nauc_precision_at_100_std": 0.254807, + "nauc_precision_at_100_diff1": -0.030874, + "nauc_precision_at_1000_max": 0.084067, + "nauc_precision_at_1000_std": 0.141724, + "nauc_precision_at_1000_diff1": -0.080799, + "nauc_mrr_at_1_max": 0.220453, + "nauc_mrr_at_1_std": 0.129685, + "nauc_mrr_at_1_diff1": 0.358779, + "nauc_mrr_at_3_max": 0.244006, + "nauc_mrr_at_3_std": 0.15808, + "nauc_mrr_at_3_diff1": 0.352555, + "nauc_mrr_at_5_max": 0.24196, + "nauc_mrr_at_5_std": 0.153046, + "nauc_mrr_at_5_diff1": 0.354744, + "nauc_mrr_at_10_max": 0.251505, + "nauc_mrr_at_10_std": 0.157889, + "nauc_mrr_at_10_diff1": 0.350868, + "nauc_mrr_at_20_max": 0.250883, + "nauc_mrr_at_20_std": 0.159553, + "nauc_mrr_at_20_diff1": 0.349688, + "nauc_mrr_at_100_max": 0.250814, + "nauc_mrr_at_100_std": 0.159834, + "nauc_mrr_at_100_diff1": 0.351946, + "nauc_mrr_at_1000_max": 0.250549, + "nauc_mrr_at_1000_std": 0.159527, + "nauc_mrr_at_1000_diff1": 0.351839, + "hit_rate_at_1": 0.45977, + "hit_rate_at_3": 0.64368, + "hit_rate_at_5": 0.70115, + "hit_rate_at_10": 0.77586, + "hit_rate_at_20": 0.86494, + "hit_rate_at_100": 0.9569, + "hit_rate_at_1000": 0.99713, + "main_score": 0.44696, + "hf_subset": "spanish", + "languages": [ + "spa-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.52155, + "ndcg_at_3": 0.50838, + "ndcg_at_5": 0.52237, + "ndcg_at_10": 0.55397, + "ndcg_at_20": 0.58897, + "ndcg_at_100": 0.63671, + "ndcg_at_1000": 0.65674, + "map_at_1": 0.21586, + "map_at_3": 0.34096, + "map_at_5": 0.38516, + "map_at_10": 0.42805, + "map_at_20": 0.45381, + "map_at_100": 0.4725, + "map_at_1000": 0.47477, + "recall_at_1": 0.21586, + "recall_at_3": 0.40182, + "recall_at_5": 0.49881, + "recall_at_10": 0.62719, + "recall_at_20": 0.72869, + "recall_at_100": 0.88301, + "recall_at_1000": 0.97704, + "accuracy": 0.21586, + "precision_at_1": 0.55172, + "precision_at_3": 0.38793, + "precision_at_5": 0.31092, + "precision_at_10": 0.21264, + "precision_at_20": 0.13721, + "precision_at_100": 0.03764, + "precision_at_1000": 0.00436, + "mrr_at_1": 0.551724, + "mrr_at_3": 0.649425, + "mrr_at_5": 0.6625, + "mrr_at_10": 0.672677, + "mrr_at_20": 0.674898, + "mrr_at_100": 0.675903, + "mrr_at_1000": 0.67603, + "nauc_ndcg_at_1_max": 0.225306, + "nauc_ndcg_at_1_std": 0.018516, + "nauc_ndcg_at_1_diff1": 0.411955, + "nauc_ndcg_at_3_max": 0.193908, + "nauc_ndcg_at_3_std": -0.03054, + "nauc_ndcg_at_3_diff1": 0.319586, + "nauc_ndcg_at_5_max": 0.172169, + "nauc_ndcg_at_5_std": -0.036375, + "nauc_ndcg_at_5_diff1": 0.309519, + "nauc_ndcg_at_10_max": 0.200325, + "nauc_ndcg_at_10_std": -0.014866, + "nauc_ndcg_at_10_diff1": 0.329427, + "nauc_ndcg_at_20_max": 0.228482, + "nauc_ndcg_at_20_std": 0.019191, + "nauc_ndcg_at_20_diff1": 0.331028, + "nauc_ndcg_at_100_max": 0.264854, + "nauc_ndcg_at_100_std": 0.067803, + "nauc_ndcg_at_100_diff1": 0.337356, + "nauc_ndcg_at_1000_max": 0.25569, + "nauc_ndcg_at_1000_std": 0.042483, + "nauc_ndcg_at_1000_diff1": 0.348889, + "nauc_map_at_1_max": 0.073829, + "nauc_map_at_1_std": -0.049769, + "nauc_map_at_1_diff1": 0.326465, + "nauc_map_at_3_max": 0.097257, + "nauc_map_at_3_std": -0.077888, + "nauc_map_at_3_diff1": 0.293592, + "nauc_map_at_5_max": 0.108098, + "nauc_map_at_5_std": -0.070761, + "nauc_map_at_5_diff1": 0.288857, + "nauc_map_at_10_max": 0.153917, + "nauc_map_at_10_std": -0.054882, + "nauc_map_at_10_diff1": 0.296147, + "nauc_map_at_20_max": 0.182549, + "nauc_map_at_20_std": -0.025895, + "nauc_map_at_20_diff1": 0.297556, + "nauc_map_at_100_max": 0.201265, + "nauc_map_at_100_std": -0.004529, + "nauc_map_at_100_diff1": 0.301069, + "nauc_map_at_1000_max": 0.200869, + "nauc_map_at_1000_std": -0.005514, + "nauc_map_at_1000_diff1": 0.302462, + "nauc_recall_at_1_max": 0.073829, + "nauc_recall_at_1_std": -0.049769, + "nauc_recall_at_1_diff1": 0.326465, + "nauc_recall_at_3_max": 0.070636, + "nauc_recall_at_3_std": -0.092226, + "nauc_recall_at_3_diff1": 0.229693, + "nauc_recall_at_5_max": 0.033479, + "nauc_recall_at_5_std": -0.075515, + "nauc_recall_at_5_diff1": 0.186929, + "nauc_recall_at_10_max": 0.113148, + "nauc_recall_at_10_std": -0.025408, + "nauc_recall_at_10_diff1": 0.196499, + "nauc_recall_at_20_max": 0.194375, + "nauc_recall_at_20_std": 0.089966, + "nauc_recall_at_20_diff1": 0.186846, + "nauc_recall_at_100_max": 0.394632, + "nauc_recall_at_100_std": 0.449699, + "nauc_recall_at_100_diff1": 0.154805, + "nauc_recall_at_1000_max": 0.680635, + "nauc_recall_at_1000_std": 0.698122, + "nauc_recall_at_1000_diff1": 0.297841, + "nauc_precision_at_1_max": 0.218904, + "nauc_precision_at_1_std": 0.049491, + "nauc_precision_at_1_diff1": 0.411143, + "nauc_precision_at_3_max": 0.204514, + "nauc_precision_at_3_std": -0.002539, + "nauc_precision_at_3_diff1": 0.172695, + "nauc_precision_at_5_max": 0.20051, + "nauc_precision_at_5_std": 0.023277, + "nauc_precision_at_5_diff1": 0.114879, + "nauc_precision_at_10_max": 0.259893, + "nauc_precision_at_10_std": 0.096244, + "nauc_precision_at_10_diff1": 0.082922, + "nauc_precision_at_20_max": 0.267071, + "nauc_precision_at_20_std": 0.169893, + "nauc_precision_at_20_diff1": 0.036508, + "nauc_precision_at_100_max": 0.211693, + "nauc_precision_at_100_std": 0.199299, + "nauc_precision_at_100_diff1": -0.0095, + "nauc_precision_at_1000_max": 0.135466, + "nauc_precision_at_1000_std": 0.12391, + "nauc_precision_at_1000_diff1": -0.024198, + "nauc_mrr_at_1_max": 0.218904, + "nauc_mrr_at_1_std": 0.049491, + "nauc_mrr_at_1_diff1": 0.411143, + "nauc_mrr_at_3_max": 0.252554, + "nauc_mrr_at_3_std": 0.040234, + "nauc_mrr_at_3_diff1": 0.398937, + "nauc_mrr_at_5_max": 0.252818, + "nauc_mrr_at_5_std": 0.05395, + "nauc_mrr_at_5_diff1": 0.401555, + "nauc_mrr_at_10_max": 0.25452, + "nauc_mrr_at_10_std": 0.057887, + "nauc_mrr_at_10_diff1": 0.404193, + "nauc_mrr_at_20_max": 0.252732, + "nauc_mrr_at_20_std": 0.057115, + "nauc_mrr_at_20_diff1": 0.404591, + "nauc_mrr_at_100_max": 0.25325, + "nauc_mrr_at_100_std": 0.057361, + "nauc_mrr_at_100_diff1": 0.404438, + "nauc_mrr_at_1000_max": 0.253109, + "nauc_mrr_at_1000_std": 0.057134, + "nauc_mrr_at_1000_diff1": 0.404446, + "hit_rate_at_1": 0.55172, + "hit_rate_at_3": 0.77011, + "hit_rate_at_5": 0.83046, + "hit_rate_at_10": 0.9023, + "hit_rate_at_20": 0.93678, + "hit_rate_at_100": 0.97126, + "hit_rate_at_1000": 0.99713, + "main_score": 0.55397, + "hf_subset": "english", + "languages": [ + "eng-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.42816, + "ndcg_at_3": 0.40558, + "ndcg_at_5": 0.41519, + "ndcg_at_10": 0.44356, + "ndcg_at_20": 0.48527, + "ndcg_at_100": 0.54415, + "ndcg_at_1000": 0.57281, + "map_at_1": 0.1687, + "map_at_3": 0.26182, + "map_at_5": 0.29711, + "map_at_10": 0.33291, + "map_at_20": 0.35949, + "map_at_100": 0.3791, + "map_at_1000": 0.38205, + "recall_at_1": 0.1687, + "recall_at_3": 0.31392, + "recall_at_5": 0.38944, + "recall_at_10": 0.49907, + "recall_at_20": 0.63197, + "recall_at_100": 0.83342, + "recall_at_1000": 0.97092, + "accuracy": 0.1687, + "precision_at_1": 0.4454, + "precision_at_3": 0.31418, + "precision_at_5": 0.2523, + "precision_at_10": 0.17759, + "precision_at_20": 0.1181, + "precision_at_100": 0.03486, + "precision_at_1000": 0.00431, + "mrr_at_1": 0.445402, + "mrr_at_3": 0.524425, + "mrr_at_5": 0.535345, + "mrr_at_10": 0.54584, + "mrr_at_20": 0.55226, + "mrr_at_100": 0.555012, + "mrr_at_1000": 0.555251, + "nauc_ndcg_at_1_max": 0.280697, + "nauc_ndcg_at_1_std": 0.154393, + "nauc_ndcg_at_1_diff1": 0.467371, + "nauc_ndcg_at_3_max": 0.261966, + "nauc_ndcg_at_3_std": 0.123105, + "nauc_ndcg_at_3_diff1": 0.427184, + "nauc_ndcg_at_5_max": 0.266448, + "nauc_ndcg_at_5_std": 0.114903, + "nauc_ndcg_at_5_diff1": 0.426934, + "nauc_ndcg_at_10_max": 0.291286, + "nauc_ndcg_at_10_std": 0.137358, + "nauc_ndcg_at_10_diff1": 0.418164, + "nauc_ndcg_at_20_max": 0.318638, + "nauc_ndcg_at_20_std": 0.163689, + "nauc_ndcg_at_20_diff1": 0.436638, + "nauc_ndcg_at_100_max": 0.332754, + "nauc_ndcg_at_100_std": 0.216636, + "nauc_ndcg_at_100_diff1": 0.433339, + "nauc_ndcg_at_1000_max": 0.321238, + "nauc_ndcg_at_1000_std": 0.19272, + "nauc_ndcg_at_1000_diff1": 0.439037, + "nauc_map_at_1_max": 0.195392, + "nauc_map_at_1_std": 0.025181, + "nauc_map_at_1_diff1": 0.424614, + "nauc_map_at_3_max": 0.215072, + "nauc_map_at_3_std": 0.028039, + "nauc_map_at_3_diff1": 0.413949, + "nauc_map_at_5_max": 0.221062, + "nauc_map_at_5_std": 0.038326, + "nauc_map_at_5_diff1": 0.40641, + "nauc_map_at_10_max": 0.245426, + "nauc_map_at_10_std": 0.070526, + "nauc_map_at_10_diff1": 0.401179, + "nauc_map_at_20_max": 0.267075, + "nauc_map_at_20_std": 0.098762, + "nauc_map_at_20_diff1": 0.411663, + "nauc_map_at_100_max": 0.277961, + "nauc_map_at_100_std": 0.126583, + "nauc_map_at_100_diff1": 0.408122, + "nauc_map_at_1000_max": 0.277774, + "nauc_map_at_1000_std": 0.126498, + "nauc_map_at_1000_diff1": 0.409309, + "nauc_recall_at_1_max": 0.195392, + "nauc_recall_at_1_std": 0.025181, + "nauc_recall_at_1_diff1": 0.424614, + "nauc_recall_at_3_max": 0.198485, + "nauc_recall_at_3_std": 0.039356, + "nauc_recall_at_3_diff1": 0.356115, + "nauc_recall_at_5_max": 0.20423, + "nauc_recall_at_5_std": 0.058004, + "nauc_recall_at_5_diff1": 0.336982, + "nauc_recall_at_10_max": 0.265934, + "nauc_recall_at_10_std": 0.118195, + "nauc_recall_at_10_diff1": 0.307615, + "nauc_recall_at_20_max": 0.337209, + "nauc_recall_at_20_std": 0.194665, + "nauc_recall_at_20_diff1": 0.329258, + "nauc_recall_at_100_max": 0.43745, + "nauc_recall_at_100_std": 0.475933, + "nauc_recall_at_100_diff1": 0.317898, + "nauc_recall_at_1000_max": 0.754107, + "nauc_recall_at_1000_std": 0.678293, + "nauc_recall_at_1000_diff1": 0.57171, + "nauc_precision_at_1_max": 0.278544, + "nauc_precision_at_1_std": 0.173026, + "nauc_precision_at_1_diff1": 0.474055, + "nauc_precision_at_3_max": 0.235258, + "nauc_precision_at_3_std": 0.16437, + "nauc_precision_at_3_diff1": 0.321822, + "nauc_precision_at_5_max": 0.215396, + "nauc_precision_at_5_std": 0.17402, + "nauc_precision_at_5_diff1": 0.252086, + "nauc_precision_at_10_max": 0.215999, + "nauc_precision_at_10_std": 0.222677, + "nauc_precision_at_10_diff1": 0.16601, + "nauc_precision_at_20_max": 0.229143, + "nauc_precision_at_20_std": 0.268665, + "nauc_precision_at_20_diff1": 0.129114, + "nauc_precision_at_100_max": 0.160326, + "nauc_precision_at_100_std": 0.301737, + "nauc_precision_at_100_diff1": 0.002544, + "nauc_precision_at_1000_max": 0.050236, + "nauc_precision_at_1000_std": 0.179318, + "nauc_precision_at_1000_diff1": -0.050887, + "nauc_mrr_at_1_max": 0.278544, + "nauc_mrr_at_1_std": 0.173026, + "nauc_mrr_at_1_diff1": 0.474055, + "nauc_mrr_at_3_max": 0.291374, + "nauc_mrr_at_3_std": 0.20768, + "nauc_mrr_at_3_diff1": 0.467919, + "nauc_mrr_at_5_max": 0.294644, + "nauc_mrr_at_5_std": 0.211237, + "nauc_mrr_at_5_diff1": 0.47193, + "nauc_mrr_at_10_max": 0.296471, + "nauc_mrr_at_10_std": 0.20831, + "nauc_mrr_at_10_diff1": 0.467494, + "nauc_mrr_at_20_max": 0.298145, + "nauc_mrr_at_20_std": 0.209323, + "nauc_mrr_at_20_diff1": 0.46814, + "nauc_mrr_at_100_max": 0.297041, + "nauc_mrr_at_100_std": 0.209812, + "nauc_mrr_at_100_diff1": 0.468523, + "nauc_mrr_at_1000_max": 0.29695, + "nauc_mrr_at_1000_std": 0.209598, + "nauc_mrr_at_1000_diff1": 0.468584, + "hit_rate_at_1": 0.4454, + "hit_rate_at_3": 0.63218, + "hit_rate_at_5": 0.68103, + "hit_rate_at_10": 0.75862, + "hit_rate_at_20": 0.8477, + "hit_rate_at_100": 0.95115, + "hit_rate_at_1000": 0.99425, + "main_score": 0.44356, + "hf_subset": "german", + "languages": [ + "deu-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.42098, + "ndcg_at_3": 0.42898, + "ndcg_at_5": 0.42466, + "ndcg_at_10": 0.45021, + "ndcg_at_20": 0.48974, + "ndcg_at_100": 0.55296, + "ndcg_at_1000": 0.58055, + "map_at_1": 0.16994, + "map_at_3": 0.28269, + "map_at_5": 0.31131, + "map_at_10": 0.34297, + "map_at_20": 0.36447, + "map_at_100": 0.38548, + "map_at_1000": 0.38849, + "recall_at_1": 0.16994, + "recall_at_3": 0.3437, + "recall_at_5": 0.39535, + "recall_at_10": 0.50413, + "recall_at_20": 0.62858, + "recall_at_100": 0.83635, + "recall_at_1000": 0.96963, + "accuracy": 0.16994, + "precision_at_1": 0.43966, + "precision_at_3": 0.33525, + "precision_at_5": 0.25287, + "precision_at_10": 0.17011, + "precision_at_20": 0.1125, + "precision_at_100": 0.03506, + "precision_at_1000": 0.00431, + "mrr_at_1": 0.439655, + "mrr_at_3": 0.539272, + "mrr_at_5": 0.544732, + "mrr_at_10": 0.556255, + "mrr_at_20": 0.5639, + "mrr_at_100": 0.566311, + "mrr_at_1000": 0.566436, + "nauc_ndcg_at_1_max": 0.238647, + "nauc_ndcg_at_1_std": 0.062149, + "nauc_ndcg_at_1_diff1": 0.425184, + "nauc_ndcg_at_3_max": 0.211637, + "nauc_ndcg_at_3_std": 0.030566, + "nauc_ndcg_at_3_diff1": 0.331402, + "nauc_ndcg_at_5_max": 0.222751, + "nauc_ndcg_at_5_std": 0.031203, + "nauc_ndcg_at_5_diff1": 0.345492, + "nauc_ndcg_at_10_max": 0.245535, + "nauc_ndcg_at_10_std": 0.056108, + "nauc_ndcg_at_10_diff1": 0.353139, + "nauc_ndcg_at_20_max": 0.269294, + "nauc_ndcg_at_20_std": 0.084673, + "nauc_ndcg_at_20_diff1": 0.355514, + "nauc_ndcg_at_100_max": 0.283914, + "nauc_ndcg_at_100_std": 0.137627, + "nauc_ndcg_at_100_diff1": 0.369342, + "nauc_ndcg_at_1000_max": 0.27009, + "nauc_ndcg_at_1000_std": 0.107183, + "nauc_ndcg_at_1000_diff1": 0.378318, + "nauc_map_at_1_max": 0.138411, + "nauc_map_at_1_std": -0.03228, + "nauc_map_at_1_diff1": 0.363946, + "nauc_map_at_3_max": 0.173693, + "nauc_map_at_3_std": -0.051309, + "nauc_map_at_3_diff1": 0.313177, + "nauc_map_at_5_max": 0.192252, + "nauc_map_at_5_std": -0.029711, + "nauc_map_at_5_diff1": 0.311756, + "nauc_map_at_10_max": 0.210924, + "nauc_map_at_10_std": 0.001169, + "nauc_map_at_10_diff1": 0.316574, + "nauc_map_at_20_max": 0.227699, + "nauc_map_at_20_std": 0.022222, + "nauc_map_at_20_diff1": 0.321406, + "nauc_map_at_100_max": 0.233965, + "nauc_map_at_100_std": 0.043712, + "nauc_map_at_100_diff1": 0.326918, + "nauc_map_at_1000_max": 0.23289, + "nauc_map_at_1000_std": 0.041709, + "nauc_map_at_1000_diff1": 0.327861, + "nauc_recall_at_1_max": 0.138411, + "nauc_recall_at_1_std": -0.03228, + "nauc_recall_at_1_diff1": 0.363946, + "nauc_recall_at_3_max": 0.168073, + "nauc_recall_at_3_std": -0.036324, + "nauc_recall_at_3_diff1": 0.265092, + "nauc_recall_at_5_max": 0.189567, + "nauc_recall_at_5_std": -0.005676, + "nauc_recall_at_5_diff1": 0.259406, + "nauc_recall_at_10_max": 0.228851, + "nauc_recall_at_10_std": 0.076637, + "nauc_recall_at_10_diff1": 0.249897, + "nauc_recall_at_20_max": 0.280648, + "nauc_recall_at_20_std": 0.150291, + "nauc_recall_at_20_diff1": 0.251533, + "nauc_recall_at_100_max": 0.428771, + "nauc_recall_at_100_std": 0.50471, + "nauc_recall_at_100_diff1": 0.299575, + "nauc_recall_at_1000_max": 0.686171, + "nauc_recall_at_1000_std": 0.797388, + "nauc_recall_at_1000_diff1": 0.649404, + "nauc_precision_at_1_max": 0.24935, + "nauc_precision_at_1_std": 0.067694, + "nauc_precision_at_1_diff1": 0.4167, + "nauc_precision_at_3_max": 0.210343, + "nauc_precision_at_3_std": 0.086128, + "nauc_precision_at_3_diff1": 0.206281, + "nauc_precision_at_5_max": 0.204394, + "nauc_precision_at_5_std": 0.116166, + "nauc_precision_at_5_diff1": 0.182648, + "nauc_precision_at_10_max": 0.19485, + "nauc_precision_at_10_std": 0.156352, + "nauc_precision_at_10_diff1": 0.161086, + "nauc_precision_at_20_max": 0.194653, + "nauc_precision_at_20_std": 0.195729, + "nauc_precision_at_20_diff1": 0.117075, + "nauc_precision_at_100_max": 0.096705, + "nauc_precision_at_100_std": 0.209796, + "nauc_precision_at_100_diff1": 0.03614, + "nauc_precision_at_1000_max": -0.009316, + "nauc_precision_at_1000_std": 0.083741, + "nauc_precision_at_1000_diff1": -0.003429, + "nauc_mrr_at_1_max": 0.24935, + "nauc_mrr_at_1_std": 0.067694, + "nauc_mrr_at_1_diff1": 0.4167, + "nauc_mrr_at_3_max": 0.24909, + "nauc_mrr_at_3_std": 0.10272, + "nauc_mrr_at_3_diff1": 0.414293, + "nauc_mrr_at_5_max": 0.252377, + "nauc_mrr_at_5_std": 0.102478, + "nauc_mrr_at_5_diff1": 0.416944, + "nauc_mrr_at_10_max": 0.254614, + "nauc_mrr_at_10_std": 0.104854, + "nauc_mrr_at_10_diff1": 0.416055, + "nauc_mrr_at_20_max": 0.254858, + "nauc_mrr_at_20_std": 0.103828, + "nauc_mrr_at_20_diff1": 0.41415, + "nauc_mrr_at_100_max": 0.254651, + "nauc_mrr_at_100_std": 0.104198, + "nauc_mrr_at_100_diff1": 0.413837, + "nauc_mrr_at_1000_max": 0.254541, + "nauc_mrr_at_1000_std": 0.104005, + "nauc_mrr_at_1000_diff1": 0.414058, + "hit_rate_at_1": 0.43966, + "hit_rate_at_3": 0.66092, + "hit_rate_at_5": 0.68391, + "hit_rate_at_10": 0.76724, + "hit_rate_at_20": 0.87644, + "hit_rate_at_100": 0.96264, + "hit_rate_at_1000": 0.98851, + "main_score": 0.45021, + "hf_subset": "italian", + "languages": [ + "ita-Latn" + ], + "mteb_version": "2.26.10" + }, + { + "ndcg_at_1": 0.42529, + "ndcg_at_3": 0.4222, + "ndcg_at_5": 0.42472, + "ndcg_at_10": 0.45386, + "ndcg_at_20": 0.48901, + "ndcg_at_100": 0.55049, + "ndcg_at_1000": 0.57943, + "map_at_1": 0.17519, + "map_at_3": 0.27794, + "map_at_5": 0.3106, + "map_at_10": 0.3457, + "map_at_20": 0.36697, + "map_at_100": 0.3873, + "map_at_1000": 0.39033, + "recall_at_1": 0.17519, + "recall_at_3": 0.32759, + "recall_at_5": 0.39633, + "recall_at_10": 0.51286, + "recall_at_20": 0.62245, + "recall_at_100": 0.82993, + "recall_at_1000": 0.97433, + "accuracy": 0.17519, + "precision_at_1": 0.45115, + "precision_at_3": 0.33238, + "precision_at_5": 0.25632, + "precision_at_10": 0.17701, + "precision_at_20": 0.11595, + "precision_at_100": 0.03523, + "precision_at_1000": 0.00433, + "mrr_at_1": 0.451149, + "mrr_at_3": 0.541667, + "mrr_at_5": 0.554454, + "mrr_at_10": 0.56628, + "mrr_at_20": 0.570595, + "mrr_at_100": 0.572952, + "mrr_at_1000": 0.573184, + "nauc_ndcg_at_1_max": 0.353517, + "nauc_ndcg_at_1_std": 0.104441, + "nauc_ndcg_at_1_diff1": 0.388208, + "nauc_ndcg_at_3_max": 0.311237, + "nauc_ndcg_at_3_std": 0.087046, + "nauc_ndcg_at_3_diff1": 0.281644, + "nauc_ndcg_at_5_max": 0.329373, + "nauc_ndcg_at_5_std": 0.089553, + "nauc_ndcg_at_5_diff1": 0.314426, + "nauc_ndcg_at_10_max": 0.347052, + "nauc_ndcg_at_10_std": 0.114999, + "nauc_ndcg_at_10_diff1": 0.331647, + "nauc_ndcg_at_20_max": 0.359238, + "nauc_ndcg_at_20_std": 0.13526, + "nauc_ndcg_at_20_diff1": 0.330893, + "nauc_ndcg_at_100_max": 0.390206, + "nauc_ndcg_at_100_std": 0.184185, + "nauc_ndcg_at_100_diff1": 0.317504, + "nauc_ndcg_at_1000_max": 0.372931, + "nauc_ndcg_at_1000_std": 0.161082, + "nauc_ndcg_at_1000_diff1": 0.320068, + "nauc_map_at_1_max": 0.224677, + "nauc_map_at_1_std": -0.013717, + "nauc_map_at_1_diff1": 0.310967, + "nauc_map_at_3_max": 0.24119, + "nauc_map_at_3_std": 0.001046, + "nauc_map_at_3_diff1": 0.282087, + "nauc_map_at_5_max": 0.280225, + "nauc_map_at_5_std": 0.022292, + "nauc_map_at_5_diff1": 0.291713, + "nauc_map_at_10_max": 0.307337, + "nauc_map_at_10_std": 0.049675, + "nauc_map_at_10_diff1": 0.295138, + "nauc_map_at_20_max": 0.325725, + "nauc_map_at_20_std": 0.068682, + "nauc_map_at_20_diff1": 0.296336, + "nauc_map_at_100_max": 0.341099, + "nauc_map_at_100_std": 0.090463, + "nauc_map_at_100_diff1": 0.291013, + "nauc_map_at_1000_max": 0.340552, + "nauc_map_at_1000_std": 0.089831, + "nauc_map_at_1000_diff1": 0.290581, + "nauc_recall_at_1_max": 0.224677, + "nauc_recall_at_1_std": -0.013717, + "nauc_recall_at_1_diff1": 0.310967, + "nauc_recall_at_3_max": 0.194758, + "nauc_recall_at_3_std": 0.008232, + "nauc_recall_at_3_diff1": 0.217854, + "nauc_recall_at_5_max": 0.230236, + "nauc_recall_at_5_std": 0.032902, + "nauc_recall_at_5_diff1": 0.254426, + "nauc_recall_at_10_max": 0.293107, + "nauc_recall_at_10_std": 0.112643, + "nauc_recall_at_10_diff1": 0.277542, + "nauc_recall_at_20_max": 0.325931, + "nauc_recall_at_20_std": 0.167711, + "nauc_recall_at_20_diff1": 0.262481, + "nauc_recall_at_100_max": 0.513265, + "nauc_recall_at_100_std": 0.441094, + "nauc_recall_at_100_diff1": 0.20678, + "nauc_recall_at_1000_max": 0.75105, + "nauc_recall_at_1000_std": 0.757772, + "nauc_recall_at_1000_diff1": 0.357207, + "nauc_precision_at_1_max": 0.363857, + "nauc_precision_at_1_std": 0.102607, + "nauc_precision_at_1_diff1": 0.378659, + "nauc_precision_at_3_max": 0.306761, + "nauc_precision_at_3_std": 0.128902, + "nauc_precision_at_3_diff1": 0.17617, + "nauc_precision_at_5_max": 0.338465, + "nauc_precision_at_5_std": 0.16657, + "nauc_precision_at_5_diff1": 0.151279, + "nauc_precision_at_10_max": 0.327949, + "nauc_precision_at_10_std": 0.218946, + "nauc_precision_at_10_diff1": 0.112017, + "nauc_precision_at_20_max": 0.301006, + "nauc_precision_at_20_std": 0.247263, + "nauc_precision_at_20_diff1": 0.061071, + "nauc_precision_at_100_max": 0.203553, + "nauc_precision_at_100_std": 0.251958, + "nauc_precision_at_100_diff1": -0.051691, + "nauc_precision_at_1000_max": 0.080598, + "nauc_precision_at_1000_std": 0.146816, + "nauc_precision_at_1000_diff1": -0.093095, + "nauc_mrr_at_1_max": 0.363857, + "nauc_mrr_at_1_std": 0.102607, + "nauc_mrr_at_1_diff1": 0.378659, + "nauc_mrr_at_3_max": 0.349792, + "nauc_mrr_at_3_std": 0.13359, + "nauc_mrr_at_3_diff1": 0.339835, + "nauc_mrr_at_5_max": 0.35137, + "nauc_mrr_at_5_std": 0.135889, + "nauc_mrr_at_5_diff1": 0.350187, + "nauc_mrr_at_10_max": 0.358341, + "nauc_mrr_at_10_std": 0.145587, + "nauc_mrr_at_10_diff1": 0.355129, + "nauc_mrr_at_20_max": 0.357727, + "nauc_mrr_at_20_std": 0.144336, + "nauc_mrr_at_20_diff1": 0.355551, + "nauc_mrr_at_100_max": 0.358255, + "nauc_mrr_at_100_std": 0.143897, + "nauc_mrr_at_100_diff1": 0.35457, + "nauc_mrr_at_1000_max": 0.357959, + "nauc_mrr_at_1000_std": 0.143475, + "nauc_mrr_at_1000_diff1": 0.354635, + "hit_rate_at_1": 0.45115, + "hit_rate_at_3": 0.65517, + "hit_rate_at_5": 0.70977, + "hit_rate_at_10": 0.80172, + "hit_rate_at_20": 0.86782, + "hit_rate_at_100": 0.95402, + "hit_rate_at_1000": 0.99713, + "main_score": 0.45386, + "hf_subset": "portuguese", + "languages": [ + "por-Latn" + ], + "mteb_version": "2.26.10" + } + ] + }, + "evaluation_time": 3060.337668285938, + "kg_co2_emissions": null, + "date": 1792353017.794177, + "evaluation_phases": [ + { + "name": "Data loading", + "start": 8.933013305068016e-06, + "end": 25.464354566996917, + "split": "", + "subset": "" + }, + { + "name": "Dataset transform", + "start": 25.464392894064076, + "end": 25.46439877303783, + "split": "", + "subset": "" + }, + { + "name": "Encoding queries", + "start": 25.48963578999974, + "end": 27.58650594903156, + "split": "test", + "subset": "french" + }, + { + "name": "Searching corpus", + "start": 27.588436674093828, + "end": 491.3535022120923, + "split": "test", + "subset": "french" + }, + { + "name": "Scoring", + "start": 491.39683305309154, + "end": 491.6190490970621, + "split": "test", + "subset": "french" + }, + { + "name": "Encoding queries", + "start": 491.6546724230284, + "end": 492.0005191230448, + "split": "test", + "subset": "spanish" + }, + { + "name": "Searching corpus", + "start": 492.00240499002393, + "end": 1041.9073307840154, + "split": "test", + "subset": "spanish" + }, + { + "name": "Scoring", + "start": 1041.95316511509, + "end": 1042.1677710220683, + "split": "test", + "subset": "spanish" + }, + { + "name": "Encoding queries", + "start": 1042.205366611015, + "end": 1042.7103547520237, + "split": "test", + "subset": "english" + }, + { + "name": "Searching corpus", + "start": 1042.712244405062, + "end": 1631.995800729026, + "split": "test", + "subset": "english" + }, + { + "name": "Scoring", + "start": 1632.0443293640856, + "end": 1632.2550185209839, + "split": "test", + "subset": "english" + }, + { + "name": "Encoding queries", + "start": 1632.2919770199805, + "end": 1632.67109587905, + "split": "test", + "subset": "german" + }, + { + "name": "Searching corpus", + "start": 1632.6731431380613, + "end": 2098.9987504960736, + "split": "test", + "subset": "german" + }, + { + "name": "Scoring", + "start": 2099.043132291059, + "end": 2099.251403394039, + "split": "test", + "subset": "german" + }, + { + "name": "Encoding queries", + "start": 2099.2866663760506, + "end": 2099.5375391960843, + "split": "test", + "subset": "italian" + }, + { + "name": "Searching corpus", + "start": 2099.539716480067, + "end": 2587.983643447049, + "split": "test", + "subset": "italian" + }, + { + "name": "Scoring", + "start": 2588.0502921750303, + "end": 2588.2672875830904, + "split": "test", + "subset": "italian" + }, + { + "name": "Encoding queries", + "start": 2588.306328315055, + "end": 2588.5665530820843, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Searching corpus", + "start": 2588.5684233449865, + "end": 3060.034744929988, + "split": "test", + "subset": "portuguese" + }, + { + "name": "Scoring", + "start": 3060.0866611670936, + "end": 3060.320300724008, + "split": "test", + "subset": "portuguese" + } + ] +} \ No newline at end of file diff --git a/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/run_settings.jsonl b/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/run_settings.jsonl index 4b997bb3e7..a412c251dd 100644 --- a/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/run_settings.jsonl +++ b/results/athrael-soju__BiUltraVBERT/fdb566a46a37a84d740dfa643ac1bf3ddba83c52/run_settings.jsonl @@ -20,3 +20,7 @@ {"task": "Vidore2BioMedicalLecturesRetrieval", "splits": ["test"], "version": {"mteb": "2.26.10", "torch": "2.14.0+cu130", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "spanish"]} {"task": "Vidore3ComputerScienceRetrieval", "splits": ["test"], "version": {"mteb": "2.26.10", "torch": "2.14.0+cu130", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} {"task": "Vidore3HrRetrieval", "splits": ["test"], "version": {"mteb": "2.26.10", "torch": "2.14.0+cu130", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3TelecomRetrieval", "splits": ["test"], "version": {"mteb": "2.26.10", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3NuclearRetrieval", "splits": ["test"], "version": {"mteb": "2.26.10", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3TelecomRetrieval.v2", "splits": ["test"], "version": {"mteb": "2.26.10", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]} +{"task": "Vidore3NuclearRetrieval.v2", "splits": ["test"], "version": {"mteb": "2.26.10", "torch": "2.14.1", "sentence-transformers": "6.1.0", "flash-attn": null, "transformers": "5.20.1"}, "encode_kwargs": {"batch_size": 32}, "subsets": ["english", "french", "german", "italian", "portuguese", "spanish"]}