diff --git "a/scores/LamRA-Ret.json" "b/scores/LamRA-Ret.json" --- "a/scores/LamRA-Ret.json" +++ "b/scores/LamRA-Ret.json" @@ -1,10 +1,7 @@ { "metadata": { "model_name": "LamRA-Ret", - "report_generated_date": "2025-06-09T07:03:51.413144", - "model_size": 8.29, - "url": "https://huggingface.co/code-kunkun/LamRA-Ret", - "data_source": "TIGER-Lab" + "report_generated_date": "2025-06-28T08:54:18.459213" }, "metrics": { "image": { @@ -1017,765 +1014,287 @@ "num_data": 1000 } }, - "visdoc": { - "ViDoRe_arxivqa": { - "hit@1": 0.052, - "hit@5": 0.172, - "hit@10": 0.286, - "ndcg_linear@1": 0.052, - "ndcg_linear@5": 0.10801878370138777, - "ndcg_linear@10": 0.14373908906362196, - "ndcg_exponential@1": 0.052, - "ndcg_exponential@5": 0.10801878370138777, - "ndcg_exponential@10": 0.14373908906362196, - "precision@1": 0.052, - "precision@5": 0.03440000000000001, - "precision@10": 0.0286, - "recall@1": 0.052, - "recall@5": 0.172, - "recall@10": 0.286, - "f1@1": 0.052, - "f1@5": 0.05733333333333333, - "f1@10": 0.052, - "map@1": 0.052, - "map@5": 0.08753333333333334, - "map@10": 0.10160873015873015, - "mrr@1": 0.052, - "mrr@5": 0.08753333333333334, - "mrr@10": 0.10160873015873015, - "num_pred": 500, - "num_data": 500 - }, - "ViDoRe_docvqa": { - "hit@1": 0.13747228381374724, - "hit@5": 0.2549889135254989, - "hit@10": 0.3148558758314856, - "ndcg_linear@1": 0.13747228381374724, - "ndcg_linear@5": 0.19129511635223329, - "ndcg_linear@10": 0.2105752120749416, - "ndcg_exponential@1": 0.13747228381374724, - "ndcg_exponential@5": 0.19129511635223329, - "ndcg_exponential@10": 0.2105752120749416, - "precision@1": 0.13747228381374724, - "precision@5": 0.050997782705099776, - "precision@10": 0.031929046563192905, - "recall@1": 0.13458980044345897, - "recall@5": 0.24811529933481155, - "recall@10": 0.30842572062084256, - "f1@1": 0.1352549889135255, - "f1@5": 0.08408826945412314, - "f1@10": 0.057562319424847144, - "map@1": 0.13747228381374724, - "map@5": 0.17147080561714712, - "map@10": 0.17921637982613592, - "mrr@1": 0.13747228381374724, - "mrr@5": 0.17597930524759794, - "mrr@10": 0.1834661950515609, - "num_pred": 451, - "num_data": 451 - }, - "ViDoRe_infovqa": { - "hit@1": 0.3319838056680162, - "hit@5": 0.5748987854251012, - "hit@10": 0.6923076923076923, - "ndcg_linear@1": 0.3319838056680162, - "ndcg_linear@5": 0.46253535711806587, - "ndcg_linear@10": 0.5004886054716596, - "ndcg_exponential@1": 0.3319838056680162, - "ndcg_exponential@5": 0.46253535711806587, - "ndcg_exponential@10": 0.5004886054716596, - "precision@1": 0.3319838056680162, - "precision@5": 0.11497975708502024, - "precision@10": 0.06923076923076922, - "recall@1": 0.3319838056680162, - "recall@5": 0.5748987854251012, - "recall@10": 0.6912955465587044, - "f1@1": 0.3319838056680162, - "f1@5": 0.19163292847503377, - "f1@10": 0.12584345479082323, - "map@1": 0.3319838056680162, - "map@5": 0.4250337381916329, - "map@10": 0.4408127691022428, - "mrr@1": 0.3319838056680162, - "mrr@5": 0.4250337381916329, - "mrr@10": 0.4409252297410192, - "num_pred": 494, - "num_data": 494 - }, - "ViDoRe_tabfquad": { - "hit@1": 0.275, - "hit@5": 0.5678571428571428, - "hit@10": 0.6928571428571428, - "ndcg_linear@1": 0.275, - "ndcg_linear@5": 0.42836855281007846, - "ndcg_linear@10": 0.4688038965463907, - "ndcg_exponential@1": 0.275, - "ndcg_exponential@5": 0.42836855281007846, - "ndcg_exponential@10": 0.4688038965463907, - "precision@1": 0.275, - "precision@5": 0.11357142857142856, - "precision@10": 0.06928571428571428, - "recall@1": 0.275, - "recall@5": 0.5678571428571428, - "recall@10": 0.6928571428571428, - "f1@1": 0.275, - "f1@5": 0.1892857142857143, - "f1@10": 0.125974025974026, - "map@1": 0.275, - "map@5": 0.3823214285714286, - "map@10": 0.3990192743764172, - "mrr@1": 0.275, - "mrr@5": 0.3823214285714286, - "mrr@10": 0.3990192743764172, - "num_pred": 280, - "num_data": 280 - }, - "ViDoRe_tatdqa": { - "hit@1": 0.06804374240583232, - "hit@5": 0.1591737545565006, - "hit@10": 0.22053462940461727, - "ndcg_linear@1": 0.06804374240583232, - "ndcg_linear@5": 0.11430035232416713, - "ndcg_linear@10": 0.13412008753307786, - "ndcg_exponential@1": 0.06804374240583232, - "ndcg_exponential@5": 0.11430035232416713, - "ndcg_exponential@10": 0.13412008753307786, - "precision@1": 0.06804374240583232, - "precision@5": 0.03183475091130013, - "precision@10": 0.022053462940461725, - "recall@1": 0.06804374240583232, - "recall@5": 0.1586269744835966, - "recall@10": 0.21998784933171325, - "f1@1": 0.06804374240583232, - "f1@5": 0.052936411502632646, - "f1@10": 0.04004749806693913, - "map@1": 0.06804374240583232, - "map@5": 0.09965978128797083, - "map@10": 0.1078442689347914, - "mrr@1": 0.06804374240583232, - "mrr@5": 0.09975698663426488, - "mrr@10": 0.10795362494937222, - "num_pred": 1646, - "num_data": 1646 - }, - "ViDoRe_shiftproject": { - "hit@1": 0.07, - "hit@5": 0.17, - "hit@10": 0.27, - "ndcg_linear@1": 0.07, - "ndcg_linear@5": 0.12035565121611999, - "ndcg_linear@10": 0.1523429715395014, - "ndcg_exponential@1": 0.07, - "ndcg_exponential@5": 0.12035565121611999, - "ndcg_exponential@10": 0.1523429715395014, - "precision@1": 0.07, - "precision@5": 0.034, - "precision@10": 0.027000000000000003, - "recall@1": 0.07, - "recall@5": 0.17, - "recall@10": 0.27, - "f1@1": 0.07, - "f1@5": 0.05666666666666667, - "f1@10": 0.049090909090909095, - "map@1": 0.07, - "map@5": 0.10400000000000001, - "map@10": 0.116984126984127, - "mrr@1": 0.07, - "mrr@5": 0.10400000000000001, - "mrr@10": 0.116984126984127, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.05, - "hit@5": 0.15, - "hit@10": 0.22, - "ndcg_linear@1": 0.05, - "ndcg_linear@5": 0.1031574868850601, - "ndcg_linear@10": 0.12568086090486966, - "ndcg_exponential@1": 0.05, - "ndcg_exponential@5": 0.1031574868850601, - "ndcg_exponential@10": 0.12568086090486966, - "precision@1": 0.05, - "precision@5": 0.030000000000000006, - "precision@10": 0.022, - "recall@1": 0.05, - "recall@5": 0.15, - "recall@10": 0.22, - "f1@1": 0.05, - "f1@5": 0.05, - "f1@10": 0.04, - "map@1": 0.05, - "map@5": 0.0875, - "map@10": 0.09673412698412699, - "mrr@1": 0.05, - "mrr@5": 0.0875, - "mrr@10": 0.09673412698412699, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.18, - "hit@5": 0.31, - "hit@10": 0.41, - "ndcg_linear@1": 0.18, - "ndcg_linear@5": 0.24771924937667159, - "ndcg_linear@10": 0.2802583416144934, - "ndcg_exponential@1": 0.18, - "ndcg_exponential@5": 0.24771924937667159, - "ndcg_exponential@10": 0.2802583416144934, - "precision@1": 0.18, - "precision@5": 0.062, - "precision@10": 0.040999999999999995, - "recall@1": 0.18, - "recall@5": 0.31, - "recall@10": 0.41, - "f1@1": 0.18, - "f1@5": 0.10333333333333336, - "f1@10": 0.07454545454545455, - "map@1": 0.18, - "map@5": 0.22699999999999998, - "map@10": 0.24053968253968258, - "mrr@1": 0.18, - "mrr@5": 0.22699999999999998, - "mrr@10": 0.24053968253968258, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.09, - "hit@5": 0.23, - "hit@10": 0.27, - "ndcg_linear@1": 0.09, - "ndcg_linear@5": 0.16359030940399763, - "ndcg_linear@10": 0.17675975017719422, - "ndcg_exponential@1": 0.09, - "ndcg_exponential@5": 0.16359030940399763, - "ndcg_exponential@10": 0.17675975017719422, - "precision@1": 0.09, - "precision@5": 0.046000000000000006, - "precision@10": 0.027000000000000007, - "recall@1": 0.09, - "recall@5": 0.23, - "recall@10": 0.27, - "f1@1": 0.09, - "f1@5": 0.07666666666666667, - "f1@10": 0.04909090909090908, - "map@1": 0.09, - "map@5": 0.14150000000000001, - "map@10": 0.14708333333333334, - "mrr@1": 0.09, - "mrr@5": 0.14150000000000001, - "mrr@10": 0.14708333333333334, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.14, - "hit@5": 0.36, - "hit@10": 0.4, - "ndcg_linear@1": 0.14, - "ndcg_linear@5": 0.2593663913604869, - "ndcg_linear@10": 0.2717200597145647, - "ndcg_exponential@1": 0.14, - "ndcg_exponential@5": 0.2593663913604869, - "ndcg_exponential@10": 0.2717200597145647, - "precision@1": 0.14, - "precision@5": 0.07200000000000001, - "precision@10": 0.04, - "recall@1": 0.14, - "recall@5": 0.36, - "recall@10": 0.4, - "f1@1": 0.14, - "f1@5": 0.12000000000000002, - "f1@10": 0.07272727272727274, - "map@1": 0.14, - "map@5": 0.22566666666666665, - "map@10": 0.2304444444444444, - "mrr@1": 0.14, - "mrr@5": 0.22566666666666665, - "mrr@10": 0.2304444444444444, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_esg_reports_human_labeled_v2": { - "hit@1": 0.038461538461538464, - "hit@5": 0.17307692307692307, - "hit@10": 0.23076923076923078, - "ndcg_linear@1": 0.038461538461538464, - "ndcg_linear@5": 0.07581945100217463, - "ndcg_linear@10": 0.08884346676199335, - "ndcg_exponential@1": 0.038461538461538464, - "ndcg_exponential@5": 0.07527592503218486, - "ndcg_exponential@10": 0.0882008820280547, - "precision@1": 0.038461538461538464, - "precision@5": 0.05000000000000001, - "precision@10": 0.038461538461538464, - "recall@1": 0.020833333333333332, - "recall@5": 0.10961538461538461, - "recall@10": 0.14761904761904762, - "f1@1": 0.022189349112426034, - "f1@5": 0.057642031171442946, - "f1@10": 0.054193845370315956, - "map@1": 0.038461538461538464, - "map@5": 0.05387820512820513, - "map@10": 0.05896291208791209, - "mrr@1": 0.038461538461538464, - "mrr@5": 0.07724358974358975, - "mrr@10": 0.08392094017094018, - "num_pred": 52, - "num_data": 52 - }, - "ViDoRe_biomedical_lectures_v2": { - "hit@1": 0.08125, - "hit@5": 0.275, - "hit@10": 0.3625, - "ndcg_linear@1": 0.08125, - "ndcg_linear@5": 0.11996797729916849, - "ndcg_linear@10": 0.14608837569913175, - "ndcg_exponential@1": 0.08125, - "ndcg_exponential@5": 0.11996797729916849, - "ndcg_exponential@10": 0.14608837569913175, - "precision@1": 0.08125, - "precision@5": 0.06250000000000001, - "precision@10": 0.05, - "recall@1": 0.04401041666666666, - "recall@5": 0.15211647727272729, - "recall@10": 0.22817048253470668, - "f1@1": 0.05030055014430015, - "f1@5": 0.0768830821955822, - "f1@10": 0.0729347127545657, - "map@1": 0.08125, - "map@5": 0.08664583333333334, - "map@10": 0.09615652754157218, - "mrr@1": 0.08125, - "mrr@5": 0.14697916666666666, - "mrr@10": 0.15870287698412697, - "num_pred": 160, - "num_data": 160 - }, - "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.096875, - "hit@5": 0.296875, - "hit@10": 0.3796875, - "ndcg_linear@1": 0.096875, - "ndcg_linear@5": 0.1327817509891042, - "ndcg_linear@10": 0.15578005969658662, - "ndcg_exponential@1": 0.096875, - "ndcg_exponential@5": 0.1327817509891042, - "ndcg_exponential@10": 0.15578005969658662, - "precision@1": 0.096875, - "precision@5": 0.06625, - "precision@10": 0.05015625, - "recall@1": 0.05542025862068965, - "recall@5": 0.1645906222010748, - "recall@10": 0.23159770265864968, - "f1@1": 0.06360192325036075, - "f1@5": 0.08230852582874641, - "f1@10": 0.07338045655325068, - "map@1": 0.096875, - "map@5": 0.09772092013888889, - "map@10": 0.10609532519605062, - "mrr@1": 0.096875, - "mrr@5": 0.16531250000000003, - "mrr@10": 0.1761204117063492, - "num_pred": 640, - "num_data": 640 - }, - "ViDoRe_economics_reports_v2": { - "hit@1": 0.1896551724137931, - "hit@5": 0.5, - "hit@10": 0.6551724137931034, - "ndcg_linear@1": 0.1896551724137931, - "ndcg_linear@5": 0.2016813712542325, - "ndcg_linear@10": 0.2099594095539958, - "ndcg_exponential@1": 0.1896551724137931, - "ndcg_exponential@5": 0.2016813712542325, - "ndcg_exponential@10": 0.2099594095539958, - "precision@1": 0.1896551724137931, - "precision@5": 0.19999999999999998, - "precision@10": 0.18620689655172412, - "recall@1": 0.014798040733571795, - "recall@5": 0.07392554687834921, - "recall@10": 0.16365628632168921, - "f1@1": 0.026836516108849087, - "f1@5": 0.09510056346464353, - "f1@10": 0.14028471558301375, - "map@1": 0.1896551724137931, - "map@5": 0.13538793103448277, - "map@10": 0.11604274180936743, - "mrr@1": 0.1896551724137931, - "mrr@5": 0.30517241379310345, - "mrr@10": 0.32358374384236455, - "num_pred": 58, - "num_data": 58 - }, - "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.1939655172413793, - "hit@5": 0.46551724137931033, - "hit@10": 0.5948275862068966, - "ndcg_linear@1": 0.1939655172413793, - "ndcg_linear@5": 0.19075341400656756, - "ndcg_linear@10": 0.19262260915535673, - "ndcg_exponential@1": 0.1939655172413793, - "ndcg_exponential@5": 0.19075341400656748, - "ndcg_exponential@10": 0.19262260915535673, - "precision@1": 0.1939655172413793, - "precision@5": 0.18448275862068964, - "precision@10": 0.16724137931034483, - "recall@1": 0.016014854892863833, - "recall@5": 0.07300449328968175, - "recall@10": 0.14033065985008555, - "f1@1": 0.029036373118143485, - "f1@5": 0.0903316153795555, - "f1@10": 0.12402634113864117, - "map@1": 0.1939655172413793, - "map@5": 0.12855603448275862, - "map@10": 0.1083281829653956, - "mrr@1": 0.1939655172413793, - "mrr@5": 0.2959051724137931, - "mrr@10": 0.3119937055281883, - "num_pred": 232, - "num_data": 232 - }, - "ViDoRe_esg_reports_v2": { - "hit@1": 0.03508771929824561, - "hit@5": 0.21052631578947367, - "hit@10": 0.2631578947368421, - "ndcg_linear@1": 0.03508771929824561, - "ndcg_linear@5": 0.07661358667909549, - "ndcg_linear@10": 0.10141753829490283, - "ndcg_exponential@1": 0.03508771929824561, - "ndcg_exponential@5": 0.07661358667909549, - "ndcg_exponential@10": 0.10141753829490285, - "precision@1": 0.03508771929824561, - "precision@5": 0.06315789473684212, - "precision@10": 0.05789473684210526, - "recall@1": 0.007309941520467836, - "recall@5": 0.11549707602339182, - "recall@10": 0.1769005847953216, - "f1@1": 0.011470985155195682, - "f1@5": 0.07200625312080422, - "f1@10": 0.07855740750477593, - "map@1": 0.03508771929824561, - "map@5": 0.045, - "map@10": 0.05800177527151211, - "mrr@1": 0.03508771929824561, - "mrr@5": 0.0783625730994152, - "mrr@10": 0.08567251461988304, - "num_pred": 57, - "num_data": 57 - }, - "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.043859649122807015, - "hit@5": 0.16228070175438597, - "hit@10": 0.25877192982456143, - "ndcg_linear@1": 0.043859649122807015, - "ndcg_linear@5": 0.05924300567334283, - "ndcg_linear@10": 0.09666072127808396, - "ndcg_exponential@1": 0.043859649122807015, - "ndcg_exponential@5": 0.05924300567334283, - "ndcg_exponential@10": 0.09666072127808396, - "precision@1": 0.043859649122807015, - "precision@5": 0.04912280701754387, - "precision@10": 0.054385964912280704, - "recall@1": 0.008406432748538011, - "recall@5": 0.07328216374269006, - "recall@10": 0.17160087719298245, - "f1@1": 0.013420302893987103, - "f1@5": 0.05224007874936668, - "f1@10": 0.07439770268717638, - "map@1": 0.043859649122807015, - "map@5": 0.03577850877192982, - "map@10": 0.0544104619186856, - "mrr@1": 0.043859649122807015, - "mrr@5": 0.07339181286549706, - "mrr@10": 0.08703181565023671, - "num_pred": 228, - "num_data": 228 - }, - "VisRAG_ArxivQA": { - "hit@1": 0.012254901960784314, - "hit@5": 0.02696078431372549, - "hit@10": 0.03553921568627451, - "ndcg_linear@1": 0.012254901960784314, - "ndcg_linear@5": 0.020155243188820424, - "ndcg_linear@10": 0.022825943867728897, - "ndcg_exponential@1": 0.012254901960784314, - "ndcg_exponential@5": 0.020155243188820424, - "ndcg_exponential@10": 0.022825943867728897, - "precision@1": 0.012254901960784314, - "precision@5": 0.005392156862745099, - "precision@10": 0.0035539215686274514, - "recall@1": 0.012254901960784314, - "recall@5": 0.02696078431372549, - "recall@10": 0.03553921568627451, - "f1@1": 0.012254901960784314, - "f1@5": 0.008986928104575164, - "f1@10": 0.006461675579322638, - "map@1": 0.012254901960784314, - "map@5": 0.01787173202614379, - "map@10": 0.018913398692810456, - "mrr@1": 0.012254901960784314, - "mrr@5": 0.01787173202614379, - "mrr@10": 0.018913398692810456, - "num_pred": 816, - "num_data": 816 - }, - "VisRAG_ChartQA": { - "hit@1": 0.31746031746031744, - "hit@5": 0.49206349206349204, - "hit@10": 0.5396825396825397, - "ndcg_linear@1": 0.31746031746031744, - "ndcg_linear@5": 0.41335716785860044, - "ndcg_linear@10": 0.42995633781970166, - "ndcg_exponential@1": 0.31746031746031744, - "ndcg_exponential@5": 0.41335716785860044, - "ndcg_exponential@10": 0.42995633781970166, - "precision@1": 0.31746031746031744, - "precision@5": 0.09841269841269841, - "precision@10": 0.05396825396825397, - "recall@1": 0.31746031746031744, - "recall@5": 0.49206349206349204, - "recall@10": 0.5396825396825397, - "f1@1": 0.31746031746031744, - "f1@5": 0.1640211640211641, - "f1@10": 0.09812409812409813, - "map@1": 0.31746031746031744, - "map@5": 0.3867724867724867, - "map@10": 0.39433106575963717, - "mrr@1": 0.31746031746031744, - "mrr@5": 0.3867724867724867, - "mrr@10": 0.39433106575963717, - "num_pred": 63, - "num_data": 63 + "video": { + "K700": { + "hit@1": 0.423, + "hit@5": 0.653, + "hit@10": 0.731, + "ndcg_linear@1": 0.423, + "ndcg_linear@5": 0.546371218016443, + "ndcg_linear@10": 0.5718997969605862, + "ndcg_exponential@1": 0.423, + "ndcg_exponential@5": 0.546371218016443, + "ndcg_exponential@10": 0.5718997969605862, + "precision@1": 0.423, + "precision@5": 0.13059999999999997, + "precision@10": 0.0731, + "recall@1": 0.423, + "recall@5": 0.653, + "recall@10": 0.731, + "f1@1": 0.423, + "f1@5": 0.21766666666666673, + "f1@10": 0.13290909090909092, + "map@1": 0.423, + "map@5": 0.5107999999999999, + "map@10": 0.5215190476190475, + "mrr@1": 0.423, + "mrr@5": 0.5107999999999999, + "mrr@10": 0.5215190476190475, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_MP-DocVQA": { - "hit@1": 0.2436548223350254, - "hit@5": 0.41624365482233505, - "hit@10": 0.48223350253807107, - "ndcg_linear@1": 0.2436548223350254, - "ndcg_linear@5": 0.33378850914695807, - "ndcg_linear@10": 0.3551570419412752, - "ndcg_exponential@1": 0.2436548223350254, - "ndcg_exponential@5": 0.33378850914695807, - "ndcg_exponential@10": 0.3551570419412752, - "precision@1": 0.2436548223350254, - "precision@5": 0.08324873096446701, - "precision@10": 0.048223350253807105, - "recall@1": 0.2436548223350254, - "recall@5": 0.41624365482233505, - "recall@10": 0.48223350253807107, - "f1@1": 0.2436548223350254, - "f1@5": 0.13874788494077836, - "f1@10": 0.08767881864328564, - "map@1": 0.2436548223350254, - "map@5": 0.30654258319232935, - "map@10": 0.31538353073886066, - "mrr@1": 0.2436548223350254, - "mrr@5": 0.30654258319232935, - "mrr@10": 0.31538353073886066, - "num_pred": 591, - "num_data": 591 + "SmthSmthV2": { + "hit@1": 0.363, + "hit@5": 0.602, + "hit@10": 0.698, + "ndcg_linear@1": 0.363, + "ndcg_linear@5": 0.4890606806277336, + "ndcg_linear@10": 0.5200630033328641, + "ndcg_exponential@1": 0.363, + "ndcg_exponential@5": 0.4890606806277336, + "ndcg_exponential@10": 0.5200630033328641, + "precision@1": 0.363, + "precision@5": 0.1204, + "precision@10": 0.06979999999999999, + "recall@1": 0.363, + "recall@5": 0.602, + "recall@10": 0.698, + "f1@1": 0.363, + "f1@5": 0.20066666666666674, + "f1@10": 0.12690909090909094, + "map@1": 0.363, + "map@5": 0.45156666666666667, + "map@10": 0.4643380952380952, + "mrr@1": 0.363, + "mrr@5": 0.45156666666666667, + "mrr@10": 0.4643380952380952, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_SlideVQA": { - "hit@1": 0.44964028776978415, - "hit@5": 0.7086330935251799, - "hit@10": 0.7859712230215827, - "ndcg_linear@1": 0.44964028776978415, - "ndcg_linear@5": 0.5648847466987875, - "ndcg_linear@10": 0.5955590007686733, - "ndcg_exponential@1": 0.44964028776978415, - "ndcg_exponential@5": 0.5648847466987875, - "ndcg_exponential@10": 0.5955590007686733, - "precision@1": 0.44964028776978415, - "precision@5": 0.1723021582733813, - "precision@10": 0.09766187050359712, - "recall@1": 0.36960431654676257, - "recall@5": 0.6735611510791367, - "recall@10": 0.7616906474820144, - "f1@1": 0.3962829736211031, - "f1@5": 0.2692702980472765, - "f1@10": 0.17105406583823854, - "map@1": 0.44964028776978415, - "map@5": 0.5169964028776978, - "map@10": 0.5313963115222108, - "mrr@1": 0.44964028776978415, - "mrr@5": 0.5481115107913669, - "mrr@10": 0.5588072399223478, - "num_pred": 556, - "num_data": 556 + "HMDB51": { + "hit@1": 0.405, + "hit@5": 0.658, + "hit@10": 0.783, + "ndcg_linear@1": 0.405, + "ndcg_linear@5": 0.5405643217050999, + "ndcg_linear@10": 0.5815107823708526, + "ndcg_exponential@1": 0.405, + "ndcg_exponential@5": 0.5405643217050999, + "ndcg_exponential@10": 0.5815107823708526, + "precision@1": 0.405, + "precision@5": 0.1316, + "precision@10": 0.0783, + "recall@1": 0.405, + "recall@5": 0.658, + "recall@10": 0.783, + "f1@1": 0.405, + "f1@5": 0.21933333333333338, + "f1@10": 0.1423636363636364, + "map@1": 0.405, + "map@5": 0.5013833333333333, + "map@10": 0.5185896825396826, + "mrr@1": 0.405, + "mrr@5": 0.5013833333333333, + "mrr@10": 0.5185896825396826, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_InfoVQA": { - "hit@1": 0.4415041782729805, - "hit@5": 0.6713091922005571, - "hit@10": 0.754874651810585, - "ndcg_linear@1": 0.4415041782729805, - "ndcg_linear@5": 0.5633673128694291, - "ndcg_linear@10": 0.5903761050947809, - "ndcg_exponential@1": 0.4415041782729805, - "ndcg_exponential@5": 0.5633673128694291, - "ndcg_exponential@10": 0.5903761050947809, - "precision@1": 0.4415041782729805, - "precision@5": 0.13426183844011141, - "precision@10": 0.07548746518105848, - "recall@1": 0.4415041782729805, - "recall@5": 0.6713091922005571, - "recall@10": 0.754874651810585, - "f1@1": 0.4415041782729805, - "f1@5": 0.2237697307335191, - "f1@10": 0.13724993669283364, - "map@1": 0.4415041782729805, - "map@5": 0.5274373259052925, - "map@10": 0.5385661228279612, - "mrr@1": 0.4415041782729805, - "mrr@5": 0.5274373259052925, - "mrr@10": 0.5385661228279612, - "num_pred": 718, - "num_data": 718 + "UCF101": { + "hit@1": 0.604, + "hit@5": 0.852, + "hit@10": 0.89, + "ndcg_linear@1": 0.604, + "ndcg_linear@5": 0.7417818963613594, + "ndcg_linear@10": 0.7541096692471368, + "ndcg_exponential@1": 0.604, + "ndcg_exponential@5": 0.7417818963613594, + "ndcg_exponential@10": 0.7541096692471368, + "precision@1": 0.604, + "precision@5": 0.17040000000000002, + "precision@10": 0.08900000000000001, + "recall@1": 0.604, + "recall@5": 0.852, + "recall@10": 0.89, + "f1@1": 0.604, + "f1@5": 0.284, + "f1@10": 0.1618181818181818, + "map@1": 0.604, + "map@5": 0.7046166666666666, + "map@10": 0.7097305555555556, + "mrr@1": 0.604, + "mrr@5": 0.7046166666666666, + "mrr@10": 0.7097305555555556, + "num_pred": 1000, + "num_data": 1000 }, - "VisRAG_PlotQA": { - "hit@1": 0.2410196987253766, - "hit@5": 0.4380069524913094, - "hit@10": 0.5365005793742758, - "ndcg_linear@1": 0.2410196987253766, - "ndcg_linear@5": 0.3455373205552205, - "ndcg_linear@10": 0.3774838122200841, - "ndcg_exponential@1": 0.2410196987253766, - "ndcg_exponential@5": 0.3455373205552205, - "ndcg_exponential@10": 0.3774838122200841, - "precision@1": 0.2410196987253766, - "precision@5": 0.08760139049826188, - "precision@10": 0.053650057937427574, - "recall@1": 0.2410196987253766, - "recall@5": 0.4380069524913094, - "recall@10": 0.5365005793742758, - "f1@1": 0.2410196987253766, - "f1@5": 0.14600231749710316, - "f1@10": 0.09754555988623198, - "map@1": 0.2410196987253766, - "map@5": 0.3148319814600232, - "map@10": 0.32806792473652263, - "mrr@1": 0.2410196987253766, - "mrr@5": 0.3148319814600232, - "mrr@10": 0.32806792473652263, - "num_pred": 863, - "num_data": 863 + "Breakfast": { + "hit@1": 0.16859122401847576, + "hit@5": 0.6235565819861432, + "hit@10": 1.0, + "ndcg_linear@1": 0.16859122401847576, + "ndcg_linear@5": 0.400691767704696, + "ndcg_linear@10": 0.5235817090775436, + "ndcg_exponential@1": 0.16859122401847576, + "ndcg_exponential@5": 0.400691767704696, + "ndcg_exponential@10": 0.5235817090775436, + "precision@1": 0.16859122401847576, + "precision@5": 0.12471131639722864, + "precision@10": 0.1, + "recall@1": 0.16859122401847576, + "recall@5": 0.6235565819861432, + "recall@10": 1.0, + "f1@1": 0.16859122401847576, + "f1@5": 0.20785219399538113, + "f1@10": 0.1818181818181818, + "map@1": 0.16859122401847576, + "map@5": 0.32763664357197847, + "map@10": 0.37902140840939913, + "mrr@1": 0.16859122401847576, + "mrr@5": 0.32763664357197847, + "mrr@10": 0.37902140840939913, + "num_pred": 433, + "num_data": 433 }, - "ViDoSeek-page": { - "hit@1": 0.040280210157618214, - "hit@5": 0.18563922942206654, - "hit@10": 0.287215411558669, - "ndcg_linear@1": 0.040280210157618214, - "ndcg_linear@5": 0.11294100510293231, - "ndcg_linear@10": 0.14594536433240723, - "ndcg_exponential@1": 0.040280210157618214, - "ndcg_exponential@5": 0.11294100510293234, - "ndcg_exponential@10": 0.14594536433240726, - "precision@1": 0.040280210157618214, - "precision@5": 0.03712784588441331, - "precision@10": 0.028721541155866902, - "recall@1": 0.040280210157618214, - "recall@5": 0.18563922942206654, - "recall@10": 0.287215411558669, - "f1@1": 0.040280210157618214, - "f1@5": 0.06187974314068886, - "f1@10": 0.052220983919758, - "map@1": 0.040280210157618214, - "map@5": 0.08917104495037945, - "map@10": 0.10288862480193478, - "mrr@1": 0.040280210157618214, - "mrr@5": 0.08917104495037945, - "mrr@10": 0.10288862480193478, - "num_pred": 1142, - "num_data": 1142 + "MVBench": { + "hit@1": 0.372, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.372, + "ndcg_linear@5": 0.7114371453270959, + "ndcg_linear@10": 0.7114371453270959, + "ndcg_exponential@1": 0.372, + "ndcg_exponential@5": 0.7114371453270959, + "ndcg_exponential@10": 0.7114371453270959, + "precision@1": 0.372, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.372, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.372, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.372, + "map@5": 0.6143833333333334, + "map@10": 0.6143833333333334, + "mrr@1": 0.372, + "mrr@5": 0.6143833333333334, + "mrr@10": 0.6143833333333334, + "num_pred": 4000, + "num_data": 4000 }, - "ViDoSeek-doc": { - "hit@1": 0.4982486865148862, - "hit@5": 0.6567425569176882, - "hit@10": 0.7338003502626971, - "ndcg_linear@1": 0.3429655575014594, - "ndcg_linear@5": 0.3714170601904285, - "ndcg_linear@10": 0.35019901577656104, - "ndcg_exponential@1": 0.23204903677758318, - "ndcg_exponential@5": 0.3146728814563305, - "ndcg_exponential@10": 0.3162182310328309, - "precision@1": 0.4982486865148862, - "precision@5": 0.4001751313485114, - "precision@10": 0.3430823117338004, - "recall@1": 0.02844268284063671, - "recall@5": 0.11354378337205699, - "recall@10": 0.19116020046136178, - "f1@1": 0.05349597259677307, - "f1@5": 0.1736704243149188, - "f1@10": 0.24068949848035567, - "map@1": 0.4982486865148862, - "map@5": 0.3603765323992995, - "map@10": 0.2910387701490404, - "mrr@1": 0.4982486865148862, - "mrr@5": 0.5563776999416229, - "mrr@10": 0.5669967753592973, - "num_pred": 1142, - "num_data": 1142 + "Video-MME": { + "hit@1": 0.34074074074074073, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.34074074074074073, + "ndcg_linear@5": 0.6905732385187221, + "ndcg_linear@10": 0.6905732385187221, + "ndcg_exponential@1": 0.34074074074074073, + "ndcg_exponential@5": 0.6905732385187221, + "ndcg_exponential@10": 0.6905732385187221, + "precision@1": 0.34074074074074073, + "precision@5": 0.19999999999999996, + "precision@10": 0.09999999999999998, + "recall@1": 0.34074074074074073, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.34074074074074073, + "f1@5": 0.3333333333333334, + "f1@10": 0.18181818181818188, + "map@1": 0.34074074074074073, + "map@5": 0.5870987654320988, + "map@10": 0.5870987654320988, + "mrr@1": 0.34074074074074073, + "mrr@5": 0.5870987654320988, + "mrr@10": 0.5870987654320988, + "num_pred": 2700, + "num_data": 2700 }, - "MMLongBench-page": { - "hit@1": 0.059665871121718374, - "hit@5": 0.14797136038186157, - "hit@10": 0.2159904534606205, - "ndcg_linear@1": 0.059665871121718374, - "ndcg_linear@5": 0.07951086728623444, - "ndcg_linear@10": 0.09880430458699743, - "ndcg_exponential@1": 0.059665871121718374, - "ndcg_exponential@5": 0.07951086728623444, - "ndcg_exponential@10": 0.09880430458699746, - "precision@1": 0.059665871121718374, - "precision@5": 0.03293556085918855, - "precision@10": 0.0256563245823389, - "recall@1": 0.038929139674963066, - "recall@5": 0.10381908929044967, - "recall@10": 0.15813207940296245, - "f1@1": 0.04444323907331067, - "f1@5": 0.04737172798031987, - "f1@10": 0.042217267935243885, - "map@1": 0.059665871121718374, - "map@5": 0.06207769822328295, - "map@10": 0.06994553516182395, - "mrr@1": 0.059665871121718374, - "mrr@5": 0.08904136833731105, - "mrr@10": 0.09803860287153843, - "num_pred": 838, - "num_data": 838 + "NExTQA": { + "hit@1": 0.4371788883699206, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.4371788883699206, + "ndcg_linear@5": 0.7269325524352686, + "ndcg_linear@10": 0.7269325524352686, + "ndcg_exponential@1": 0.4371788883699206, + "ndcg_exponential@5": 0.7269325524352686, + "ndcg_exponential@10": 0.7269325524352686, + "precision@1": 0.4371788883699206, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.4371788883699206, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.4371788883699206, + "f1@5": 0.3333333333333333, + "f1@10": 0.1818181818181818, + "map@1": 0.4371788883699206, + "map@5": 0.6369803829985987, + "map@10": 0.6369803829985987, + "mrr@1": 0.4371788883699206, + "mrr@5": 0.6367857698894598, + "mrr@10": 0.6367857698894598, + "num_pred": 8564, + "num_data": 8564 + }, + "EgoSchema": { + "hit@1": 0.448, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.448, + "ndcg_linear@5": 0.7459721375093364, + "ndcg_linear@10": 0.7459721375093364, + "ndcg_exponential@1": 0.448, + "ndcg_exponential@5": 0.7459721375093364, + "ndcg_exponential@10": 0.7459721375093364, + "precision@1": 0.448, + "precision@5": 0.2, + "precision@10": 0.1, + "recall@1": 0.448, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.448, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.448, + "map@5": 0.6608999999999999, + "map@10": 0.6608999999999999, + "mrr@1": 0.448, + "mrr@5": 0.6608999999999999, + "mrr@10": 0.6608999999999999, + "num_pred": 500, + "num_data": 500 + }, + "ActivityNetQA": { + "hit@1": 0.532, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.532, + "ndcg_linear@5": 0.8272751246714422, + "ndcg_linear@10": 0.8272751246714422, + "ndcg_exponential@1": 0.532, + "ndcg_exponential@5": 0.8272751246714422, + "ndcg_exponential@10": 0.8272751246714422, + "precision@1": 0.532, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.532, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.532, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.532, + "map@5": 0.766, + "map@10": 0.766, + "mrr@1": 0.532, + "mrr@5": 0.766, + "mrr@10": 0.766, + "num_pred": 1000, + "num_data": 1000 }, - "MMLongBench-doc": { - "hit@1": 0.34725536992840095, - "hit@5": 0.5238663484486874, - "hit@10": 0.5883054892601431, - "ndcg_linear@1": 0.24701670644391405, - "ndcg_linear@5": 0.2758146276737757, - "ndcg_linear@10": 0.2683418556525889, - "ndcg_exponential@1": 0.17541766109785203, - "ndcg_exponential@5": 0.2302031604572782, - "ndcg_exponential@10": 0.23834477001816015, - "precision@1": 0.34725536992840095, - "precision@5": 0.31169451073985677, - "precision@10": 0.27792362768496426, - "recall@1": 0.011458219705852956, - "recall@5": 0.0473748134486053, - "recall@10": 0.08146579365576306, - "f1@1": 0.021977357536621576, - "f1@5": 0.0790594246656723, - "f1@10": 0.11819035378881726, - "map@1": 0.34725536992840095, - "map@5": 0.2678878281622912, - "map@10": 0.22411547713755348, - "mrr@1": 0.34725536992840095, - "mrr@5": 0.41585123309466987, - "mrr@10": 0.4247144561882032, - "num_pred": 838, - "num_data": 838 - } - }, - "video": { "DiDeMo": { "hit@1": 0.24800796812749004, "hit@5": 0.4711155378486056, @@ -1804,481 +1323,875 @@ "num_pred": 1004, "num_data": 1004 }, - "MSR-VTT": { - "hit@1": 0.221, - "hit@5": 0.441, - "hit@10": 0.558, - "ndcg_linear@1": 0.221, - "ndcg_linear@5": 0.3378462330246669, - "ndcg_linear@10": 0.37615489678599645, - "ndcg_exponential@1": 0.221, - "ndcg_exponential@5": 0.3378462330246669, - "ndcg_exponential@10": 0.37615489678599645, - "precision@1": 0.221, - "precision@5": 0.08819999999999999, - "precision@10": 0.055799999999999995, - "recall@1": 0.221, - "recall@5": 0.441, - "recall@10": 0.558, - "f1@1": 0.221, - "f1@5": 0.14700000000000002, - "f1@10": 0.10145454545454546, - "map@1": 0.221, - "map@5": 0.30355, - "map@10": 0.31964126984126984, - "mrr@1": 0.221, - "mrr@5": 0.30355, - "mrr@10": 0.31964126984126984, - "num_pred": 1000, - "num_data": 1000 + "MSR-VTT": { + "hit@1": 0.221, + "hit@5": 0.441, + "hit@10": 0.558, + "ndcg_linear@1": 0.221, + "ndcg_linear@5": 0.3378462330246669, + "ndcg_linear@10": 0.37615489678599645, + "ndcg_exponential@1": 0.221, + "ndcg_exponential@5": 0.3378462330246669, + "ndcg_exponential@10": 0.37615489678599645, + "precision@1": 0.221, + "precision@5": 0.08819999999999999, + "precision@10": 0.055799999999999995, + "recall@1": 0.221, + "recall@5": 0.441, + "recall@10": 0.558, + "f1@1": 0.221, + "f1@5": 0.14700000000000002, + "f1@10": 0.10145454545454546, + "map@1": 0.221, + "map@5": 0.30355, + "map@10": 0.31964126984126984, + "mrr@1": 0.221, + "mrr@5": 0.30355, + "mrr@10": 0.31964126984126984, + "num_pred": 1000, + "num_data": 1000 + }, + "MSVD": { + "hit@1": 0.46119402985074626, + "hit@5": 0.7, + "hit@10": 0.7850746268656716, + "ndcg_linear@1": 0.46119402985074626, + "ndcg_linear@5": 0.5909143508859, + "ndcg_linear@10": 0.6184940826509501, + "ndcg_exponential@1": 0.46119402985074626, + "ndcg_exponential@5": 0.5909143508859, + "ndcg_exponential@10": 0.6184940826509501, + "precision@1": 0.46119402985074626, + "precision@5": 0.13999999999999999, + "precision@10": 0.07850746268656715, + "recall@1": 0.46119402985074626, + "recall@5": 0.7, + "recall@10": 0.7850746268656716, + "f1@1": 0.46119402985074626, + "f1@5": 0.2333333333333334, + "f1@10": 0.14274084124830397, + "map@1": 0.46119402985074626, + "map@5": 0.5543781094527364, + "map@10": 0.5658019426676143, + "mrr@1": 0.46119402985074626, + "mrr@5": 0.5543781094527364, + "mrr@10": 0.5658019426676143, + "num_pred": 670, + "num_data": 670 + }, + "VATEX": { + "hit@1": 0.19138008039303261, + "hit@5": 0.41692719964269764, + "hit@10": 0.5225547119249665, + "ndcg_linear@1": 0.19138008039303261, + "ndcg_linear@5": 0.31066559826092355, + "ndcg_linear@10": 0.3449304405187584, + "ndcg_exponential@1": 0.19138008039303261, + "ndcg_exponential@5": 0.31066559826092355, + "ndcg_exponential@10": 0.3449304405187584, + "precision@1": 0.19138008039303261, + "precision@5": 0.08338543992853951, + "precision@10": 0.05225547119249665, + "recall@1": 0.19138008039303261, + "recall@5": 0.41692719964269764, + "recall@10": 0.5225547119249665, + "f1@1": 0.19138008039303261, + "f1@5": 0.13897573321423257, + "f1@10": 0.09500994762272119, + "map@1": 0.19138008039303261, + "map@5": 0.27535730236712824, + "map@10": 0.28956069177708305, + "mrr@1": 0.19138008039303261, + "mrr@5": 0.27535730236712824, + "mrr@10": 0.28956069177708305, + "num_pred": 4478, + "num_data": 4478 + }, + "YouCook2": { + "hit@1": 0.09153821956590123, + "hit@5": 0.21358917898710286, + "hit@10": 0.29285938974520287, + "ndcg_linear@1": 0.09153821956590123, + "ndcg_linear@5": 0.15446626761982413, + "ndcg_linear@10": 0.18024480954320224, + "ndcg_exponential@1": 0.09153821956590123, + "ndcg_exponential@5": 0.15446626761982413, + "ndcg_exponential@10": 0.18024480954320224, + "precision@1": 0.09153821956590123, + "precision@5": 0.04271783579742058, + "precision@10": 0.02928593897452029, + "recall@1": 0.09153821956590123, + "recall@5": 0.21358917898710286, + "recall@10": 0.29285938974520287, + "f1@1": 0.09153821956590123, + "f1@5": 0.07119639299570096, + "f1@10": 0.053247161771855074, + "map@1": 0.09153821956590123, + "map@5": 0.13498479605746042, + "map@10": 0.14571006655781743, + "mrr@1": 0.09153821956590123, + "mrr@5": 0.13498479605746042, + "mrr@10": 0.14571006655781743, + "num_pred": 3179, + "num_data": 3179 + }, + "QVHighlight": { + "hit@1": 0.5383194829178208, + "hit@5": 0.863342566943675, + "hit@10": 1.0, + "ndcg_linear@1": 0.5383194829178208, + "ndcg_linear@5": 0.7079527214519653, + "ndcg_linear@10": 0.7516086585604039, + "ndcg_exponential@1": 0.5383194829178208, + "ndcg_exponential@5": 0.7079527214519653, + "ndcg_exponential@10": 0.7516086585604039, + "precision@1": 0.5383194829178208, + "precision@5": 0.17266851338873498, + "precision@10": 0.09999999999999998, + "recall@1": 0.5383194829178208, + "recall@5": 0.863342566943675, + "recall@10": 1.0, + "f1@1": 0.5383194829178208, + "f1@5": 0.28778085564789174, + "f1@10": 0.18181818181818185, + "map@1": 0.5383194829178208, + "map@5": 0.6565712526931364, + "map@10": 0.6742862272640666, + "mrr@1": 0.5383194829178208, + "mrr@5": 0.6565712526931364, + "mrr@10": 0.6742862272640666, + "num_pred": 1083, + "num_data": 1083 + }, + "Charades-STA": { + "hit@1": 0.10866574965612105, + "hit@5": 0.4786795048143054, + "hit@10": 1.0, + "ndcg_linear@1": 0.10866574965612105, + "ndcg_linear@5": 0.2926265568216908, + "ndcg_linear@10": 0.4575713199023012, + "ndcg_exponential@1": 0.10866574965612105, + "ndcg_exponential@5": 0.2926265568216908, + "ndcg_exponential@10": 0.4575713199023012, + "precision@1": 0.10866574965612105, + "precision@5": 0.09573590096286107, + "precision@10": 0.09999999999999998, + "recall@1": 0.10866574965612105, + "recall@5": 0.4786795048143054, + "recall@10": 1.0, + "f1@1": 0.10866574965612105, + "f1@5": 0.15955983493810183, + "f1@10": 0.18181818181818185, + "map@1": 0.10866574965612105, + "map@5": 0.23207244383310408, + "map@10": 0.2980508067509443, + "mrr@1": 0.10866574965612105, + "mrr@5": 0.23207244383310408, + "mrr@10": 0.2980508067509443, + "num_pred": 727, + "num_data": 727 + }, + "MomentSeeker": { + "hit@1": 0.3383333333333333, + "hit@5": 0.8205555555555556, + "hit@10": 1.0, + "ndcg_linear@1": 0.3383333333333333, + "ndcg_linear@5": 0.5821798365943692, + "ndcg_linear@10": 0.6491829437988351, + "ndcg_exponential@1": 0.3383333333333333, + "ndcg_exponential@5": 0.5821798365943692, + "ndcg_exponential@10": 0.6491829437988351, + "precision@1": 0.3383333333333333, + "precision@5": 0.17922222222222223, + "precision@10": 0.11311111111111112, + "recall@1": 0.31546296296296295, + "recall@5": 0.8009259259259259, + "recall@10": 0.9988888888888889, + "f1@1": 0.3220299823633157, + "f1@5": 0.28780555863889207, + "f1@10": 0.20050498883832216, + "map@1": 0.3383333333333333, + "map@5": 0.5041177469135804, + "map@10": 0.5349385196208113, + "mrr@1": 0.3383333333333333, + "mrr@5": 0.5154722222222222, + "mrr@10": 0.540335758377425, + "num_pred": 1800, + "num_data": 1800 + } + }, + "visdoc": { + "ViDoRe_arxivqa": { + "hit@1": 0.052, + "hit@5": 0.172, + "hit@10": 0.286, + "ndcg_linear@1": 0.052, + "ndcg_linear@5": 0.10801878370138777, + "ndcg_linear@10": 0.14373908906362196, + "ndcg_exponential@1": 0.052, + "ndcg_exponential@5": 0.10801878370138777, + "ndcg_exponential@10": 0.14373908906362196, + "precision@1": 0.052, + "precision@5": 0.03440000000000001, + "precision@10": 0.0286, + "recall@1": 0.052, + "recall@5": 0.172, + "recall@10": 0.286, + "f1@1": 0.052, + "f1@5": 0.05733333333333333, + "f1@10": 0.052, + "map@1": 0.052, + "map@5": 0.08753333333333334, + "map@10": 0.10160873015873015, + "mrr@1": 0.052, + "mrr@5": 0.08753333333333334, + "mrr@10": 0.10160873015873015, + "num_pred": 500, + "num_data": 500 + }, + "ViDoRe_docvqa": { + "hit@1": 0.13747228381374724, + "hit@5": 0.2549889135254989, + "hit@10": 0.3148558758314856, + "ndcg_linear@1": 0.13747228381374724, + "ndcg_linear@5": 0.19129511635223329, + "ndcg_linear@10": 0.2105752120749416, + "ndcg_exponential@1": 0.13747228381374724, + "ndcg_exponential@5": 0.19129511635223329, + "ndcg_exponential@10": 0.2105752120749416, + "precision@1": 0.13747228381374724, + "precision@5": 0.050997782705099776, + "precision@10": 0.031929046563192905, + "recall@1": 0.13458980044345897, + "recall@5": 0.24811529933481155, + "recall@10": 0.30842572062084256, + "f1@1": 0.1352549889135255, + "f1@5": 0.08408826945412314, + "f1@10": 0.057562319424847144, + "map@1": 0.13747228381374724, + "map@5": 0.17147080561714712, + "map@10": 0.17921637982613592, + "mrr@1": 0.13747228381374724, + "mrr@5": 0.17597930524759794, + "mrr@10": 0.1834661950515609, + "num_pred": 451, + "num_data": 451 + }, + "ViDoRe_infovqa": { + "hit@1": 0.3319838056680162, + "hit@5": 0.5748987854251012, + "hit@10": 0.6923076923076923, + "ndcg_linear@1": 0.3319838056680162, + "ndcg_linear@5": 0.46253535711806587, + "ndcg_linear@10": 0.5004886054716596, + "ndcg_exponential@1": 0.3319838056680162, + "ndcg_exponential@5": 0.46253535711806587, + "ndcg_exponential@10": 0.5004886054716596, + "precision@1": 0.3319838056680162, + "precision@5": 0.11497975708502024, + "precision@10": 0.06923076923076922, + "recall@1": 0.3319838056680162, + "recall@5": 0.5748987854251012, + "recall@10": 0.6912955465587044, + "f1@1": 0.3319838056680162, + "f1@5": 0.19163292847503377, + "f1@10": 0.12584345479082323, + "map@1": 0.3319838056680162, + "map@5": 0.4250337381916329, + "map@10": 0.4408127691022428, + "mrr@1": 0.3319838056680162, + "mrr@5": 0.4250337381916329, + "mrr@10": 0.4409252297410192, + "num_pred": 494, + "num_data": 494 + }, + "ViDoRe_tabfquad": { + "hit@1": 0.275, + "hit@5": 0.5678571428571428, + "hit@10": 0.6928571428571428, + "ndcg_linear@1": 0.275, + "ndcg_linear@5": 0.42836855281007846, + "ndcg_linear@10": 0.4688038965463907, + "ndcg_exponential@1": 0.275, + "ndcg_exponential@5": 0.42836855281007846, + "ndcg_exponential@10": 0.4688038965463907, + "precision@1": 0.275, + "precision@5": 0.11357142857142856, + "precision@10": 0.06928571428571428, + "recall@1": 0.275, + "recall@5": 0.5678571428571428, + "recall@10": 0.6928571428571428, + "f1@1": 0.275, + "f1@5": 0.1892857142857143, + "f1@10": 0.125974025974026, + "map@1": 0.275, + "map@5": 0.3823214285714286, + "map@10": 0.3990192743764172, + "mrr@1": 0.275, + "mrr@5": 0.3823214285714286, + "mrr@10": 0.3990192743764172, + "num_pred": 280, + "num_data": 280 }, - "MSVD": { - "hit@1": 0.46119402985074626, - "hit@5": 0.7, - "hit@10": 0.7850746268656716, - "ndcg_linear@1": 0.46119402985074626, - "ndcg_linear@5": 0.5909143508859, - "ndcg_linear@10": 0.6184940826509501, - "ndcg_exponential@1": 0.46119402985074626, - "ndcg_exponential@5": 0.5909143508859, - "ndcg_exponential@10": 0.6184940826509501, - "precision@1": 0.46119402985074626, - "precision@5": 0.13999999999999999, - "precision@10": 0.07850746268656715, - "recall@1": 0.46119402985074626, - "recall@5": 0.7, - "recall@10": 0.7850746268656716, - "f1@1": 0.46119402985074626, - "f1@5": 0.2333333333333334, - "f1@10": 0.14274084124830397, - "map@1": 0.46119402985074626, - "map@5": 0.5543781094527364, - "map@10": 0.5658019426676143, - "mrr@1": 0.46119402985074626, - "mrr@5": 0.5543781094527364, - "mrr@10": 0.5658019426676143, - "num_pred": 670, - "num_data": 670 + "ViDoRe_tatdqa": { + "hit@1": 0.06804374240583232, + "hit@5": 0.1591737545565006, + "hit@10": 0.22053462940461727, + "ndcg_linear@1": 0.06804374240583232, + "ndcg_linear@5": 0.11430035232416713, + "ndcg_linear@10": 0.13412008753307786, + "ndcg_exponential@1": 0.06804374240583232, + "ndcg_exponential@5": 0.11430035232416713, + "ndcg_exponential@10": 0.13412008753307786, + "precision@1": 0.06804374240583232, + "precision@5": 0.03183475091130013, + "precision@10": 0.022053462940461725, + "recall@1": 0.06804374240583232, + "recall@5": 0.1586269744835966, + "recall@10": 0.21998784933171325, + "f1@1": 0.06804374240583232, + "f1@5": 0.052936411502632646, + "f1@10": 0.04004749806693913, + "map@1": 0.06804374240583232, + "map@5": 0.09965978128797083, + "map@10": 0.1078442689347914, + "mrr@1": 0.06804374240583232, + "mrr@5": 0.09975698663426488, + "mrr@10": 0.10795362494937222, + "num_pred": 1646, + "num_data": 1646 }, - "VATEX": { - "hit@1": 0.19138008039303261, - "hit@5": 0.41692719964269764, - "hit@10": 0.5225547119249665, - "ndcg_linear@1": 0.19138008039303261, - "ndcg_linear@5": 0.31066559826092355, - "ndcg_linear@10": 0.3449304405187584, - "ndcg_exponential@1": 0.19138008039303261, - "ndcg_exponential@5": 0.31066559826092355, - "ndcg_exponential@10": 0.3449304405187584, - "precision@1": 0.19138008039303261, - "precision@5": 0.08338543992853951, - "precision@10": 0.05225547119249665, - "recall@1": 0.19138008039303261, - "recall@5": 0.41692719964269764, - "recall@10": 0.5225547119249665, - "f1@1": 0.19138008039303261, - "f1@5": 0.13897573321423257, - "f1@10": 0.09500994762272119, - "map@1": 0.19138008039303261, - "map@5": 0.27535730236712824, - "map@10": 0.28956069177708305, - "mrr@1": 0.19138008039303261, - "mrr@5": 0.27535730236712824, - "mrr@10": 0.28956069177708305, - "num_pred": 4478, - "num_data": 4478 + "ViDoRe_shiftproject": { + "hit@1": 0.07, + "hit@5": 0.17, + "hit@10": 0.27, + "ndcg_linear@1": 0.07, + "ndcg_linear@5": 0.12035565121611999, + "ndcg_linear@10": 0.1523429715395014, + "ndcg_exponential@1": 0.07, + "ndcg_exponential@5": 0.12035565121611999, + "ndcg_exponential@10": 0.1523429715395014, + "precision@1": 0.07, + "precision@5": 0.034, + "precision@10": 0.027000000000000003, + "recall@1": 0.07, + "recall@5": 0.17, + "recall@10": 0.27, + "f1@1": 0.07, + "f1@5": 0.05666666666666667, + "f1@10": 0.049090909090909095, + "map@1": 0.07, + "map@5": 0.10400000000000001, + "map@10": 0.116984126984127, + "mrr@1": 0.07, + "mrr@5": 0.10400000000000001, + "mrr@10": 0.116984126984127, + "num_pred": 100, + "num_data": 100 }, - "YouCook2": { - "hit@1": 0.09153821956590123, - "hit@5": 0.21358917898710286, - "hit@10": 0.29285938974520287, - "ndcg_linear@1": 0.09153821956590123, - "ndcg_linear@5": 0.15446626761982413, - "ndcg_linear@10": 0.18024480954320224, - "ndcg_exponential@1": 0.09153821956590123, - "ndcg_exponential@5": 0.15446626761982413, - "ndcg_exponential@10": 0.18024480954320224, - "precision@1": 0.09153821956590123, - "precision@5": 0.04271783579742058, - "precision@10": 0.02928593897452029, - "recall@1": 0.09153821956590123, - "recall@5": 0.21358917898710286, - "recall@10": 0.29285938974520287, - "f1@1": 0.09153821956590123, - "f1@5": 0.07119639299570096, - "f1@10": 0.053247161771855074, - "map@1": 0.09153821956590123, - "map@5": 0.13498479605746042, - "map@10": 0.14571006655781743, - "mrr@1": 0.09153821956590123, - "mrr@5": 0.13498479605746042, - "mrr@10": 0.14571006655781743, - "num_pred": 3179, - "num_data": 3179 + "ViDoRe_syntheticDocQA_artificial_intelligence": { + "hit@1": 0.05, + "hit@5": 0.15, + "hit@10": 0.22, + "ndcg_linear@1": 0.05, + "ndcg_linear@5": 0.1031574868850601, + "ndcg_linear@10": 0.12568086090486966, + "ndcg_exponential@1": 0.05, + "ndcg_exponential@5": 0.1031574868850601, + "ndcg_exponential@10": 0.12568086090486966, + "precision@1": 0.05, + "precision@5": 0.030000000000000006, + "precision@10": 0.022, + "recall@1": 0.05, + "recall@5": 0.15, + "recall@10": 0.22, + "f1@1": 0.05, + "f1@5": 0.05, + "f1@10": 0.04, + "map@1": 0.05, + "map@5": 0.0875, + "map@10": 0.09673412698412699, + "mrr@1": 0.05, + "mrr@5": 0.0875, + "mrr@10": 0.09673412698412699, + "num_pred": 100, + "num_data": 100 + }, + "ViDoRe_syntheticDocQA_energy": { + "hit@1": 0.18, + "hit@5": 0.31, + "hit@10": 0.41, + "ndcg_linear@1": 0.18, + "ndcg_linear@5": 0.24771924937667159, + "ndcg_linear@10": 0.2802583416144934, + "ndcg_exponential@1": 0.18, + "ndcg_exponential@5": 0.24771924937667159, + "ndcg_exponential@10": 0.2802583416144934, + "precision@1": 0.18, + "precision@5": 0.062, + "precision@10": 0.040999999999999995, + "recall@1": 0.18, + "recall@5": 0.31, + "recall@10": 0.41, + "f1@1": 0.18, + "f1@5": 0.10333333333333336, + "f1@10": 0.07454545454545455, + "map@1": 0.18, + "map@5": 0.22699999999999998, + "map@10": 0.24053968253968258, + "mrr@1": 0.18, + "mrr@5": 0.22699999999999998, + "mrr@10": 0.24053968253968258, + "num_pred": 100, + "num_data": 100 }, - "QVHighlight": { - "hit@1": 0.5383194829178208, - "hit@5": 0.863342566943675, - "hit@10": 1.0, - "ndcg_linear@1": 0.5383194829178208, - "ndcg_linear@5": 0.7079527214519653, - "ndcg_linear@10": 0.7516086585604039, - "ndcg_exponential@1": 0.5383194829178208, - "ndcg_exponential@5": 0.7079527214519653, - "ndcg_exponential@10": 0.7516086585604039, - "precision@1": 0.5383194829178208, - "precision@5": 0.17266851338873498, - "precision@10": 0.09999999999999998, - "recall@1": 0.5383194829178208, - "recall@5": 0.863342566943675, - "recall@10": 1.0, - "f1@1": 0.5383194829178208, - "f1@5": 0.28778085564789174, - "f1@10": 0.18181818181818185, - "map@1": 0.5383194829178208, - "map@5": 0.6565712526931364, - "map@10": 0.6742862272640666, - "mrr@1": 0.5383194829178208, - "mrr@5": 0.6565712526931364, - "mrr@10": 0.6742862272640666, - "num_pred": 1083, - "num_data": 1083 + "ViDoRe_syntheticDocQA_government_reports": { + "hit@1": 0.09, + "hit@5": 0.23, + "hit@10": 0.27, + "ndcg_linear@1": 0.09, + "ndcg_linear@5": 0.16359030940399763, + "ndcg_linear@10": 0.17675975017719422, + "ndcg_exponential@1": 0.09, + "ndcg_exponential@5": 0.16359030940399763, + "ndcg_exponential@10": 0.17675975017719422, + "precision@1": 0.09, + "precision@5": 0.046000000000000006, + "precision@10": 0.027000000000000007, + "recall@1": 0.09, + "recall@5": 0.23, + "recall@10": 0.27, + "f1@1": 0.09, + "f1@5": 0.07666666666666667, + "f1@10": 0.04909090909090908, + "map@1": 0.09, + "map@5": 0.14150000000000001, + "map@10": 0.14708333333333334, + "mrr@1": 0.09, + "mrr@5": 0.14150000000000001, + "mrr@10": 0.14708333333333334, + "num_pred": 100, + "num_data": 100 }, - "Charades-STA": { - "hit@1": 0.10866574965612105, - "hit@5": 0.4786795048143054, - "hit@10": 1.0, - "ndcg_linear@1": 0.10866574965612105, - "ndcg_linear@5": 0.2926265568216908, - "ndcg_linear@10": 0.4575713199023012, - "ndcg_exponential@1": 0.10866574965612105, - "ndcg_exponential@5": 0.2926265568216908, - "ndcg_exponential@10": 0.4575713199023012, - "precision@1": 0.10866574965612105, - "precision@5": 0.09573590096286107, - "precision@10": 0.09999999999999998, - "recall@1": 0.10866574965612105, - "recall@5": 0.4786795048143054, - "recall@10": 1.0, - "f1@1": 0.10866574965612105, - "f1@5": 0.15955983493810183, - "f1@10": 0.18181818181818185, - "map@1": 0.10866574965612105, - "map@5": 0.23207244383310408, - "map@10": 0.2980508067509443, - "mrr@1": 0.10866574965612105, - "mrr@5": 0.23207244383310408, - "mrr@10": 0.2980508067509443, - "num_pred": 727, - "num_data": 727 + "ViDoRe_syntheticDocQA_healthcare_industry": { + "hit@1": 0.14, + "hit@5": 0.36, + "hit@10": 0.4, + "ndcg_linear@1": 0.14, + "ndcg_linear@5": 0.2593663913604869, + "ndcg_linear@10": 0.2717200597145647, + "ndcg_exponential@1": 0.14, + "ndcg_exponential@5": 0.2593663913604869, + "ndcg_exponential@10": 0.2717200597145647, + "precision@1": 0.14, + "precision@5": 0.07200000000000001, + "precision@10": 0.04, + "recall@1": 0.14, + "recall@5": 0.36, + "recall@10": 0.4, + "f1@1": 0.14, + "f1@5": 0.12000000000000002, + "f1@10": 0.07272727272727274, + "map@1": 0.14, + "map@5": 0.22566666666666665, + "map@10": 0.2304444444444444, + "mrr@1": 0.14, + "mrr@5": 0.22566666666666665, + "mrr@10": 0.2304444444444444, + "num_pred": 100, + "num_data": 100 }, - "MomentSeeker": { - "hit@1": 0.39, - "hit@5": 0.823, - "hit@10": 1.0, - "ndcg_linear@1": 0.39, - "ndcg_linear@5": 0.6111730991071609, - "ndcg_linear@10": 0.6713060355016324, - "ndcg_exponential@1": 0.39, - "ndcg_exponential@5": 0.6111730991071609, - "ndcg_exponential@10": 0.6713060355016324, - "precision@1": 0.39, - "precision@5": 0.1732, - "precision@10": 0.10660000000000003, - "recall@1": 0.37683333333333335, - "recall@5": 0.8175, - "recall@10": 1.0, - "f1@1": 0.380415873015873, - "f1@5": 0.2822383172383172, - "f1@10": 0.19060245310245308, - "map@1": 0.39, - "map@5": 0.5413308333333334, - "map@10": 0.5673437003968254, - "mrr@1": 0.39, - "mrr@5": 0.5443833333333333, - "mrr@10": 0.5687698412698412, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_esg_reports_human_labeled_v2": { + "hit@1": 0.038461538461538464, + "hit@5": 0.17307692307692307, + "hit@10": 0.23076923076923078, + "ndcg_linear@1": 0.038461538461538464, + "ndcg_linear@5": 0.07581945100217463, + "ndcg_linear@10": 0.08884346676199335, + "ndcg_exponential@1": 0.038461538461538464, + "ndcg_exponential@5": 0.07527592503218486, + "ndcg_exponential@10": 0.0882008820280547, + "precision@1": 0.038461538461538464, + "precision@5": 0.05000000000000001, + "precision@10": 0.038461538461538464, + "recall@1": 0.020833333333333332, + "recall@5": 0.10961538461538461, + "recall@10": 0.14761904761904762, + "f1@1": 0.022189349112426034, + "f1@5": 0.057642031171442946, + "f1@10": 0.054193845370315956, + "map@1": 0.038461538461538464, + "map@5": 0.05387820512820513, + "map@10": 0.05896291208791209, + "mrr@1": 0.038461538461538464, + "mrr@5": 0.07724358974358975, + "mrr@10": 0.08392094017094018, + "num_pred": 52, + "num_data": 52 }, - "K700": { - "hit@1": 0.423, - "hit@5": 0.653, - "hit@10": 0.731, - "ndcg_linear@1": 0.423, - "ndcg_linear@5": 0.546371218016443, - "ndcg_linear@10": 0.5718997969605862, - "ndcg_exponential@1": 0.423, - "ndcg_exponential@5": 0.546371218016443, - "ndcg_exponential@10": 0.5718997969605862, - "precision@1": 0.423, - "precision@5": 0.13059999999999997, - "precision@10": 0.0731, - "recall@1": 0.423, - "recall@5": 0.653, - "recall@10": 0.731, - "f1@1": 0.423, - "f1@5": 0.21766666666666673, - "f1@10": 0.13290909090909092, - "map@1": 0.423, - "map@5": 0.5107999999999999, - "map@10": 0.5215190476190475, - "mrr@1": 0.423, - "mrr@5": 0.5107999999999999, - "mrr@10": 0.5215190476190475, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_biomedical_lectures_v2_multilingual": { + "hit@1": 0.096875, + "hit@5": 0.296875, + "hit@10": 0.3796875, + "ndcg_linear@1": 0.096875, + "ndcg_linear@5": 0.1327817509891042, + "ndcg_linear@10": 0.15578005969658662, + "ndcg_exponential@1": 0.096875, + "ndcg_exponential@5": 0.1327817509891042, + "ndcg_exponential@10": 0.15578005969658662, + "precision@1": 0.096875, + "precision@5": 0.06625, + "precision@10": 0.05015625, + "recall@1": 0.05542025862068965, + "recall@5": 0.1645906222010748, + "recall@10": 0.23159770265864968, + "f1@1": 0.06360192325036075, + "f1@5": 0.08230852582874641, + "f1@10": 0.07338045655325068, + "map@1": 0.096875, + "map@5": 0.09772092013888889, + "map@10": 0.10609532519605062, + "mrr@1": 0.096875, + "mrr@5": 0.16531250000000003, + "mrr@10": 0.1761204117063492, + "num_pred": 640, + "num_data": 640 }, - "SmthSmthV2": { - "hit@1": 0.363, - "hit@5": 0.602, - "hit@10": 0.698, - "ndcg_linear@1": 0.363, - "ndcg_linear@5": 0.4890606806277336, - "ndcg_linear@10": 0.5200630033328641, - "ndcg_exponential@1": 0.363, - "ndcg_exponential@5": 0.4890606806277336, - "ndcg_exponential@10": 0.5200630033328641, - "precision@1": 0.363, - "precision@5": 0.1204, - "precision@10": 0.06979999999999999, - "recall@1": 0.363, - "recall@5": 0.602, - "recall@10": 0.698, - "f1@1": 0.363, - "f1@5": 0.20066666666666674, - "f1@10": 0.12690909090909094, - "map@1": 0.363, - "map@5": 0.45156666666666667, - "map@10": 0.4643380952380952, - "mrr@1": 0.363, - "mrr@5": 0.45156666666666667, - "mrr@10": 0.4643380952380952, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_economics_reports_v2_multilingual": { + "hit@1": 0.1939655172413793, + "hit@5": 0.46551724137931033, + "hit@10": 0.5948275862068966, + "ndcg_linear@1": 0.1939655172413793, + "ndcg_linear@5": 0.19075341400656756, + "ndcg_linear@10": 0.19262260915535673, + "ndcg_exponential@1": 0.1939655172413793, + "ndcg_exponential@5": 0.19075341400656748, + "ndcg_exponential@10": 0.19262260915535673, + "precision@1": 0.1939655172413793, + "precision@5": 0.18448275862068964, + "precision@10": 0.16724137931034483, + "recall@1": 0.016014854892863833, + "recall@5": 0.07300449328968175, + "recall@10": 0.14033065985008555, + "f1@1": 0.029036373118143485, + "f1@5": 0.0903316153795555, + "f1@10": 0.12402634113864117, + "map@1": 0.1939655172413793, + "map@5": 0.12855603448275862, + "map@10": 0.1083281829653956, + "mrr@1": 0.1939655172413793, + "mrr@5": 0.2959051724137931, + "mrr@10": 0.3119937055281883, + "num_pred": 232, + "num_data": 232 + }, + "ViDoRe_esg_reports_v2_multilingual": { + "hit@1": 0.043859649122807015, + "hit@5": 0.16228070175438597, + "hit@10": 0.25877192982456143, + "ndcg_linear@1": 0.043859649122807015, + "ndcg_linear@5": 0.05924300567334283, + "ndcg_linear@10": 0.09666072127808396, + "ndcg_exponential@1": 0.043859649122807015, + "ndcg_exponential@5": 0.05924300567334283, + "ndcg_exponential@10": 0.09666072127808396, + "precision@1": 0.043859649122807015, + "precision@5": 0.04912280701754387, + "precision@10": 0.054385964912280704, + "recall@1": 0.008406432748538011, + "recall@5": 0.07328216374269006, + "recall@10": 0.17160087719298245, + "f1@1": 0.013420302893987103, + "f1@5": 0.05224007874936668, + "f1@10": 0.07439770268717638, + "map@1": 0.043859649122807015, + "map@5": 0.03577850877192982, + "map@10": 0.0544104619186856, + "mrr@1": 0.043859649122807015, + "mrr@5": 0.07339181286549706, + "mrr@10": 0.08703181565023671, + "num_pred": 228, + "num_data": 228 }, - "HMDB51": { - "hit@1": 0.405, - "hit@5": 0.658, - "hit@10": 0.783, - "ndcg_linear@1": 0.405, - "ndcg_linear@5": 0.5405643217050999, - "ndcg_linear@10": 0.5815107823708526, - "ndcg_exponential@1": 0.405, - "ndcg_exponential@5": 0.5405643217050999, - "ndcg_exponential@10": 0.5815107823708526, - "precision@1": 0.405, - "precision@5": 0.1316, - "precision@10": 0.0783, - "recall@1": 0.405, - "recall@5": 0.658, - "recall@10": 0.783, - "f1@1": 0.405, - "f1@5": 0.21933333333333338, - "f1@10": 0.1423636363636364, - "map@1": 0.405, - "map@5": 0.5013833333333333, - "map@10": 0.5185896825396826, - "mrr@1": 0.405, - "mrr@5": 0.5013833333333333, - "mrr@10": 0.5185896825396826, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ArxivQA": { + "hit@1": 0.012254901960784314, + "hit@5": 0.02696078431372549, + "hit@10": 0.03553921568627451, + "ndcg_linear@1": 0.012254901960784314, + "ndcg_linear@5": 0.020155243188820424, + "ndcg_linear@10": 0.022825943867728897, + "ndcg_exponential@1": 0.012254901960784314, + "ndcg_exponential@5": 0.020155243188820424, + "ndcg_exponential@10": 0.022825943867728897, + "precision@1": 0.012254901960784314, + "precision@5": 0.005392156862745099, + "precision@10": 0.0035539215686274514, + "recall@1": 0.012254901960784314, + "recall@5": 0.02696078431372549, + "recall@10": 0.03553921568627451, + "f1@1": 0.012254901960784314, + "f1@5": 0.008986928104575164, + "f1@10": 0.006461675579322638, + "map@1": 0.012254901960784314, + "map@5": 0.01787173202614379, + "map@10": 0.018913398692810456, + "mrr@1": 0.012254901960784314, + "mrr@5": 0.01787173202614379, + "mrr@10": 0.018913398692810456, + "num_pred": 816, + "num_data": 816 }, - "UCF101": { - "hit@1": 0.604, - "hit@5": 0.852, - "hit@10": 0.89, - "ndcg_linear@1": 0.604, - "ndcg_linear@5": 0.7417818963613594, - "ndcg_linear@10": 0.7541096692471368, - "ndcg_exponential@1": 0.604, - "ndcg_exponential@5": 0.7417818963613594, - "ndcg_exponential@10": 0.7541096692471368, - "precision@1": 0.604, - "precision@5": 0.17040000000000002, - "precision@10": 0.08900000000000001, - "recall@1": 0.604, - "recall@5": 0.852, - "recall@10": 0.89, - "f1@1": 0.604, - "f1@5": 0.284, - "f1@10": 0.1618181818181818, - "map@1": 0.604, - "map@5": 0.7046166666666666, - "map@10": 0.7097305555555556, - "mrr@1": 0.604, - "mrr@5": 0.7046166666666666, - "mrr@10": 0.7097305555555556, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ChartQA": { + "hit@1": 0.31746031746031744, + "hit@5": 0.49206349206349204, + "hit@10": 0.5396825396825397, + "ndcg_linear@1": 0.31746031746031744, + "ndcg_linear@5": 0.41335716785860044, + "ndcg_linear@10": 0.42995633781970166, + "ndcg_exponential@1": 0.31746031746031744, + "ndcg_exponential@5": 0.41335716785860044, + "ndcg_exponential@10": 0.42995633781970166, + "precision@1": 0.31746031746031744, + "precision@5": 0.09841269841269841, + "precision@10": 0.05396825396825397, + "recall@1": 0.31746031746031744, + "recall@5": 0.49206349206349204, + "recall@10": 0.5396825396825397, + "f1@1": 0.31746031746031744, + "f1@5": 0.1640211640211641, + "f1@10": 0.09812409812409813, + "map@1": 0.31746031746031744, + "map@5": 0.3867724867724867, + "map@10": 0.39433106575963717, + "mrr@1": 0.31746031746031744, + "mrr@5": 0.3867724867724867, + "mrr@10": 0.39433106575963717, + "num_pred": 63, + "num_data": 63 }, - "Breakfast": { - "hit@1": 0.16859122401847576, - "hit@5": 0.6235565819861432, - "hit@10": 1.0, - "ndcg_linear@1": 0.16859122401847576, - "ndcg_linear@5": 0.400691767704696, - "ndcg_linear@10": 0.5235817090775436, - "ndcg_exponential@1": 0.16859122401847576, - "ndcg_exponential@5": 0.400691767704696, - "ndcg_exponential@10": 0.5235817090775436, - "precision@1": 0.16859122401847576, - "precision@5": 0.12471131639722864, - "precision@10": 0.1, - "recall@1": 0.16859122401847576, - "recall@5": 0.6235565819861432, - "recall@10": 1.0, - "f1@1": 0.16859122401847576, - "f1@5": 0.20785219399538113, - "f1@10": 0.1818181818181818, - "map@1": 0.16859122401847576, - "map@5": 0.32763664357197847, - "map@10": 0.37902140840939913, - "mrr@1": 0.16859122401847576, - "mrr@5": 0.32763664357197847, - "mrr@10": 0.37902140840939913, - "num_pred": 433, - "num_data": 433 + "VisRAG_MP-DocVQA": { + "hit@1": 0.2436548223350254, + "hit@5": 0.41624365482233505, + "hit@10": 0.48223350253807107, + "ndcg_linear@1": 0.2436548223350254, + "ndcg_linear@5": 0.33378850914695807, + "ndcg_linear@10": 0.3551570419412752, + "ndcg_exponential@1": 0.2436548223350254, + "ndcg_exponential@5": 0.33378850914695807, + "ndcg_exponential@10": 0.3551570419412752, + "precision@1": 0.2436548223350254, + "precision@5": 0.08324873096446701, + "precision@10": 0.048223350253807105, + "recall@1": 0.2436548223350254, + "recall@5": 0.41624365482233505, + "recall@10": 0.48223350253807107, + "f1@1": 0.2436548223350254, + "f1@5": 0.13874788494077836, + "f1@10": 0.08767881864328564, + "map@1": 0.2436548223350254, + "map@5": 0.30654258319232935, + "map@10": 0.31538353073886066, + "mrr@1": 0.2436548223350254, + "mrr@5": 0.30654258319232935, + "mrr@10": 0.31538353073886066, + "num_pred": 591, + "num_data": 591 }, - "MVBench": { - "hit@1": 0.372, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.372, - "ndcg_linear@5": 0.7114371453270959, - "ndcg_linear@10": 0.7114371453270959, - "ndcg_exponential@1": 0.372, - "ndcg_exponential@5": 0.7114371453270959, - "ndcg_exponential@10": 0.7114371453270959, - "precision@1": 0.372, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.372, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.372, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.372, - "map@5": 0.6143833333333334, - "map@10": 0.6143833333333334, - "mrr@1": 0.372, - "mrr@5": 0.6143833333333334, - "mrr@10": 0.6143833333333334, - "num_pred": 4000, - "num_data": 4000 + "VisRAG_SlideVQA": { + "hit@1": 0.44964028776978415, + "hit@5": 0.7086330935251799, + "hit@10": 0.7859712230215827, + "ndcg_linear@1": 0.44964028776978415, + "ndcg_linear@5": 0.5648847466987875, + "ndcg_linear@10": 0.5955590007686733, + "ndcg_exponential@1": 0.44964028776978415, + "ndcg_exponential@5": 0.5648847466987875, + "ndcg_exponential@10": 0.5955590007686733, + "precision@1": 0.44964028776978415, + "precision@5": 0.1723021582733813, + "precision@10": 0.09766187050359712, + "recall@1": 0.36960431654676257, + "recall@5": 0.6735611510791367, + "recall@10": 0.7616906474820144, + "f1@1": 0.3962829736211031, + "f1@5": 0.2692702980472765, + "f1@10": 0.17105406583823854, + "map@1": 0.44964028776978415, + "map@5": 0.5169964028776978, + "map@10": 0.5313963115222108, + "mrr@1": 0.44964028776978415, + "mrr@5": 0.5481115107913669, + "mrr@10": 0.5588072399223478, + "num_pred": 556, + "num_data": 556 + }, + "VisRAG_InfoVQA": { + "hit@1": 0.4415041782729805, + "hit@5": 0.6713091922005571, + "hit@10": 0.754874651810585, + "ndcg_linear@1": 0.4415041782729805, + "ndcg_linear@5": 0.5633673128694291, + "ndcg_linear@10": 0.5903761050947809, + "ndcg_exponential@1": 0.4415041782729805, + "ndcg_exponential@5": 0.5633673128694291, + "ndcg_exponential@10": 0.5903761050947809, + "precision@1": 0.4415041782729805, + "precision@5": 0.13426183844011141, + "precision@10": 0.07548746518105848, + "recall@1": 0.4415041782729805, + "recall@5": 0.6713091922005571, + "recall@10": 0.754874651810585, + "f1@1": 0.4415041782729805, + "f1@5": 0.2237697307335191, + "f1@10": 0.13724993669283364, + "map@1": 0.4415041782729805, + "map@5": 0.5274373259052925, + "map@10": 0.5385661228279612, + "mrr@1": 0.4415041782729805, + "mrr@5": 0.5274373259052925, + "mrr@10": 0.5385661228279612, + "num_pred": 718, + "num_data": 718 }, - "Video-MME": { - "hit@1": 0.34074074074074073, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.34074074074074073, - "ndcg_linear@5": 0.6905732385187221, - "ndcg_linear@10": 0.6905732385187221, - "ndcg_exponential@1": 0.34074074074074073, - "ndcg_exponential@5": 0.6905732385187221, - "ndcg_exponential@10": 0.6905732385187221, - "precision@1": 0.34074074074074073, - "precision@5": 0.19999999999999996, - "precision@10": 0.09999999999999998, - "recall@1": 0.34074074074074073, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.34074074074074073, - "f1@5": 0.3333333333333334, - "f1@10": 0.18181818181818188, - "map@1": 0.34074074074074073, - "map@5": 0.5870987654320988, - "map@10": 0.5870987654320988, - "mrr@1": 0.34074074074074073, - "mrr@5": 0.5870987654320988, - "mrr@10": 0.5870987654320988, - "num_pred": 2700, - "num_data": 2700 + "VisRAG_PlotQA": { + "hit@1": 0.2410196987253766, + "hit@5": 0.4380069524913094, + "hit@10": 0.5365005793742758, + "ndcg_linear@1": 0.2410196987253766, + "ndcg_linear@5": 0.3455373205552205, + "ndcg_linear@10": 0.3774838122200841, + "ndcg_exponential@1": 0.2410196987253766, + "ndcg_exponential@5": 0.3455373205552205, + "ndcg_exponential@10": 0.3774838122200841, + "precision@1": 0.2410196987253766, + "precision@5": 0.08760139049826188, + "precision@10": 0.053650057937427574, + "recall@1": 0.2410196987253766, + "recall@5": 0.4380069524913094, + "recall@10": 0.5365005793742758, + "f1@1": 0.2410196987253766, + "f1@5": 0.14600231749710316, + "f1@10": 0.09754555988623198, + "map@1": 0.2410196987253766, + "map@5": 0.3148319814600232, + "map@10": 0.32806792473652263, + "mrr@1": 0.2410196987253766, + "mrr@5": 0.3148319814600232, + "mrr@10": 0.32806792473652263, + "num_pred": 863, + "num_data": 863 }, - "NExTQA": { - "hit@1": 0.4371788883699206, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.4371788883699206, - "ndcg_linear@5": 0.7269325524352686, - "ndcg_linear@10": 0.7269325524352686, - "ndcg_exponential@1": 0.4371788883699206, - "ndcg_exponential@5": 0.7269325524352686, - "ndcg_exponential@10": 0.7269325524352686, - "precision@1": 0.4371788883699206, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.4371788883699206, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.4371788883699206, - "f1@5": 0.3333333333333333, - "f1@10": 0.1818181818181818, - "map@1": 0.4371788883699206, - "map@5": 0.6369803829985987, - "map@10": 0.6369803829985987, - "mrr@1": 0.4371788883699206, - "mrr@5": 0.6367857698894598, - "mrr@10": 0.6367857698894598, - "num_pred": 8564, - "num_data": 8564 + "ViDoSeek-page": { + "hit@1": 0.040280210157618214, + "hit@5": 0.18563922942206654, + "hit@10": 0.287215411558669, + "ndcg_linear@1": 0.040280210157618214, + "ndcg_linear@5": 0.11294100510293231, + "ndcg_linear@10": 0.14594536433240723, + "ndcg_exponential@1": 0.040280210157618214, + "ndcg_exponential@5": 0.11294100510293234, + "ndcg_exponential@10": 0.14594536433240726, + "precision@1": 0.040280210157618214, + "precision@5": 0.03712784588441331, + "precision@10": 0.028721541155866902, + "recall@1": 0.040280210157618214, + "recall@5": 0.18563922942206654, + "recall@10": 0.287215411558669, + "f1@1": 0.040280210157618214, + "f1@5": 0.06187974314068886, + "f1@10": 0.052220983919758, + "map@1": 0.040280210157618214, + "map@5": 0.08917104495037945, + "map@10": 0.10288862480193478, + "mrr@1": 0.040280210157618214, + "mrr@5": 0.08917104495037945, + "mrr@10": 0.10288862480193478, + "num_pred": 1142, + "num_data": 1142 }, - "EgoSchema": { - "hit@1": 0.448, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.448, - "ndcg_linear@5": 0.7459721375093364, - "ndcg_linear@10": 0.7459721375093364, - "ndcg_exponential@1": 0.448, - "ndcg_exponential@5": 0.7459721375093364, - "ndcg_exponential@10": 0.7459721375093364, - "precision@1": 0.448, - "precision@5": 0.2, - "precision@10": 0.1, - "recall@1": 0.448, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.448, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.448, - "map@5": 0.6608999999999999, - "map@10": 0.6608999999999999, - "mrr@1": 0.448, - "mrr@5": 0.6608999999999999, - "mrr@10": 0.6608999999999999, - "num_pred": 500, - "num_data": 500 + "ViDoSeek-doc": { + "hit@1": 0.4982486865148862, + "hit@5": 0.6567425569176882, + "hit@10": 0.7338003502626971, + "ndcg_linear@1": 0.3429655575014594, + "ndcg_linear@5": 0.3714170601904285, + "ndcg_linear@10": 0.35019901577656104, + "ndcg_exponential@1": 0.23204903677758318, + "ndcg_exponential@5": 0.3146728814563305, + "ndcg_exponential@10": 0.3162182310328309, + "precision@1": 0.4982486865148862, + "precision@5": 0.4001751313485114, + "precision@10": 0.3430823117338004, + "recall@1": 0.02844268284063671, + "recall@5": 0.11354378337205699, + "recall@10": 0.19116020046136178, + "f1@1": 0.05349597259677307, + "f1@5": 0.1736704243149188, + "f1@10": 0.24068949848035567, + "map@1": 0.4982486865148862, + "map@5": 0.3603765323992995, + "map@10": 0.2910387701490404, + "mrr@1": 0.4982486865148862, + "mrr@5": 0.5563776999416229, + "mrr@10": 0.5669967753592973, + "num_pred": 1142, + "num_data": 1142 }, - "ActivityNetQA": { - "hit@1": 0.532, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.532, - "ndcg_linear@5": 0.8272751246714422, - "ndcg_linear@10": 0.8272751246714422, - "ndcg_exponential@1": 0.532, - "ndcg_exponential@5": 0.8272751246714422, - "ndcg_exponential@10": 0.8272751246714422, - "precision@1": 0.532, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.532, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.532, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.532, - "map@5": 0.766, - "map@10": 0.766, - "mrr@1": 0.532, - "mrr@5": 0.766, - "mrr@10": 0.766, - "num_pred": 1000, - "num_data": 1000 + "MMLongBench-page": { + "hit@1": 0.059665871121718374, + "hit@5": 0.14797136038186157, + "hit@10": 0.2159904534606205, + "ndcg_linear@1": 0.059665871121718374, + "ndcg_linear@5": 0.07951086728623444, + "ndcg_linear@10": 0.09880430458699743, + "ndcg_exponential@1": 0.059665871121718374, + "ndcg_exponential@5": 0.07951086728623444, + "ndcg_exponential@10": 0.09880430458699746, + "precision@1": 0.059665871121718374, + "precision@5": 0.03293556085918855, + "precision@10": 0.0256563245823389, + "recall@1": 0.038929139674963066, + "recall@5": 0.10381908929044967, + "recall@10": 0.15813207940296245, + "f1@1": 0.04444323907331067, + "f1@5": 0.04737172798031987, + "f1@10": 0.042217267935243885, + "map@1": 0.059665871121718374, + "map@5": 0.06207769822328295, + "map@10": 0.06994553516182395, + "mrr@1": 0.059665871121718374, + "mrr@5": 0.08904136833731105, + "mrr@10": 0.09803860287153843, + "num_pred": 838, + "num_data": 838 + }, + "MMLongBench-doc": { + "hit@1": 0.34725536992840095, + "hit@5": 0.5238663484486874, + "hit@10": 0.5883054892601431, + "ndcg_linear@1": 0.24701670644391405, + "ndcg_linear@5": 0.2758146276737757, + "ndcg_linear@10": 0.2683418556525889, + "ndcg_exponential@1": 0.17541766109785203, + "ndcg_exponential@5": 0.2302031604572782, + "ndcg_exponential@10": 0.23834477001816015, + "precision@1": 0.34725536992840095, + "precision@5": 0.31169451073985677, + "precision@10": 0.27792362768496426, + "recall@1": 0.011458219705852956, + "recall@5": 0.0473748134486053, + "recall@10": 0.08146579365576306, + "f1@1": 0.021977357536621576, + "f1@5": 0.0790594246656723, + "f1@10": 0.11819035378881726, + "map@1": 0.34725536992840095, + "map@5": 0.2678878281622912, + "map@10": 0.22411547713755348, + "mrr@1": 0.34725536992840095, + "mrr@5": 0.41585123309466987, + "mrr@10": 0.4247144561882032, + "num_pred": 838, + "num_data": 838 } } }