diff --git "a/scores/VLM2Vec-V1-Qwen2VL-7B.json" "b/scores/VLM2Vec-V1-Qwen2VL-7B.json" --- "a/scores/VLM2Vec-V1-Qwen2VL-7B.json" +++ "b/scores/VLM2Vec-V1-Qwen2VL-7B.json" @@ -1,10 +1,7 @@ { "metadata": { "model_name": "VLM2Vec-V1-Qwen2VL-7B", - "report_generated_date": "2025-06-08T08:08:07.905654", - "model_size": 8.29, - "url": "https://huggingface.co/TIGER-Lab/VLM2Vec-Qwen2VL-7B", - "data_source": "TIGER-Lab" + "report_generated_date": "2025-06-28T08:54:18.948514" }, "metrics": { "image": { @@ -1017,1268 +1014,1184 @@ "num_data": 1000 } }, - "visdoc": { - "ViDoRe_arxivqa": { - "hit@1": 0.506, + "video": { + "K700": { + "hit@1": 0.355, + "hit@5": 0.6, + "hit@10": 0.679, + "ndcg_linear@1": 0.355, + "ndcg_linear@5": 0.4845102488689986, + "ndcg_linear@10": 0.510304213516978, + "ndcg_exponential@1": 0.355, + "ndcg_exponential@5": 0.4845102488689986, + "ndcg_exponential@10": 0.510304213516978, + "precision@1": 0.355, + "precision@5": 0.12, + "precision@10": 0.06789999999999999, + "recall@1": 0.355, + "recall@5": 0.6, + "recall@10": 0.679, + "f1@1": 0.355, + "f1@5": 0.20000000000000007, + "f1@10": 0.12345454545454547, + "map@1": 0.355, + "map@5": 0.44616666666666666, + "map@10": 0.4569583333333333, + "mrr@1": 0.355, + "mrr@5": 0.44616666666666666, + "mrr@10": 0.4569583333333333, + "num_pred": 1000, + "num_data": 1000 + }, + "SmthSmthV2": { + "hit@1": 0.321, + "hit@5": 0.593, + "hit@10": 0.695, + "ndcg_linear@1": 0.321, + "ndcg_linear@5": 0.464820244083625, + "ndcg_linear@10": 0.49815926994038934, + "ndcg_exponential@1": 0.321, + "ndcg_exponential@5": 0.464820244083625, + "ndcg_exponential@10": 0.49815926994038934, + "precision@1": 0.321, + "precision@5": 0.1186, + "precision@10": 0.0695, + "recall@1": 0.321, + "recall@5": 0.593, + "recall@10": 0.695, + "f1@1": 0.321, + "f1@5": 0.19766666666666674, + "f1@10": 0.1263636363636364, + "map@1": 0.321, + "map@5": 0.4222333333333333, + "map@10": 0.43619880952380946, + "mrr@1": 0.321, + "mrr@5": 0.4222333333333333, + "mrr@10": 0.43619880952380946, + "num_pred": 1000, + "num_data": 1000 + }, + "HMDB51": { + "hit@1": 0.422, "hit@5": 0.684, - "hit@10": 0.736, - "ndcg_linear@1": 0.506, - "ndcg_linear@5": 0.6019848283346673, - "ndcg_linear@10": 0.618842382116875, - "ndcg_exponential@1": 0.506, - "ndcg_exponential@5": 0.6019848283346673, - "ndcg_exponential@10": 0.618842382116875, - "precision@1": 0.506, - "precision@5": 0.13679999999999998, - "precision@10": 0.0736, - "recall@1": 0.506, + "hit@10": 0.83, + "ndcg_linear@1": 0.422, + "ndcg_linear@5": 0.5578823382949479, + "ndcg_linear@10": 0.605271745232144, + "ndcg_exponential@1": 0.422, + "ndcg_exponential@5": 0.5578823382949479, + "ndcg_exponential@10": 0.605271745232144, + "precision@1": 0.422, + "precision@5": 0.1368, + "precision@10": 0.083, + "recall@1": 0.422, "recall@5": 0.684, - "recall@10": 0.736, - "f1@1": 0.506, + "recall@10": 0.83, + "f1@1": 0.422, "f1@5": 0.22800000000000006, - "f1@10": 0.1338181818181818, - "map@1": 0.506, - "map@5": 0.5745333333333333, - "map@10": 0.5815087301587302, - "mrr@1": 0.506, - "mrr@5": 0.5745333333333333, - "mrr@10": 0.5815087301587302, - "num_pred": 500, - "num_data": 500 - }, - "ViDoRe_docvqa": { - "hit@1": 0.270509977827051, - "hit@5": 0.42572062084257206, - "hit@10": 0.5188470066518847, - "ndcg_linear@1": 0.270509977827051, - "ndcg_linear@5": 0.3471381693961765, - "ndcg_linear@10": 0.3761073342818233, - "ndcg_exponential@1": 0.270509977827051, - "ndcg_exponential@5": 0.3471381693961765, - "ndcg_exponential@10": 0.3761073342818233, - "precision@1": 0.270509977827051, - "precision@5": 0.08558758314855876, - "precision@10": 0.05232815964523281, - "recall@1": 0.2668144863266815, - "recall@5": 0.42165558019216554, - "recall@10": 0.5104212860310421, - "f1@1": 0.2679231337767923, - "f1@5": 0.14195966634991028, - "f1@10": 0.09464696427002414, - "map@1": 0.270509977827051, - "map@5": 0.32139689578713965, - "map@10": 0.33320047161510574, - "mrr@1": 0.270509977827051, - "mrr@5": 0.3251662971175166, - "mrr@10": 0.3374995600605356, - "num_pred": 451, - "num_data": 451 + "f1@10": 0.15090909090909094, + "map@1": 0.422, + "map@5": 0.5162333333333333, + "map@10": 0.5358904761904761, + "mrr@1": 0.422, + "mrr@5": 0.5162333333333333, + "mrr@10": 0.5358904761904761, + "num_pred": 1000, + "num_data": 1000 }, - "ViDoRe_infovqa": { - "hit@1": 0.597165991902834, - "hit@5": 0.7894736842105263, - "hit@10": 0.8522267206477733, - "ndcg_linear@1": 0.597165991902834, - "ndcg_linear@5": 0.7037600234071207, - "ndcg_linear@10": 0.7251514237863242, - "ndcg_exponential@1": 0.597165991902834, - "ndcg_exponential@5": 0.7037600234071207, - "ndcg_exponential@10": 0.7251514237863242, - "precision@1": 0.597165991902834, - "precision@5": 0.15789473684210525, - "precision@10": 0.08542510121457489, - "recall@1": 0.597165991902834, - "recall@5": 0.7884615384615384, - "recall@10": 0.8522267206477733, - "f1@1": 0.597165991902834, - "f1@5": 0.26306149990360517, - "f1@10": 0.1552570236780763, - "map@1": 0.597165991902834, - "map@5": 0.6750674763832659, - "map@10": 0.6844386607544503, - "mrr@1": 0.597165991902834, - "mrr@5": 0.6752699055330635, - "mrr@10": 0.6843037079879185, - "num_pred": 494, - "num_data": 494 + "UCF101": { + "hit@1": 0.618, + "hit@5": 0.889, + "hit@10": 0.94, + "ndcg_linear@1": 0.618, + "ndcg_linear@5": 0.772430845929742, + "ndcg_linear@10": 0.7893494678275759, + "ndcg_exponential@1": 0.618, + "ndcg_exponential@5": 0.772430845929742, + "ndcg_exponential@10": 0.7893494678275759, + "precision@1": 0.618, + "precision@5": 0.1778, + "precision@10": 0.09400000000000001, + "recall@1": 0.618, + "recall@5": 0.889, + "recall@10": 0.94, + "f1@1": 0.618, + "f1@5": 0.2963333333333334, + "f1@10": 0.1709090909090909, + "map@1": 0.618, + "map@5": 0.7326833333333334, + "map@10": 0.7399230158730159, + "mrr@1": 0.618, + "mrr@5": 0.7326833333333334, + "mrr@10": 0.7399230158730159, + "num_pred": 1000, + "num_data": 1000 }, - "ViDoRe_tabfquad": { - "hit@1": 0.675, - "hit@5": 0.8714285714285714, - "hit@10": 0.9178571428571428, - "ndcg_linear@1": 0.675, - "ndcg_linear@5": 0.7818109382746695, - "ndcg_linear@10": 0.7971064823405059, - "ndcg_exponential@1": 0.675, - "ndcg_exponential@5": 0.7818109382746695, - "ndcg_exponential@10": 0.7971064823405059, - "precision@1": 0.675, - "precision@5": 0.17428571428571427, - "precision@10": 0.09178571428571428, - "recall@1": 0.675, - "recall@5": 0.8714285714285714, - "recall@10": 0.9178571428571428, - "f1@1": 0.675, - "f1@5": 0.2904761904761905, - "f1@10": 0.1668831168831169, - "map@1": 0.675, - "map@5": 0.7517857142857143, - "map@10": 0.7582624716553289, - "mrr@1": 0.675, - "mrr@5": 0.7517857142857143, - "mrr@10": 0.7582624716553289, - "num_pred": 280, - "num_data": 280 + "Breakfast": { + "hit@1": 0.23787528868360278, + "hit@5": 0.651270207852194, + "hit@10": 1.0, + "ndcg_linear@1": 0.23787528868360278, + "ndcg_linear@5": 0.4481888236256693, + "ndcg_linear@10": 0.5633176174385133, + "ndcg_exponential@1": 0.23787528868360278, + "ndcg_exponential@5": 0.4481888236256693, + "ndcg_exponential@10": 0.5633176174385133, + "precision@1": 0.23787528868360278, + "precision@5": 0.13025404157043877, + "precision@10": 0.1, + "recall@1": 0.23787528868360278, + "recall@5": 0.651270207852194, + "recall@10": 1.0, + "f1@1": 0.23787528868360278, + "f1@5": 0.21709006928406474, + "f1@10": 0.1818181818181818, + "map@1": 0.23787528868360278, + "map@5": 0.3814087759815243, + "map@10": 0.4303328567762748, + "mrr@1": 0.23787528868360278, + "mrr@5": 0.3814087759815243, + "mrr@10": 0.4303328567762748, + "num_pred": 433, + "num_data": 433 }, - "ViDoRe_tatdqa": { - "hit@1": 0.17071688942891858, - "hit@5": 0.37484811664641554, - "hit@10": 0.4756986634264885, - "ndcg_linear@1": 0.17071688942891858, - "ndcg_linear@5": 0.27623880236260684, - "ndcg_linear@10": 0.30892627107009063, - "ndcg_exponential@1": 0.17071688942891858, - "ndcg_exponential@5": 0.27623880236260684, - "ndcg_exponential@10": 0.30892627107009063, - "precision@1": 0.17071688942891858, - "precision@5": 0.07509113001215068, - "precision@10": 0.047691373025516404, - "recall@1": 0.1704131227217497, - "recall@5": 0.3745443499392467, - "recall@10": 0.4756986634264885, - "f1@1": 0.17051437829080598, - "f1@5": 0.12506509286582193, - "f1@10": 0.08667476711219119, - "map@1": 0.17071688942891858, - "map@5": 0.24375759416767923, - "map@10": 0.25726243514821884, - "mrr@1": 0.17071688942891858, - "mrr@5": 0.24383353584447146, - "mrr@10": 0.2572371212559548, - "num_pred": 1646, - "num_data": 1646 + "MVBench": { + "hit@1": 0.28475, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.28475, + "ndcg_linear@5": 0.6742128560665944, + "ndcg_linear@10": 0.6742128560665944, + "ndcg_exponential@1": 0.28475, + "ndcg_exponential@5": 0.6742128560665944, + "ndcg_exponential@10": 0.6742128560665944, + "precision@1": 0.28475, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.28475, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.28475, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.28475, + "map@5": 0.5643916666666666, + "map@10": 0.5643916666666666, + "mrr@1": 0.28475, + "mrr@5": 0.5643916666666666, + "mrr@10": 0.5643916666666666, + "num_pred": 4000, + "num_data": 4000 }, - "ViDoRe_shiftproject": { - "hit@1": 0.23, - "hit@5": 0.52, - "hit@10": 0.63, - "ndcg_linear@1": 0.23, - "ndcg_linear@5": 0.3862145807098324, - "ndcg_linear@10": 0.4228517985151625, - "ndcg_exponential@1": 0.23, - "ndcg_exponential@5": 0.3862145807098324, - "ndcg_exponential@10": 0.4228517985151625, - "precision@1": 0.23, - "precision@5": 0.10399999999999998, - "precision@10": 0.06299999999999999, - "recall@1": 0.23, - "recall@5": 0.52, - "recall@10": 0.63, - "f1@1": 0.23, - "f1@5": 0.17333333333333337, - "f1@10": 0.11454545454545455, - "map@1": 0.23, - "map@5": 0.34149999999999997, - "map@10": 0.3572420634920635, - "mrr@1": 0.23, - "mrr@5": 0.34149999999999997, - "mrr@10": 0.3572420634920635, - "num_pred": 100, - "num_data": 100 + "Video-MME": { + "hit@1": 0.27814814814814814, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.27814814814814814, + "ndcg_linear@5": 0.660078255498731, + "ndcg_linear@10": 0.660078255498731, + "ndcg_exponential@1": 0.27814814814814814, + "ndcg_exponential@5": 0.660078255498731, + "ndcg_exponential@10": 0.660078255498731, + "precision@1": 0.27814814814814814, + "precision@5": 0.19999999999999996, + "precision@10": 0.09999999999999998, + "recall@1": 0.27814814814814814, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.27814814814814814, + "f1@5": 0.3333333333333334, + "f1@10": 0.18181818181818188, + "map@1": 0.27814814814814814, + "map@5": 0.5465432098765431, + "map@10": 0.5465432098765431, + "mrr@1": 0.27814814814814814, + "mrr@5": 0.5465432098765431, + "mrr@10": 0.5465432098765431, + "num_pred": 2700, + "num_data": 2700 }, - "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.55, - "hit@5": 0.78, - "hit@10": 0.82, - "ndcg_linear@1": 0.55, - "ndcg_linear@5": 0.6765467489235274, - "ndcg_linear@10": 0.6892234506542988, - "ndcg_exponential@1": 0.55, - "ndcg_exponential@5": 0.6765467489235274, - "ndcg_exponential@10": 0.6892234506542988, - "precision@1": 0.55, - "precision@5": 0.15599999999999997, - "precision@10": 0.08199999999999999, - "recall@1": 0.55, - "recall@5": 0.78, - "recall@10": 0.82, - "f1@1": 0.55, - "f1@5": 0.26000000000000006, - "f1@10": 0.1490909090909091, - "map@1": 0.55, - "map@5": 0.6418333333333334, - "map@10": 0.6469285714285715, - "mrr@1": 0.55, - "mrr@5": 0.6418333333333334, - "mrr@10": 0.6469285714285715, - "num_pred": 100, - "num_data": 100 + "NExTQA": { + "hit@1": 0.20294255021018215, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.20294255021018215, + "ndcg_linear@5": 0.604249894952272, + "ndcg_linear@10": 0.604249894952272, + "ndcg_exponential@1": 0.20294255021018215, + "ndcg_exponential@5": 0.604249894952272, + "ndcg_exponential@10": 0.604249894952272, + "precision@1": 0.20294255021018215, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.20294255021018215, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.20294255021018215, + "f1@5": 0.3333333333333333, + "f1@10": 0.1818181818181818, + "map@1": 0.20294255021018215, + "map@5": 0.4747664642690332, + "map@10": 0.4747664642690332, + "mrr@1": 0.20294255021018215, + "mrr@5": 0.4746029892573564, + "mrr@10": 0.4746029892573564, + "num_pred": 8564, + "num_data": 8564 }, - "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.49, - "hit@5": 0.7, - "hit@10": 0.85, - "ndcg_linear@1": 0.49, - "ndcg_linear@5": 0.604188565752427, - "ndcg_linear@10": 0.654079614057483, - "ndcg_exponential@1": 0.49, - "ndcg_exponential@5": 0.604188565752427, - "ndcg_exponential@10": 0.654079614057483, - "precision@1": 0.49, - "precision@5": 0.13999999999999996, - "precision@10": 0.08499999999999996, - "recall@1": 0.49, - "recall@5": 0.7, - "recall@10": 0.85, - "f1@1": 0.49, - "f1@5": 0.23333333333333336, - "f1@10": 0.15454545454545457, - "map@1": 0.49, - "map@5": 0.572, - "map@10": 0.5934246031746032, - "mrr@1": 0.49, - "mrr@5": 0.572, - "mrr@10": 0.5934246031746032, - "num_pred": 100, - "num_data": 100 + "EgoSchema": { + "hit@1": 0.218, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.218, + "ndcg_linear@5": 0.5992262804502428, + "ndcg_linear@10": 0.5992262804502428, + "ndcg_exponential@1": 0.218, + "ndcg_exponential@5": 0.5992262804502428, + "ndcg_exponential@10": 0.5992262804502428, + "precision@1": 0.218, + "precision@5": 0.2, + "precision@10": 0.1, + "recall@1": 0.218, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.218, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.218, + "map@5": 0.46926666666666667, + "map@10": 0.46926666666666667, + "mrr@1": 0.218, + "mrr@5": 0.46926666666666667, + "mrr@10": 0.46926666666666667, + "num_pred": 500, + "num_data": 500 }, - "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.42, - "hit@5": 0.78, - "hit@10": 0.88, - "ndcg_linear@1": 0.42, - "ndcg_linear@5": 0.6181938909073429, - "ndcg_linear@10": 0.6501901901304684, - "ndcg_exponential@1": 0.42, - "ndcg_exponential@5": 0.6181938909073429, - "ndcg_exponential@10": 0.6501901901304684, - "precision@1": 0.42, - "precision@5": 0.15599999999999997, - "precision@10": 0.088, - "recall@1": 0.42, - "recall@5": 0.78, - "recall@10": 0.88, - "f1@1": 0.42, - "f1@5": 0.26000000000000006, - "f1@10": 0.16000000000000003, - "map@1": 0.42, - "map@5": 0.5638333333333334, - "map@10": 0.5768253968253969, - "mrr@1": 0.42, - "mrr@5": 0.5638333333333334, - "mrr@10": 0.5768253968253969, - "num_pred": 100, - "num_data": 100 + "ActivityNetQA": { + "hit@1": 0.514, + "hit@5": 1.0, + "hit@10": 1.0, + "ndcg_linear@1": 0.514, + "ndcg_linear@5": 0.8206318602357284, + "ndcg_linear@10": 0.8206318602357284, + "ndcg_exponential@1": 0.514, + "ndcg_exponential@5": 0.8206318602357284, + "ndcg_exponential@10": 0.8206318602357284, + "precision@1": 0.514, + "precision@5": 0.20000000000000004, + "precision@10": 0.10000000000000002, + "recall@1": 0.514, + "recall@5": 1.0, + "recall@10": 1.0, + "f1@1": 0.514, + "f1@5": 0.3333333333333333, + "f1@10": 0.18181818181818182, + "map@1": 0.514, + "map@5": 0.757, + "map@10": 0.757, + "mrr@1": 0.514, + "mrr@5": 0.757, + "mrr@10": 0.757, + "num_pred": 1000, + "num_data": 1000 }, - "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.54, - "hit@5": 0.82, - "hit@10": 0.89, - "ndcg_linear@1": 0.54, - "ndcg_linear@5": 0.698531474110489, - "ndcg_linear@10": 0.7212229678465993, - "ndcg_exponential@1": 0.54, - "ndcg_exponential@5": 0.698531474110489, - "ndcg_exponential@10": 0.7212229678465993, - "precision@1": 0.54, - "precision@5": 0.16399999999999998, - "precision@10": 0.08899999999999997, - "recall@1": 0.54, - "recall@5": 0.82, - "recall@10": 0.89, - "f1@1": 0.54, - "f1@5": 0.2733333333333334, - "f1@10": 0.16181818181818186, - "map@1": 0.54, - "map@5": 0.6573333333333333, - "map@10": 0.6667619047619047, - "mrr@1": 0.54, - "mrr@5": 0.6573333333333333, - "mrr@10": 0.6667619047619047, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_esg_reports_human_labeled_v2": { - "hit@1": 0.038461538461538464, - "hit@5": 0.11538461538461539, - "hit@10": 0.23076923076923078, - "ndcg_linear@1": 0.038461538461538464, - "ndcg_linear@5": 0.06817512332990149, - "ndcg_linear@10": 0.09598030466462644, - "ndcg_exponential@1": 0.038461538461538464, - "ndcg_exponential@5": 0.06817512332990149, - "ndcg_exponential@10": 0.09599487668438379, - "precision@1": 0.038461538461538464, - "precision@5": 0.023076923076923075, - "precision@10": 0.026923076923076925, - "recall@1": 0.038461538461538464, - "recall@5": 0.09134615384615384, - "recall@10": 0.16506410256410253, - "f1@1": 0.038461538461538464, - "f1@5": 0.03540903540903541, - "f1@10": 0.04333166833166832, - "map@1": 0.038461538461538464, - "map@5": 0.057211538461538466, - "map@10": 0.06807844932844934, - "mrr@1": 0.038461538461538464, - "mrr@5": 0.06634615384615385, - "mrr@10": 0.08041819291819292, - "num_pred": 52, - "num_data": 52 - }, - "ViDoRe_biomedical_lectures_v2": { - "hit@1": 0.06875, - "hit@5": 0.16875, - "hit@10": 0.25625, - "ndcg_linear@1": 0.06875, - "ndcg_linear@5": 0.07362375599428807, - "ndcg_linear@10": 0.091648012109416, - "ndcg_exponential@1": 0.06875, - "ndcg_exponential@5": 0.07362375599428807, - "ndcg_exponential@10": 0.091648012109416, - "precision@1": 0.06875, - "precision@5": 0.036250000000000004, - "precision@10": 0.030625000000000003, - "recall@1": 0.03859375, - "recall@5": 0.07802222593582889, - "recall@10": 0.13151451489686786, - "f1@1": 0.04460858585858586, - "f1@5": 0.04316730144855145, - "f1@10": 0.04397036229021523, - "map@1": 0.06875, - "map@5": 0.05524305555555555, - "map@10": 0.060789723875661374, - "mrr@1": 0.06875, - "mrr@5": 0.10270833333333333, - "mrr@10": 0.11407490079365079, - "num_pred": 160, - "num_data": 160 + "DiDeMo": { + "hit@1": 0.29282868525896416, + "hit@5": 0.5348605577689243, + "hit@10": 0.6215139442231076, + "ndcg_linear@1": 0.29282868525896416, + "ndcg_linear@5": 0.42005110007964297, + "ndcg_linear@10": 0.4483095791534034, + "ndcg_exponential@1": 0.29282868525896416, + "ndcg_exponential@5": 0.42005110007964297, + "ndcg_exponential@10": 0.4483095791534034, + "precision@1": 0.29282868525896416, + "precision@5": 0.10697211155378486, + "precision@10": 0.062151394422310755, + "recall@1": 0.29282868525896416, + "recall@5": 0.5348605577689243, + "recall@10": 0.6215139442231076, + "f1@1": 0.29282868525896416, + "f1@5": 0.1782868525896415, + "f1@10": 0.1130025353132923, + "map@1": 0.29282868525896416, + "map@5": 0.3820053120849934, + "map@10": 0.3938029627521659, + "mrr@1": 0.29282868525896416, + "mrr@5": 0.3820053120849934, + "mrr@10": 0.3938029627521659, + "num_pred": 1004, + "num_data": 1004 }, - "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.05, - "hit@5": 0.1203125, - "hit@10": 0.1875, - "ndcg_linear@1": 0.05, - "ndcg_linear@5": 0.05064502600213018, - "ndcg_linear@10": 0.06329515996638582, - "ndcg_exponential@1": 0.05, - "ndcg_exponential@5": 0.05064502600213018, - "ndcg_exponential@10": 0.06329515996638582, - "precision@1": 0.05, - "precision@5": 0.025312500000000005, - "precision@10": 0.022031250000000002, - "recall@1": 0.027369791666666664, - "recall@5": 0.052070591020499116, - "recall@10": 0.08876272241851287, - "f1@1": 0.031846196338383835, - "f1@5": 0.029338837377899885, - "f1@10": 0.031196784815810553, - "map@1": 0.05, - "map@5": 0.03780381944444444, - "map@10": 0.041590970155423276, - "mrr@1": 0.05, - "mrr@5": 0.07333333333333333, - "mrr@10": 0.08225384424603174, - "num_pred": 640, - "num_data": 640 + "MSR-VTT": { + "hit@1": 0.345, + "hit@5": 0.585, + "hit@10": 0.679, + "ndcg_linear@1": 0.345, + "ndcg_linear@5": 0.4705689385818224, + "ndcg_linear@10": 0.5008515039770559, + "ndcg_exponential@1": 0.345, + "ndcg_exponential@5": 0.4705689385818224, + "ndcg_exponential@10": 0.5008515039770559, + "precision@1": 0.345, + "precision@5": 0.11699999999999998, + "precision@10": 0.06789999999999999, + "recall@1": 0.345, + "recall@5": 0.585, + "recall@10": 0.679, + "f1@1": 0.345, + "f1@5": 0.19500000000000006, + "f1@10": 0.12345454545454547, + "map@1": 0.345, + "map@5": 0.4327, + "map@10": 0.4451373015873016, + "mrr@1": 0.345, + "mrr@5": 0.4327, + "mrr@10": 0.4451373015873016, + "num_pred": 1000, + "num_data": 1000 }, - "ViDoRe_economics_reports_v2": { - "hit@1": 0.1896551724137931, - "hit@5": 0.43103448275862066, - "hit@10": 0.5689655172413793, - "ndcg_linear@1": 0.1896551724137931, - "ndcg_linear@5": 0.12321848766469609, - "ndcg_linear@10": 0.14235630052874335, - "ndcg_exponential@1": 0.1896551724137931, - "ndcg_exponential@5": 0.12321848766469606, - "ndcg_exponential@10": 0.14235630052874335, - "precision@1": 0.1896551724137931, - "precision@5": 0.11379310344827587, - "precision@10": 0.11551724137931033, - "recall@1": 0.01551192197008164, - "recall@5": 0.04012935831774089, - "recall@10": 0.12115233423869314, - "f1@1": 0.027570734115652442, - "f1@5": 0.05288902461685153, - "f1@10": 0.08639297997440117, - "map@1": 0.1896551724137931, - "map@5": 0.06747126436781609, - "map@10": 0.06376870096697683, - "mrr@1": 0.1896551724137931, - "mrr@5": 0.2614942528735632, - "mrr@10": 0.27941981390257253, - "num_pred": 58, - "num_data": 58 + "MSVD": { + "hit@1": 0.4671641791044776, + "hit@5": 0.744776119402985, + "hit@10": 0.8313432835820895, + "ndcg_linear@1": 0.4671641791044776, + "ndcg_linear@5": 0.6143164752221116, + "ndcg_linear@10": 0.6423987059237167, + "ndcg_exponential@1": 0.4671641791044776, + "ndcg_exponential@5": 0.6143164752221116, + "ndcg_exponential@10": 0.6423987059237167, + "precision@1": 0.4671641791044776, + "precision@5": 0.14895522388059698, + "precision@10": 0.08313432835820894, + "recall@1": 0.4671641791044776, + "recall@5": 0.744776119402985, + "recall@10": 0.8313432835820895, + "f1@1": 0.4671641791044776, + "f1@5": 0.24825870646766174, + "f1@10": 0.15115332428765269, + "map@1": 0.4671641791044776, + "map@5": 0.5709203980099502, + "map@10": 0.5825675195451315, + "mrr@1": 0.4671641791044776, + "mrr@5": 0.5709203980099502, + "mrr@10": 0.5825675195451315, + "num_pred": 670, + "num_data": 670 }, - "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.1810344827586207, - "hit@5": 0.4224137931034483, - "hit@10": 0.5646551724137931, - "ndcg_linear@1": 0.1810344827586207, - "ndcg_linear@5": 0.13875113955781276, - "ndcg_linear@10": 0.14758177623224894, - "ndcg_exponential@1": 0.1810344827586207, - "ndcg_exponential@5": 0.1387511395578127, - "ndcg_exponential@10": 0.14758177623224894, - "precision@1": 0.1810344827586207, - "precision@5": 0.11982758620689656, - "precision@10": 0.11551724137931037, - "recall@1": 0.012707666017446384, - "recall@5": 0.06501080124572646, - "recall@10": 0.12018603229901821, - "f1@1": 0.02288545832709266, - "f1@5": 0.06054340208235827, - "f1@10": 0.08247544424998887, - "map@1": 0.1810344827586207, - "map@5": 0.08251436781609195, - "map@10": 0.07246543932068357, - "mrr@1": 0.1810344827586207, - "mrr@5": 0.2608477011494253, - "mrr@10": 0.27921798029556655, - "num_pred": 232, - "num_data": 232 + "VATEX": { + "hit@1": 0.25524787851719516, + "hit@5": 0.5064761054041983, + "hit@10": 0.6192496650290308, + "ndcg_linear@1": 0.25524787851719516, + "ndcg_linear@5": 0.38626309581533674, + "ndcg_linear@10": 0.42289114395020533, + "ndcg_exponential@1": 0.25524787851719516, + "ndcg_exponential@5": 0.38626309581533674, + "ndcg_exponential@10": 0.42289114395020533, + "precision@1": 0.25524787851719516, + "precision@5": 0.10129522108083966, + "precision@10": 0.06192496650290307, + "recall@1": 0.25524787851719516, + "recall@5": 0.5064761054041983, + "recall@10": 0.6192496650290308, + "f1@1": 0.25524787851719516, + "f1@5": 0.16882536846806612, + "f1@10": 0.11259084818709653, + "map@1": 0.25524787851719516, + "map@5": 0.34648280482358196, + "map@10": 0.3616905754411337, + "mrr@1": 0.25524787851719516, + "mrr@5": 0.34648280482358196, + "mrr@10": 0.3616905754411337, + "num_pred": 4478, + "num_data": 4478 }, - "ViDoRe_esg_reports_v2": { - "hit@1": 0.08771929824561403, - "hit@5": 0.12280701754385964, - "hit@10": 0.24561403508771928, - "ndcg_linear@1": 0.08771929824561403, - "ndcg_linear@5": 0.08634423644624155, - "ndcg_linear@10": 0.10972244602793464, - "ndcg_exponential@1": 0.08771929824561403, - "ndcg_exponential@5": 0.08634423644624155, - "ndcg_exponential@10": 0.10972244602793464, - "precision@1": 0.08771929824561403, - "precision@5": 0.03508771929824561, - "precision@10": 0.03508771929824561, - "recall@1": 0.06432748538011697, - "recall@5": 0.08020050125313283, - "recall@10": 0.14223057644110276, - "f1@1": 0.06934001670843776, - "f1@5": 0.03926482873851295, - "f1@10": 0.04791426640457111, - "map@1": 0.08771929824561403, - "map@5": 0.07622807017543859, - "map@10": 0.0842564745196324, - "mrr@1": 0.08771929824561403, - "mrr@5": 0.10087719298245613, - "mrr@10": 0.11707741576162628, - "num_pred": 57, - "num_data": 57 + "YouCook2": { + "hit@1": 0.08996539792387544, + "hit@5": 0.21453287197231835, + "hit@10": 0.28908461780434097, + "ndcg_linear@1": 0.08996539792387544, + "ndcg_linear@5": 0.15381567274119107, + "ndcg_linear@10": 0.1779025046239055, + "ndcg_exponential@1": 0.08996539792387544, + "ndcg_exponential@5": 0.15381567274119107, + "ndcg_exponential@10": 0.1779025046239055, + "precision@1": 0.08996539792387544, + "precision@5": 0.04290657439446367, + "precision@10": 0.0289084617804341, + "recall@1": 0.08996539792387544, + "recall@5": 0.21453287197231835, + "recall@10": 0.28908461780434097, + "f1@1": 0.08996539792387544, + "f1@5": 0.07151095732410613, + "f1@10": 0.05256083960078927, + "map@1": 0.08996539792387544, + "map@5": 0.13384187899758834, + "map@10": 0.14376925458240336, + "mrr@1": 0.08996539792387544, + "mrr@5": 0.13384187899758834, + "mrr@10": 0.14376925458240336, + "num_pred": 3179, + "num_data": 3179 }, - "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.08333333333333333, - "hit@5": 0.2324561403508772, - "hit@10": 0.31140350877192985, - "ndcg_linear@1": 0.08333333333333333, - "ndcg_linear@5": 0.11890188811957679, - "ndcg_linear@10": 0.1391360634250716, - "ndcg_exponential@1": 0.08333333333333333, - "ndcg_exponential@5": 0.11890188811957679, - "ndcg_exponential@10": 0.1391360634250716, - "precision@1": 0.08333333333333333, - "precision@5": 0.05964912280701755, - "precision@10": 0.04649122807017544, - "recall@1": 0.05936716791979949, - "recall@5": 0.1361737677527151, - "recall@10": 0.18859649122807018, - "f1@1": 0.06306354967879468, - "f1@5": 0.07183557899502171, - "f1@10": 0.06509698173479336, - "map@1": 0.08333333333333333, - "map@5": 0.09392543859649122, - "map@10": 0.10152116402116403, - "mrr@1": 0.08333333333333333, - "mrr@5": 0.1355263157894737, - "mrr@10": 0.14601434140907826, - "num_pred": 228, - "num_data": 228 - }, - "VisRAG_ArxivQA": { - "hit@1": 0.43504901960784315, - "hit@5": 0.6029411764705882, - "hit@10": 0.6727941176470589, - "ndcg_linear@1": 0.43504901960784315, - "ndcg_linear@5": 0.5260731896593214, - "ndcg_linear@10": 0.5488301957135047, - "ndcg_exponential@1": 0.43504901960784315, - "ndcg_exponential@5": 0.5260731896593214, - "ndcg_exponential@10": 0.5488301957135047, - "precision@1": 0.43504901960784315, - "precision@5": 0.12058823529411763, - "precision@10": 0.06727941176470587, - "recall@1": 0.43504901960784315, - "recall@5": 0.6029411764705882, - "recall@10": 0.6727941176470589, - "f1@1": 0.43504901960784315, - "f1@5": 0.2009803921568628, - "f1@10": 0.12232620320855617, - "map@1": 0.43504901960784315, - "map@5": 0.5003472222222222, - "map@10": 0.5098355314347962, - "mrr@1": 0.43504901960784315, - "mrr@5": 0.5003472222222222, - "mrr@10": 0.5098355314347962, - "num_pred": 816, - "num_data": 816 - }, - "VisRAG_ChartQA": { - "hit@1": 0.6031746031746031, - "hit@5": 0.7936507936507936, - "hit@10": 0.8412698412698413, - "ndcg_linear@1": 0.6031746031746031, - "ndcg_linear@5": 0.7023156150418144, - "ndcg_linear@10": 0.7176213783260175, - "ndcg_exponential@1": 0.6031746031746031, - "ndcg_exponential@5": 0.7023156150418144, - "ndcg_exponential@10": 0.7176213783260175, - "precision@1": 0.6031746031746031, - "precision@5": 0.15873015873015867, - "precision@10": 0.08412698412698409, - "recall@1": 0.6031746031746031, - "recall@5": 0.7936507936507936, - "recall@10": 0.8412698412698413, - "f1@1": 0.6031746031746031, - "f1@5": 0.2645502645502646, - "f1@10": 0.15295815295815296, - "map@1": 0.6031746031746031, - "map@5": 0.6722222222222223, - "map@10": 0.6784580498866214, - "mrr@1": 0.6031746031746031, - "mrr@5": 0.6722222222222223, - "mrr@10": 0.6784580498866214, - "num_pred": 63, - "num_data": 63 - }, - "VisRAG_MP-DocVQA": { - "hit@1": 0.4077834179357022, - "hit@5": 0.6395939086294417, - "hit@10": 0.7123519458544839, - "ndcg_linear@1": 0.4077834179357022, - "ndcg_linear@5": 0.5280813677279776, - "ndcg_linear@10": 0.5522909877969642, - "ndcg_exponential@1": 0.4077834179357022, - "ndcg_exponential@5": 0.5280813677279776, - "ndcg_exponential@10": 0.5522909877969642, - "precision@1": 0.4077834179357022, - "precision@5": 0.1279187817258883, - "precision@10": 0.07123519458544839, - "recall@1": 0.4077834179357022, - "recall@5": 0.6395939086294417, - "recall@10": 0.7123519458544839, - "f1@1": 0.4077834179357022, - "f1@5": 0.2131979695431472, - "f1@10": 0.1295185356099062, - "map@1": 0.4077834179357022, - "map@5": 0.4912859560067682, - "map@10": 0.5016725754035398, - "mrr@1": 0.4077834179357022, - "mrr@5": 0.4912859560067682, - "mrr@10": 0.5016725754035398, - "num_pred": 591, - "num_data": 591 - }, - "VisRAG_SlideVQA": { - "hit@1": 0.6258992805755396, - "hit@5": 0.85431654676259, - "hit@10": 0.8974820143884892, - "ndcg_linear@1": 0.6258992805755396, - "ndcg_linear@5": 0.727722018029152, - "ndcg_linear@10": 0.7442517359673964, - "ndcg_exponential@1": 0.6258992805755396, - "ndcg_exponential@5": 0.727722018029152, - "ndcg_exponential@10": 0.7442517359673964, - "precision@1": 0.6258992805755396, - "precision@5": 0.2079136690647482, - "precision@10": 0.10953237410071942, - "recall@1": 0.5314748201438849, - "recall@5": 0.822841726618705, - "recall@10": 0.8714028776978417, - "f1@1": 0.5629496402877698, - "f1@5": 0.3258821514217198, - "f1@10": 0.19236429038587313, - "map@1": 0.6258992805755396, - "map@5": 0.6828387290167865, - "map@10": 0.6904986724905788, - "mrr@1": 0.6258992805755396, - "mrr@5": 0.7175659472422062, - "mrr@10": 0.7235775665182139, - "num_pred": 556, - "num_data": 556 - }, - "VisRAG_InfoVQA": { - "hit@1": 0.6002785515320335, - "hit@5": 0.8245125348189415, - "hit@10": 0.8899721448467967, - "ndcg_linear@1": 0.6002785515320335, - "ndcg_linear@5": 0.7196103081823707, - "ndcg_linear@10": 0.7410456616380353, - "ndcg_exponential@1": 0.6002785515320335, - "ndcg_exponential@5": 0.7196103081823707, - "ndcg_exponential@10": 0.7410456616380353, - "precision@1": 0.6002785515320335, - "precision@5": 0.16490250696378828, - "precision@10": 0.08899721448467965, - "recall@1": 0.6002785515320335, - "recall@5": 0.8245125348189415, - "recall@10": 0.8899721448467967, - "f1@1": 0.6002785515320335, - "f1@5": 0.27483751160631387, - "f1@10": 0.16181311724487216, - "map@1": 0.6002785515320335, - "map@5": 0.6846564531104921, - "map@10": 0.6936557677852943, - "mrr@1": 0.6002785515320335, - "mrr@5": 0.6846564531104921, - "mrr@10": 0.6936557677852943, - "num_pred": 718, - "num_data": 718 - }, - "VisRAG_PlotQA": { - "hit@1": 0.22363847045191193, - "hit@5": 0.4484356894553882, - "hit@10": 0.5654692931633836, - "ndcg_linear@1": 0.22363847045191193, - "ndcg_linear@5": 0.3434876959499088, - "ndcg_linear@10": 0.3813457018404914, - "ndcg_exponential@1": 0.22363847045191193, - "ndcg_exponential@5": 0.3434876959499088, - "ndcg_exponential@10": 0.3813457018404914, - "precision@1": 0.22363847045191193, - "precision@5": 0.08968713789107764, - "precision@10": 0.05654692931633834, - "recall@1": 0.22363847045191193, - "recall@5": 0.4484356894553882, - "recall@10": 0.5654692931633836, - "f1@1": 0.22363847045191193, - "f1@5": 0.14947856315179606, - "f1@10": 0.10281259875697883, - "map@1": 0.22363847045191193, - "map@5": 0.3085940517574354, - "map@10": 0.32422382239879344, - "mrr@1": 0.22363847045191193, - "mrr@5": 0.3085940517574354, - "mrr@10": 0.32422382239879344, - "num_pred": 863, - "num_data": 863 - }, - "ViDoSeek-page": { - "hit@1": 0.0446584938704028, - "hit@5": 0.3905429071803853, - "hit@10": 0.5971978984238179, - "ndcg_linear@1": 0.0446584938704028, - "ndcg_linear@5": 0.22283683461024645, - "ndcg_linear@10": 0.2895264993048246, - "ndcg_exponential@1": 0.0446584938704028, - "ndcg_exponential@5": 0.22283683461024653, - "ndcg_exponential@10": 0.28952649930482466, - "precision@1": 0.0446584938704028, - "precision@5": 0.07810858143607707, - "precision@10": 0.05971978984238179, - "recall@1": 0.0446584938704028, - "recall@5": 0.3905429071803853, - "recall@10": 0.5971978984238179, - "f1@1": 0.0446584938704028, - "f1@5": 0.13018096906012844, - "f1@10": 0.10858143607705782, - "map@1": 0.0446584938704028, - "map@5": 0.1676444833625219, - "map@10": 0.19508763517082256, - "mrr@1": 0.0446584938704028, - "mrr@5": 0.1676444833625219, - "mrr@10": 0.19508763517082256, - "num_pred": 1142, - "num_data": 1142 - }, - "ViDoSeek-doc": { - "hit@1": 0.968476357267951, - "hit@5": 0.9912434325744308, - "hit@10": 0.9947460595446584, - "ndcg_linear@1": 0.6625802685347344, - "ndcg_linear@5": 0.7780718841793594, - "ndcg_linear@10": 0.7502728417738824, - "ndcg_exponential@1": 0.44408306229672245, - "ndcg_exponential@5": 0.6673711755462026, - "ndcg_exponential@10": 0.6819104114803104, - "precision@1": 0.968476357267951, - "precision@5": 0.8481611208406304, - "precision@10": 0.750875656742557, - "recall@1": 0.05711879676554371, - "recall@5": 0.24781081127789126, - "recall@10": 0.4314558336731064, - "f1@1": 0.10713328875583096, - "f1@5": 0.375858198266283, - "f1@10": 0.5355661581210087, - "map@1": 0.968476357267951, - "map@5": 0.831468184471687, - "map@10": 0.7255596248277478, - "mrr@1": 0.968476357267951, - "mrr@5": 0.9776999416228838, - "mrr@10": 0.9781680010007505, - "num_pred": 1142, - "num_data": 1142 + "QVHighlight": { + "hit@1": 0.577100646352724, + "hit@5": 0.8947368421052632, + "hit@10": 1.0, + "ndcg_linear@1": 0.577100646352724, + "ndcg_linear@5": 0.7461876317485063, + "ndcg_linear@10": 0.7806995799400307, + "ndcg_exponential@1": 0.577100646352724, + "ndcg_exponential@5": 0.7461876317485063, + "ndcg_exponential@10": 0.7806995799400307, + "precision@1": 0.577100646352724, + "precision@5": 0.17894736842105263, + "precision@10": 0.09999999999999998, + "recall@1": 0.577100646352724, + "recall@5": 0.8947368421052632, + "recall@10": 1.0, + "f1@1": 0.577100646352724, + "f1@5": 0.2982456140350878, + "f1@10": 0.18181818181818185, + "map@1": 0.577100646352724, + "map@5": 0.6967220683287165, + "map@10": 0.7112386228729719, + "mrr@1": 0.577100646352724, + "mrr@5": 0.6967220683287165, + "mrr@10": 0.7112386228729719, + "num_pred": 1083, + "num_data": 1083 }, - "MMLongBench-page": { - "hit@1": 0.06801909307875895, - "hit@5": 0.2386634844868735, - "hit@10": 0.3305489260143198, - "ndcg_linear@1": 0.06801909307875895, - "ndcg_linear@5": 0.11822134474246979, - "ndcg_linear@10": 0.15022040843373538, - "ndcg_exponential@1": 0.06801909307875895, - "ndcg_exponential@5": 0.11822134474246979, - "ndcg_exponential@10": 0.15022040843373538, - "precision@1": 0.06801909307875895, - "precision@5": 0.056324582338902154, - "precision@10": 0.04379474940334129, - "recall@1": 0.038664905102852594, - "recall@5": 0.16723964844489903, - "recall@10": 0.25455198143145635, - "f1@1": 0.04688230756249849, - "f1@5": 0.07989331008424089, - "f1@10": 0.07145515842987549, - "map@1": 0.06801909307875895, - "map@5": 0.0867468841156192, - "map@10": 0.10167219917667411, - "mrr@1": 0.06801909307875895, - "mrr@5": 0.12738663484486873, - "mrr@10": 0.13961955904080012, - "num_pred": 838, - "num_data": 838 + "Charades-STA": { + "hit@1": 0.19807427785419532, + "hit@5": 0.5818431911966988, + "hit@10": 1.0, + "ndcg_linear@1": 0.19807427785419532, + "ndcg_linear@5": 0.3931810758884146, + "ndcg_linear@10": 0.5256230138842131, + "ndcg_exponential@1": 0.19807427785419532, + "ndcg_exponential@5": 0.3931810758884146, + "ndcg_exponential@10": 0.5256230138842131, + "precision@1": 0.19807427785419532, + "precision@5": 0.11636863823933974, + "precision@10": 0.09999999999999998, + "recall@1": 0.19807427785419532, + "recall@5": 0.5818431911966988, + "recall@10": 1.0, + "f1@1": 0.19807427785419532, + "f1@5": 0.19394773039889962, + "f1@10": 0.18181818181818185, + "map@1": 0.19807427785419532, + "map@5": 0.3313388353966071, + "map@10": 0.38442446671470054, + "mrr@1": 0.19807427785419532, + "mrr@5": 0.3313388353966071, + "mrr@10": 0.38442446671470054, + "num_pred": 727, + "num_data": 727 }, - "MMLongBench-doc": { - "hit@1": 0.5668257756563246, - "hit@5": 0.6885441527446301, - "hit@10": 0.747016706443914, - "ndcg_linear@1": 0.3981702466189339, - "ndcg_linear@5": 0.40462422635020384, - "ndcg_linear@10": 0.38948040985289195, - "ndcg_exponential@1": 0.27770201159222635, - "ndcg_exponential@5": 0.33626701840937107, - "ndcg_exponential@10": 0.34529122789195027, - "precision@1": 0.5668257756563246, - "precision@5": 0.4470167064439141, - "precision@10": 0.3953460620525059, - "recall@1": 0.017686024380438876, - "recall@5": 0.06659662623490364, - "recall@10": 0.11644775779833906, - "f1@1": 0.03396785786854894, - "f1@5": 0.11131694335938004, - "f1@10": 0.16871497861256565, - "map@1": 0.5668257756563246, - "map@5": 0.41111774065234685, - "map@10": 0.34533373426273184, - "mrr@1": 0.5668257756563246, - "mrr@5": 0.6124105011933175, - "mrr@10": 0.6201973709133614, - "num_pred": 838, - "num_data": 838 + "MomentSeeker": { + "hit@1": 0.3927777777777778, + "hit@5": 0.8838888888888888, + "hit@10": 1.0, + "ndcg_linear@1": 0.3927777777777778, + "ndcg_linear@5": 0.6451319149954187, + "ndcg_linear@10": 0.6903053809896799, + "ndcg_exponential@1": 0.3927777777777778, + "ndcg_exponential@5": 0.6451319149954187, + "ndcg_exponential@10": 0.6903053809896799, + "precision@1": 0.3927777777777778, + "precision@5": 0.19233333333333336, + "precision@10": 0.11294444444444444, + "recall@1": 0.37057870370370366, + "recall@5": 0.8659490740740741, + "recall@10": 0.9986574074074074, + "f1@1": 0.3774488536155203, + "f1@5": 0.31004053045719715, + "f1@10": 0.20030318446985115, + "map@1": 0.3927777777777778, + "map@5": 0.5665493827160493, + "map@10": 0.5877770980305702, + "mrr@1": 0.3927777777777778, + "mrr@5": 0.5764259259259259, + "mrr@10": 0.592172619047619, + "num_pred": 1800, + "num_data": 1800 } }, - "video": { - "DiDeMo": { - "hit@1": 0.29282868525896416, - "hit@5": 0.5348605577689243, - "hit@10": 0.6215139442231076, - "ndcg_linear@1": 0.29282868525896416, - "ndcg_linear@5": 0.42005110007964297, - "ndcg_linear@10": 0.4483095791534034, - "ndcg_exponential@1": 0.29282868525896416, - "ndcg_exponential@5": 0.42005110007964297, - "ndcg_exponential@10": 0.4483095791534034, - "precision@1": 0.29282868525896416, - "precision@5": 0.10697211155378486, - "precision@10": 0.062151394422310755, - "recall@1": 0.29282868525896416, - "recall@5": 0.5348605577689243, - "recall@10": 0.6215139442231076, - "f1@1": 0.29282868525896416, - "f1@5": 0.1782868525896415, - "f1@10": 0.1130025353132923, - "map@1": 0.29282868525896416, - "map@5": 0.3820053120849934, - "map@10": 0.3938029627521659, - "mrr@1": 0.29282868525896416, - "mrr@5": 0.3820053120849934, - "mrr@10": 0.3938029627521659, - "num_pred": 1004, - "num_data": 1004 + "visdoc": { + "ViDoRe_arxivqa": { + "hit@1": 0.506, + "hit@5": 0.684, + "hit@10": 0.736, + "ndcg_linear@1": 0.506, + "ndcg_linear@5": 0.6019848283346673, + "ndcg_linear@10": 0.618842382116875, + "ndcg_exponential@1": 0.506, + "ndcg_exponential@5": 0.6019848283346673, + "ndcg_exponential@10": 0.618842382116875, + "precision@1": 0.506, + "precision@5": 0.13679999999999998, + "precision@10": 0.0736, + "recall@1": 0.506, + "recall@5": 0.684, + "recall@10": 0.736, + "f1@1": 0.506, + "f1@5": 0.22800000000000006, + "f1@10": 0.1338181818181818, + "map@1": 0.506, + "map@5": 0.5745333333333333, + "map@10": 0.5815087301587302, + "mrr@1": 0.506, + "mrr@5": 0.5745333333333333, + "mrr@10": 0.5815087301587302, + "num_pred": 500, + "num_data": 500 + }, + "ViDoRe_docvqa": { + "hit@1": 0.270509977827051, + "hit@5": 0.42572062084257206, + "hit@10": 0.5188470066518847, + "ndcg_linear@1": 0.270509977827051, + "ndcg_linear@5": 0.3471381693961765, + "ndcg_linear@10": 0.3761073342818233, + "ndcg_exponential@1": 0.270509977827051, + "ndcg_exponential@5": 0.3471381693961765, + "ndcg_exponential@10": 0.3761073342818233, + "precision@1": 0.270509977827051, + "precision@5": 0.08558758314855876, + "precision@10": 0.05232815964523281, + "recall@1": 0.2668144863266815, + "recall@5": 0.42165558019216554, + "recall@10": 0.5104212860310421, + "f1@1": 0.2679231337767923, + "f1@5": 0.14195966634991028, + "f1@10": 0.09464696427002414, + "map@1": 0.270509977827051, + "map@5": 0.32139689578713965, + "map@10": 0.33320047161510574, + "mrr@1": 0.270509977827051, + "mrr@5": 0.3251662971175166, + "mrr@10": 0.3374995600605356, + "num_pred": 451, + "num_data": 451 + }, + "ViDoRe_infovqa": { + "hit@1": 0.597165991902834, + "hit@5": 0.7894736842105263, + "hit@10": 0.8522267206477733, + "ndcg_linear@1": 0.597165991902834, + "ndcg_linear@5": 0.7037600234071207, + "ndcg_linear@10": 0.7251514237863242, + "ndcg_exponential@1": 0.597165991902834, + "ndcg_exponential@5": 0.7037600234071207, + "ndcg_exponential@10": 0.7251514237863242, + "precision@1": 0.597165991902834, + "precision@5": 0.15789473684210525, + "precision@10": 0.08542510121457489, + "recall@1": 0.597165991902834, + "recall@5": 0.7884615384615384, + "recall@10": 0.8522267206477733, + "f1@1": 0.597165991902834, + "f1@5": 0.26306149990360517, + "f1@10": 0.1552570236780763, + "map@1": 0.597165991902834, + "map@5": 0.6750674763832659, + "map@10": 0.6844386607544503, + "mrr@1": 0.597165991902834, + "mrr@5": 0.6752699055330635, + "mrr@10": 0.6843037079879185, + "num_pred": 494, + "num_data": 494 }, - "MSR-VTT": { - "hit@1": 0.345, - "hit@5": 0.585, - "hit@10": 0.679, - "ndcg_linear@1": 0.345, - "ndcg_linear@5": 0.4705689385818224, - "ndcg_linear@10": 0.5008515039770559, - "ndcg_exponential@1": 0.345, - "ndcg_exponential@5": 0.4705689385818224, - "ndcg_exponential@10": 0.5008515039770559, - "precision@1": 0.345, - "precision@5": 0.11699999999999998, - "precision@10": 0.06789999999999999, - "recall@1": 0.345, - "recall@5": 0.585, - "recall@10": 0.679, - "f1@1": 0.345, - "f1@5": 0.19500000000000006, - "f1@10": 0.12345454545454547, - "map@1": 0.345, - "map@5": 0.4327, - "map@10": 0.4451373015873016, - "mrr@1": 0.345, - "mrr@5": 0.4327, - "mrr@10": 0.4451373015873016, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_tabfquad": { + "hit@1": 0.675, + "hit@5": 0.8714285714285714, + "hit@10": 0.9178571428571428, + "ndcg_linear@1": 0.675, + "ndcg_linear@5": 0.7818109382746695, + "ndcg_linear@10": 0.7971064823405059, + "ndcg_exponential@1": 0.675, + "ndcg_exponential@5": 0.7818109382746695, + "ndcg_exponential@10": 0.7971064823405059, + "precision@1": 0.675, + "precision@5": 0.17428571428571427, + "precision@10": 0.09178571428571428, + "recall@1": 0.675, + "recall@5": 0.8714285714285714, + "recall@10": 0.9178571428571428, + "f1@1": 0.675, + "f1@5": 0.2904761904761905, + "f1@10": 0.1668831168831169, + "map@1": 0.675, + "map@5": 0.7517857142857143, + "map@10": 0.7582624716553289, + "mrr@1": 0.675, + "mrr@5": 0.7517857142857143, + "mrr@10": 0.7582624716553289, + "num_pred": 280, + "num_data": 280 }, - "MSVD": { - "hit@1": 0.4671641791044776, - "hit@5": 0.744776119402985, - "hit@10": 0.8313432835820895, - "ndcg_linear@1": 0.4671641791044776, - "ndcg_linear@5": 0.6143164752221116, - "ndcg_linear@10": 0.6423987059237167, - "ndcg_exponential@1": 0.4671641791044776, - "ndcg_exponential@5": 0.6143164752221116, - "ndcg_exponential@10": 0.6423987059237167, - "precision@1": 0.4671641791044776, - "precision@5": 0.14895522388059698, - "precision@10": 0.08313432835820894, - "recall@1": 0.4671641791044776, - "recall@5": 0.744776119402985, - "recall@10": 0.8313432835820895, - "f1@1": 0.4671641791044776, - "f1@5": 0.24825870646766174, - "f1@10": 0.15115332428765269, - "map@1": 0.4671641791044776, - "map@5": 0.5709203980099502, - "map@10": 0.5825675195451315, - "mrr@1": 0.4671641791044776, - "mrr@5": 0.5709203980099502, - "mrr@10": 0.5825675195451315, - "num_pred": 670, - "num_data": 670 + "ViDoRe_tatdqa": { + "hit@1": 0.17071688942891858, + "hit@5": 0.37484811664641554, + "hit@10": 0.4756986634264885, + "ndcg_linear@1": 0.17071688942891858, + "ndcg_linear@5": 0.27623880236260684, + "ndcg_linear@10": 0.30892627107009063, + "ndcg_exponential@1": 0.17071688942891858, + "ndcg_exponential@5": 0.27623880236260684, + "ndcg_exponential@10": 0.30892627107009063, + "precision@1": 0.17071688942891858, + "precision@5": 0.07509113001215068, + "precision@10": 0.047691373025516404, + "recall@1": 0.1704131227217497, + "recall@5": 0.3745443499392467, + "recall@10": 0.4756986634264885, + "f1@1": 0.17051437829080598, + "f1@5": 0.12506509286582193, + "f1@10": 0.08667476711219119, + "map@1": 0.17071688942891858, + "map@5": 0.24375759416767923, + "map@10": 0.25726243514821884, + "mrr@1": 0.17071688942891858, + "mrr@5": 0.24383353584447146, + "mrr@10": 0.2572371212559548, + "num_pred": 1646, + "num_data": 1646 }, - "VATEX": { - "hit@1": 0.25524787851719516, - "hit@5": 0.5064761054041983, - "hit@10": 0.6192496650290308, - "ndcg_linear@1": 0.25524787851719516, - "ndcg_linear@5": 0.38626309581533674, - "ndcg_linear@10": 0.42289114395020533, - "ndcg_exponential@1": 0.25524787851719516, - "ndcg_exponential@5": 0.38626309581533674, - "ndcg_exponential@10": 0.42289114395020533, - "precision@1": 0.25524787851719516, - "precision@5": 0.10129522108083966, - "precision@10": 0.06192496650290307, - "recall@1": 0.25524787851719516, - "recall@5": 0.5064761054041983, - "recall@10": 0.6192496650290308, - "f1@1": 0.25524787851719516, - "f1@5": 0.16882536846806612, - "f1@10": 0.11259084818709653, - "map@1": 0.25524787851719516, - "map@5": 0.34648280482358196, - "map@10": 0.3616905754411337, - "mrr@1": 0.25524787851719516, - "mrr@5": 0.34648280482358196, - "mrr@10": 0.3616905754411337, - "num_pred": 4478, - "num_data": 4478 + "ViDoRe_shiftproject": { + "hit@1": 0.23, + "hit@5": 0.52, + "hit@10": 0.63, + "ndcg_linear@1": 0.23, + "ndcg_linear@5": 0.3862145807098324, + "ndcg_linear@10": 0.4228517985151625, + "ndcg_exponential@1": 0.23, + "ndcg_exponential@5": 0.3862145807098324, + "ndcg_exponential@10": 0.4228517985151625, + "precision@1": 0.23, + "precision@5": 0.10399999999999998, + "precision@10": 0.06299999999999999, + "recall@1": 0.23, + "recall@5": 0.52, + "recall@10": 0.63, + "f1@1": 0.23, + "f1@5": 0.17333333333333337, + "f1@10": 0.11454545454545455, + "map@1": 0.23, + "map@5": 0.34149999999999997, + "map@10": 0.3572420634920635, + "mrr@1": 0.23, + "mrr@5": 0.34149999999999997, + "mrr@10": 0.3572420634920635, + "num_pred": 100, + "num_data": 100 }, - "YouCook2": { - "hit@1": 0.08996539792387544, - "hit@5": 0.21453287197231835, - "hit@10": 0.28908461780434097, - "ndcg_linear@1": 0.08996539792387544, - "ndcg_linear@5": 0.15381567274119107, - "ndcg_linear@10": 0.1779025046239055, - "ndcg_exponential@1": 0.08996539792387544, - "ndcg_exponential@5": 0.15381567274119107, - "ndcg_exponential@10": 0.1779025046239055, - "precision@1": 0.08996539792387544, - "precision@5": 0.04290657439446367, - "precision@10": 0.0289084617804341, - "recall@1": 0.08996539792387544, - "recall@5": 0.21453287197231835, - "recall@10": 0.28908461780434097, - "f1@1": 0.08996539792387544, - "f1@5": 0.07151095732410613, - "f1@10": 0.05256083960078927, - "map@1": 0.08996539792387544, - "map@5": 0.13384187899758834, - "map@10": 0.14376925458240336, - "mrr@1": 0.08996539792387544, - "mrr@5": 0.13384187899758834, - "mrr@10": 0.14376925458240336, - "num_pred": 3179, - "num_data": 3179 + "ViDoRe_syntheticDocQA_artificial_intelligence": { + "hit@1": 0.55, + "hit@5": 0.78, + "hit@10": 0.82, + "ndcg_linear@1": 0.55, + "ndcg_linear@5": 0.6765467489235274, + "ndcg_linear@10": 0.6892234506542988, + "ndcg_exponential@1": 0.55, + "ndcg_exponential@5": 0.6765467489235274, + "ndcg_exponential@10": 0.6892234506542988, + "precision@1": 0.55, + "precision@5": 0.15599999999999997, + "precision@10": 0.08199999999999999, + "recall@1": 0.55, + "recall@5": 0.78, + "recall@10": 0.82, + "f1@1": 0.55, + "f1@5": 0.26000000000000006, + "f1@10": 0.1490909090909091, + "map@1": 0.55, + "map@5": 0.6418333333333334, + "map@10": 0.6469285714285715, + "mrr@1": 0.55, + "mrr@5": 0.6418333333333334, + "mrr@10": 0.6469285714285715, + "num_pred": 100, + "num_data": 100 + }, + "ViDoRe_syntheticDocQA_energy": { + "hit@1": 0.49, + "hit@5": 0.7, + "hit@10": 0.85, + "ndcg_linear@1": 0.49, + "ndcg_linear@5": 0.604188565752427, + "ndcg_linear@10": 0.654079614057483, + "ndcg_exponential@1": 0.49, + "ndcg_exponential@5": 0.604188565752427, + "ndcg_exponential@10": 0.654079614057483, + "precision@1": 0.49, + "precision@5": 0.13999999999999996, + "precision@10": 0.08499999999999996, + "recall@1": 0.49, + "recall@5": 0.7, + "recall@10": 0.85, + "f1@1": 0.49, + "f1@5": 0.23333333333333336, + "f1@10": 0.15454545454545457, + "map@1": 0.49, + "map@5": 0.572, + "map@10": 0.5934246031746032, + "mrr@1": 0.49, + "mrr@5": 0.572, + "mrr@10": 0.5934246031746032, + "num_pred": 100, + "num_data": 100 }, - "QVHighlight": { - "hit@1": 0.577100646352724, - "hit@5": 0.8947368421052632, - "hit@10": 1.0, - "ndcg_linear@1": 0.577100646352724, - "ndcg_linear@5": 0.7461876317485063, - "ndcg_linear@10": 0.7806995799400307, - "ndcg_exponential@1": 0.577100646352724, - "ndcg_exponential@5": 0.7461876317485063, - "ndcg_exponential@10": 0.7806995799400307, - "precision@1": 0.577100646352724, - "precision@5": 0.17894736842105263, - "precision@10": 0.09999999999999998, - "recall@1": 0.577100646352724, - "recall@5": 0.8947368421052632, - "recall@10": 1.0, - "f1@1": 0.577100646352724, - "f1@5": 0.2982456140350878, - "f1@10": 0.18181818181818185, - "map@1": 0.577100646352724, - "map@5": 0.6967220683287165, - "map@10": 0.7112386228729719, - "mrr@1": 0.577100646352724, - "mrr@5": 0.6967220683287165, - "mrr@10": 0.7112386228729719, - "num_pred": 1083, - "num_data": 1083 + "ViDoRe_syntheticDocQA_government_reports": { + "hit@1": 0.42, + "hit@5": 0.78, + "hit@10": 0.88, + "ndcg_linear@1": 0.42, + "ndcg_linear@5": 0.6181938909073429, + "ndcg_linear@10": 0.6501901901304684, + "ndcg_exponential@1": 0.42, + "ndcg_exponential@5": 0.6181938909073429, + "ndcg_exponential@10": 0.6501901901304684, + "precision@1": 0.42, + "precision@5": 0.15599999999999997, + "precision@10": 0.088, + "recall@1": 0.42, + "recall@5": 0.78, + "recall@10": 0.88, + "f1@1": 0.42, + "f1@5": 0.26000000000000006, + "f1@10": 0.16000000000000003, + "map@1": 0.42, + "map@5": 0.5638333333333334, + "map@10": 0.5768253968253969, + "mrr@1": 0.42, + "mrr@5": 0.5638333333333334, + "mrr@10": 0.5768253968253969, + "num_pred": 100, + "num_data": 100 }, - "Charades-STA": { - "hit@1": 0.19807427785419532, - "hit@5": 0.5818431911966988, - "hit@10": 1.0, - "ndcg_linear@1": 0.19807427785419532, - "ndcg_linear@5": 0.3931810758884146, - "ndcg_linear@10": 0.5256230138842131, - "ndcg_exponential@1": 0.19807427785419532, - "ndcg_exponential@5": 0.3931810758884146, - "ndcg_exponential@10": 0.5256230138842131, - "precision@1": 0.19807427785419532, - "precision@5": 0.11636863823933974, - "precision@10": 0.09999999999999998, - "recall@1": 0.19807427785419532, - "recall@5": 0.5818431911966988, - "recall@10": 1.0, - "f1@1": 0.19807427785419532, - "f1@5": 0.19394773039889962, - "f1@10": 0.18181818181818185, - "map@1": 0.19807427785419532, - "map@5": 0.3313388353966071, - "map@10": 0.38442446671470054, - "mrr@1": 0.19807427785419532, - "mrr@5": 0.3313388353966071, - "mrr@10": 0.38442446671470054, - "num_pred": 727, - "num_data": 727 + "ViDoRe_syntheticDocQA_healthcare_industry": { + "hit@1": 0.54, + "hit@5": 0.82, + "hit@10": 0.89, + "ndcg_linear@1": 0.54, + "ndcg_linear@5": 0.698531474110489, + "ndcg_linear@10": 0.7212229678465993, + "ndcg_exponential@1": 0.54, + "ndcg_exponential@5": 0.698531474110489, + "ndcg_exponential@10": 0.7212229678465993, + "precision@1": 0.54, + "precision@5": 0.16399999999999998, + "precision@10": 0.08899999999999997, + "recall@1": 0.54, + "recall@5": 0.82, + "recall@10": 0.89, + "f1@1": 0.54, + "f1@5": 0.2733333333333334, + "f1@10": 0.16181818181818186, + "map@1": 0.54, + "map@5": 0.6573333333333333, + "map@10": 0.6667619047619047, + "mrr@1": 0.54, + "mrr@5": 0.6573333333333333, + "mrr@10": 0.6667619047619047, + "num_pred": 100, + "num_data": 100 }, - "MomentSeeker": { - "hit@1": 0.444, - "hit@5": 0.899, - "hit@10": 1.0, - "ndcg_linear@1": 0.444, - "ndcg_linear@5": 0.6799955352243546, - "ndcg_linear@10": 0.7153235776220722, - "ndcg_exponential@1": 0.444, - "ndcg_exponential@5": 0.6799955352243546, - "ndcg_exponential@10": 0.7153235776220722, - "precision@1": 0.444, - "precision@5": 0.18720000000000003, - "precision@10": 0.1063, - "recall@1": 0.43504166666666666, - "recall@5": 0.892375, - "recall@10": 0.99925, - "f1@1": 0.43757460317460317, - "f1@5": 0.30658053058053064, - "f1@10": 0.1902135642135642, - "map@1": 0.444, - "map@5": 0.6077533333333334, - "map@10": 0.6233544973544974, - "mrr@1": 0.444, - "mrr@5": 0.6105, - "mrr@10": 0.624079365079365, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_esg_reports_human_labeled_v2": { + "hit@1": 0.038461538461538464, + "hit@5": 0.11538461538461539, + "hit@10": 0.23076923076923078, + "ndcg_linear@1": 0.038461538461538464, + "ndcg_linear@5": 0.06817512332990149, + "ndcg_linear@10": 0.09598030466462644, + "ndcg_exponential@1": 0.038461538461538464, + "ndcg_exponential@5": 0.06817512332990149, + "ndcg_exponential@10": 0.09599487668438379, + "precision@1": 0.038461538461538464, + "precision@5": 0.023076923076923075, + "precision@10": 0.026923076923076925, + "recall@1": 0.038461538461538464, + "recall@5": 0.09134615384615384, + "recall@10": 0.16506410256410253, + "f1@1": 0.038461538461538464, + "f1@5": 0.03540903540903541, + "f1@10": 0.04333166833166832, + "map@1": 0.038461538461538464, + "map@5": 0.057211538461538466, + "map@10": 0.06807844932844934, + "mrr@1": 0.038461538461538464, + "mrr@5": 0.06634615384615385, + "mrr@10": 0.08041819291819292, + "num_pred": 52, + "num_data": 52 }, - "K700": { - "hit@1": 0.355, - "hit@5": 0.6, - "hit@10": 0.679, - "ndcg_linear@1": 0.355, - "ndcg_linear@5": 0.4845102488689986, - "ndcg_linear@10": 0.510304213516978, - "ndcg_exponential@1": 0.355, - "ndcg_exponential@5": 0.4845102488689986, - "ndcg_exponential@10": 0.510304213516978, - "precision@1": 0.355, - "precision@5": 0.12, - "precision@10": 0.06789999999999999, - "recall@1": 0.355, - "recall@5": 0.6, - "recall@10": 0.679, - "f1@1": 0.355, - "f1@5": 0.20000000000000007, - "f1@10": 0.12345454545454547, - "map@1": 0.355, - "map@5": 0.44616666666666666, - "map@10": 0.4569583333333333, - "mrr@1": 0.355, - "mrr@5": 0.44616666666666666, - "mrr@10": 0.4569583333333333, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_biomedical_lectures_v2_multilingual": { + "hit@1": 0.05, + "hit@5": 0.1203125, + "hit@10": 0.1875, + "ndcg_linear@1": 0.05, + "ndcg_linear@5": 0.05064502600213018, + "ndcg_linear@10": 0.06329515996638582, + "ndcg_exponential@1": 0.05, + "ndcg_exponential@5": 0.05064502600213018, + "ndcg_exponential@10": 0.06329515996638582, + "precision@1": 0.05, + "precision@5": 0.025312500000000005, + "precision@10": 0.022031250000000002, + "recall@1": 0.027369791666666664, + "recall@5": 0.052070591020499116, + "recall@10": 0.08876272241851287, + "f1@1": 0.031846196338383835, + "f1@5": 0.029338837377899885, + "f1@10": 0.031196784815810553, + "map@1": 0.05, + "map@5": 0.03780381944444444, + "map@10": 0.041590970155423276, + "mrr@1": 0.05, + "mrr@5": 0.07333333333333333, + "mrr@10": 0.08225384424603174, + "num_pred": 640, + "num_data": 640 }, - "SmthSmthV2": { - "hit@1": 0.321, - "hit@5": 0.593, - "hit@10": 0.695, - "ndcg_linear@1": 0.321, - "ndcg_linear@5": 0.464820244083625, - "ndcg_linear@10": 0.49815926994038934, - "ndcg_exponential@1": 0.321, - "ndcg_exponential@5": 0.464820244083625, - "ndcg_exponential@10": 0.49815926994038934, - "precision@1": 0.321, - "precision@5": 0.1186, - "precision@10": 0.0695, - "recall@1": 0.321, - "recall@5": 0.593, - "recall@10": 0.695, - "f1@1": 0.321, - "f1@5": 0.19766666666666674, - "f1@10": 0.1263636363636364, - "map@1": 0.321, - "map@5": 0.4222333333333333, - "map@10": 0.43619880952380946, - "mrr@1": 0.321, - "mrr@5": 0.4222333333333333, - "mrr@10": 0.43619880952380946, - "num_pred": 1000, - "num_data": 1000 + "ViDoRe_economics_reports_v2_multilingual": { + "hit@1": 0.1810344827586207, + "hit@5": 0.4224137931034483, + "hit@10": 0.5646551724137931, + "ndcg_linear@1": 0.1810344827586207, + "ndcg_linear@5": 0.13875113955781276, + "ndcg_linear@10": 0.14758177623224894, + "ndcg_exponential@1": 0.1810344827586207, + "ndcg_exponential@5": 0.1387511395578127, + "ndcg_exponential@10": 0.14758177623224894, + "precision@1": 0.1810344827586207, + "precision@5": 0.11982758620689656, + "precision@10": 0.11551724137931037, + "recall@1": 0.012707666017446384, + "recall@5": 0.06501080124572646, + "recall@10": 0.12018603229901821, + "f1@1": 0.02288545832709266, + "f1@5": 0.06054340208235827, + "f1@10": 0.08247544424998887, + "map@1": 0.1810344827586207, + "map@5": 0.08251436781609195, + "map@10": 0.07246543932068357, + "mrr@1": 0.1810344827586207, + "mrr@5": 0.2608477011494253, + "mrr@10": 0.27921798029556655, + "num_pred": 232, + "num_data": 232 + }, + "ViDoRe_esg_reports_v2_multilingual": { + "hit@1": 0.08333333333333333, + "hit@5": 0.2324561403508772, + "hit@10": 0.31140350877192985, + "ndcg_linear@1": 0.08333333333333333, + "ndcg_linear@5": 0.11890188811957679, + "ndcg_linear@10": 0.1391360634250716, + "ndcg_exponential@1": 0.08333333333333333, + "ndcg_exponential@5": 0.11890188811957679, + "ndcg_exponential@10": 0.1391360634250716, + "precision@1": 0.08333333333333333, + "precision@5": 0.05964912280701755, + "precision@10": 0.04649122807017544, + "recall@1": 0.05936716791979949, + "recall@5": 0.1361737677527151, + "recall@10": 0.18859649122807018, + "f1@1": 0.06306354967879468, + "f1@5": 0.07183557899502171, + "f1@10": 0.06509698173479336, + "map@1": 0.08333333333333333, + "map@5": 0.09392543859649122, + "map@10": 0.10152116402116403, + "mrr@1": 0.08333333333333333, + "mrr@5": 0.1355263157894737, + "mrr@10": 0.14601434140907826, + "num_pred": 228, + "num_data": 228 }, - "HMDB51": { - "hit@1": 0.422, - "hit@5": 0.684, - "hit@10": 0.83, - "ndcg_linear@1": 0.422, - "ndcg_linear@5": 0.5578823382949479, - "ndcg_linear@10": 0.605271745232144, - "ndcg_exponential@1": 0.422, - "ndcg_exponential@5": 0.5578823382949479, - "ndcg_exponential@10": 0.605271745232144, - "precision@1": 0.422, - "precision@5": 0.1368, - "precision@10": 0.083, - "recall@1": 0.422, - "recall@5": 0.684, - "recall@10": 0.83, - "f1@1": 0.422, - "f1@5": 0.22800000000000006, - "f1@10": 0.15090909090909094, - "map@1": 0.422, - "map@5": 0.5162333333333333, - "map@10": 0.5358904761904761, - "mrr@1": 0.422, - "mrr@5": 0.5162333333333333, - "mrr@10": 0.5358904761904761, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ArxivQA": { + "hit@1": 0.43504901960784315, + "hit@5": 0.6029411764705882, + "hit@10": 0.6727941176470589, + "ndcg_linear@1": 0.43504901960784315, + "ndcg_linear@5": 0.5260731896593214, + "ndcg_linear@10": 0.5488301957135047, + "ndcg_exponential@1": 0.43504901960784315, + "ndcg_exponential@5": 0.5260731896593214, + "ndcg_exponential@10": 0.5488301957135047, + "precision@1": 0.43504901960784315, + "precision@5": 0.12058823529411763, + "precision@10": 0.06727941176470587, + "recall@1": 0.43504901960784315, + "recall@5": 0.6029411764705882, + "recall@10": 0.6727941176470589, + "f1@1": 0.43504901960784315, + "f1@5": 0.2009803921568628, + "f1@10": 0.12232620320855617, + "map@1": 0.43504901960784315, + "map@5": 0.5003472222222222, + "map@10": 0.5098355314347962, + "mrr@1": 0.43504901960784315, + "mrr@5": 0.5003472222222222, + "mrr@10": 0.5098355314347962, + "num_pred": 816, + "num_data": 816 }, - "UCF101": { - "hit@1": 0.618, - "hit@5": 0.889, - "hit@10": 0.94, - "ndcg_linear@1": 0.618, - "ndcg_linear@5": 0.772430845929742, - "ndcg_linear@10": 0.7893494678275759, - "ndcg_exponential@1": 0.618, - "ndcg_exponential@5": 0.772430845929742, - "ndcg_exponential@10": 0.7893494678275759, - "precision@1": 0.618, - "precision@5": 0.1778, - "precision@10": 0.09400000000000001, - "recall@1": 0.618, - "recall@5": 0.889, - "recall@10": 0.94, - "f1@1": 0.618, - "f1@5": 0.2963333333333334, - "f1@10": 0.1709090909090909, - "map@1": 0.618, - "map@5": 0.7326833333333334, - "map@10": 0.7399230158730159, - "mrr@1": 0.618, - "mrr@5": 0.7326833333333334, - "mrr@10": 0.7399230158730159, - "num_pred": 1000, - "num_data": 1000 + "VisRAG_ChartQA": { + "hit@1": 0.6031746031746031, + "hit@5": 0.7936507936507936, + "hit@10": 0.8412698412698413, + "ndcg_linear@1": 0.6031746031746031, + "ndcg_linear@5": 0.7023156150418144, + "ndcg_linear@10": 0.7176213783260175, + "ndcg_exponential@1": 0.6031746031746031, + "ndcg_exponential@5": 0.7023156150418144, + "ndcg_exponential@10": 0.7176213783260175, + "precision@1": 0.6031746031746031, + "precision@5": 0.15873015873015867, + "precision@10": 0.08412698412698409, + "recall@1": 0.6031746031746031, + "recall@5": 0.7936507936507936, + "recall@10": 0.8412698412698413, + "f1@1": 0.6031746031746031, + "f1@5": 0.2645502645502646, + "f1@10": 0.15295815295815296, + "map@1": 0.6031746031746031, + "map@5": 0.6722222222222223, + "map@10": 0.6784580498866214, + "mrr@1": 0.6031746031746031, + "mrr@5": 0.6722222222222223, + "mrr@10": 0.6784580498866214, + "num_pred": 63, + "num_data": 63 }, - "Breakfast": { - "hit@1": 0.23787528868360278, - "hit@5": 0.651270207852194, - "hit@10": 1.0, - "ndcg_linear@1": 0.23787528868360278, - "ndcg_linear@5": 0.4481888236256693, - "ndcg_linear@10": 0.5633176174385133, - "ndcg_exponential@1": 0.23787528868360278, - "ndcg_exponential@5": 0.4481888236256693, - "ndcg_exponential@10": 0.5633176174385133, - "precision@1": 0.23787528868360278, - "precision@5": 0.13025404157043877, - "precision@10": 0.1, - "recall@1": 0.23787528868360278, - "recall@5": 0.651270207852194, - "recall@10": 1.0, - "f1@1": 0.23787528868360278, - "f1@5": 0.21709006928406474, - "f1@10": 0.1818181818181818, - "map@1": 0.23787528868360278, - "map@5": 0.3814087759815243, - "map@10": 0.4303328567762748, - "mrr@1": 0.23787528868360278, - "mrr@5": 0.3814087759815243, - "mrr@10": 0.4303328567762748, - "num_pred": 433, - "num_data": 433 + "VisRAG_MP-DocVQA": { + "hit@1": 0.4077834179357022, + "hit@5": 0.6395939086294417, + "hit@10": 0.7123519458544839, + "ndcg_linear@1": 0.4077834179357022, + "ndcg_linear@5": 0.5280813677279776, + "ndcg_linear@10": 0.5522909877969642, + "ndcg_exponential@1": 0.4077834179357022, + "ndcg_exponential@5": 0.5280813677279776, + "ndcg_exponential@10": 0.5522909877969642, + "precision@1": 0.4077834179357022, + "precision@5": 0.1279187817258883, + "precision@10": 0.07123519458544839, + "recall@1": 0.4077834179357022, + "recall@5": 0.6395939086294417, + "recall@10": 0.7123519458544839, + "f1@1": 0.4077834179357022, + "f1@5": 0.2131979695431472, + "f1@10": 0.1295185356099062, + "map@1": 0.4077834179357022, + "map@5": 0.4912859560067682, + "map@10": 0.5016725754035398, + "mrr@1": 0.4077834179357022, + "mrr@5": 0.4912859560067682, + "mrr@10": 0.5016725754035398, + "num_pred": 591, + "num_data": 591 }, - "MVBench": { - "hit@1": 0.28475, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.28475, - "ndcg_linear@5": 0.6742128560665944, - "ndcg_linear@10": 0.6742128560665944, - "ndcg_exponential@1": 0.28475, - "ndcg_exponential@5": 0.6742128560665944, - "ndcg_exponential@10": 0.6742128560665944, - "precision@1": 0.28475, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.28475, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.28475, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.28475, - "map@5": 0.5643916666666666, - "map@10": 0.5643916666666666, - "mrr@1": 0.28475, - "mrr@5": 0.5643916666666666, - "mrr@10": 0.5643916666666666, - "num_pred": 4000, - "num_data": 4000 + "VisRAG_SlideVQA": { + "hit@1": 0.6258992805755396, + "hit@5": 0.85431654676259, + "hit@10": 0.8974820143884892, + "ndcg_linear@1": 0.6258992805755396, + "ndcg_linear@5": 0.727722018029152, + "ndcg_linear@10": 0.7442517359673964, + "ndcg_exponential@1": 0.6258992805755396, + "ndcg_exponential@5": 0.727722018029152, + "ndcg_exponential@10": 0.7442517359673964, + "precision@1": 0.6258992805755396, + "precision@5": 0.2079136690647482, + "precision@10": 0.10953237410071942, + "recall@1": 0.5314748201438849, + "recall@5": 0.822841726618705, + "recall@10": 0.8714028776978417, + "f1@1": 0.5629496402877698, + "f1@5": 0.3258821514217198, + "f1@10": 0.19236429038587313, + "map@1": 0.6258992805755396, + "map@5": 0.6828387290167865, + "map@10": 0.6904986724905788, + "mrr@1": 0.6258992805755396, + "mrr@5": 0.7175659472422062, + "mrr@10": 0.7235775665182139, + "num_pred": 556, + "num_data": 556 + }, + "VisRAG_InfoVQA": { + "hit@1": 0.6002785515320335, + "hit@5": 0.8245125348189415, + "hit@10": 0.8899721448467967, + "ndcg_linear@1": 0.6002785515320335, + "ndcg_linear@5": 0.7196103081823707, + "ndcg_linear@10": 0.7410456616380353, + "ndcg_exponential@1": 0.6002785515320335, + "ndcg_exponential@5": 0.7196103081823707, + "ndcg_exponential@10": 0.7410456616380353, + "precision@1": 0.6002785515320335, + "precision@5": 0.16490250696378828, + "precision@10": 0.08899721448467965, + "recall@1": 0.6002785515320335, + "recall@5": 0.8245125348189415, + "recall@10": 0.8899721448467967, + "f1@1": 0.6002785515320335, + "f1@5": 0.27483751160631387, + "f1@10": 0.16181311724487216, + "map@1": 0.6002785515320335, + "map@5": 0.6846564531104921, + "map@10": 0.6936557677852943, + "mrr@1": 0.6002785515320335, + "mrr@5": 0.6846564531104921, + "mrr@10": 0.6936557677852943, + "num_pred": 718, + "num_data": 718 }, - "Video-MME": { - "hit@1": 0.27814814814814814, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.27814814814814814, - "ndcg_linear@5": 0.660078255498731, - "ndcg_linear@10": 0.660078255498731, - "ndcg_exponential@1": 0.27814814814814814, - "ndcg_exponential@5": 0.660078255498731, - "ndcg_exponential@10": 0.660078255498731, - "precision@1": 0.27814814814814814, - "precision@5": 0.19999999999999996, - "precision@10": 0.09999999999999998, - "recall@1": 0.27814814814814814, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.27814814814814814, - "f1@5": 0.3333333333333334, - "f1@10": 0.18181818181818188, - "map@1": 0.27814814814814814, - "map@5": 0.5465432098765431, - "map@10": 0.5465432098765431, - "mrr@1": 0.27814814814814814, - "mrr@5": 0.5465432098765431, - "mrr@10": 0.5465432098765431, - "num_pred": 2700, - "num_data": 2700 + "VisRAG_PlotQA": { + "hit@1": 0.22363847045191193, + "hit@5": 0.4484356894553882, + "hit@10": 0.5654692931633836, + "ndcg_linear@1": 0.22363847045191193, + "ndcg_linear@5": 0.3434876959499088, + "ndcg_linear@10": 0.3813457018404914, + "ndcg_exponential@1": 0.22363847045191193, + "ndcg_exponential@5": 0.3434876959499088, + "ndcg_exponential@10": 0.3813457018404914, + "precision@1": 0.22363847045191193, + "precision@5": 0.08968713789107764, + "precision@10": 0.05654692931633834, + "recall@1": 0.22363847045191193, + "recall@5": 0.4484356894553882, + "recall@10": 0.5654692931633836, + "f1@1": 0.22363847045191193, + "f1@5": 0.14947856315179606, + "f1@10": 0.10281259875697883, + "map@1": 0.22363847045191193, + "map@5": 0.3085940517574354, + "map@10": 0.32422382239879344, + "mrr@1": 0.22363847045191193, + "mrr@5": 0.3085940517574354, + "mrr@10": 0.32422382239879344, + "num_pred": 863, + "num_data": 863 }, - "NExTQA": { - "hit@1": 0.20294255021018215, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.20294255021018215, - "ndcg_linear@5": 0.604249894952272, - "ndcg_linear@10": 0.604249894952272, - "ndcg_exponential@1": 0.20294255021018215, - "ndcg_exponential@5": 0.604249894952272, - "ndcg_exponential@10": 0.604249894952272, - "precision@1": 0.20294255021018215, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.20294255021018215, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.20294255021018215, - "f1@5": 0.3333333333333333, - "f1@10": 0.1818181818181818, - "map@1": 0.20294255021018215, - "map@5": 0.4747664642690332, - "map@10": 0.4747664642690332, - "mrr@1": 0.20294255021018215, - "mrr@5": 0.4746029892573564, - "mrr@10": 0.4746029892573564, - "num_pred": 8564, - "num_data": 8564 + "ViDoSeek-page": { + "hit@1": 0.0446584938704028, + "hit@5": 0.3905429071803853, + "hit@10": 0.5971978984238179, + "ndcg_linear@1": 0.0446584938704028, + "ndcg_linear@5": 0.22283683461024645, + "ndcg_linear@10": 0.2895264993048246, + "ndcg_exponential@1": 0.0446584938704028, + "ndcg_exponential@5": 0.22283683461024653, + "ndcg_exponential@10": 0.28952649930482466, + "precision@1": 0.0446584938704028, + "precision@5": 0.07810858143607707, + "precision@10": 0.05971978984238179, + "recall@1": 0.0446584938704028, + "recall@5": 0.3905429071803853, + "recall@10": 0.5971978984238179, + "f1@1": 0.0446584938704028, + "f1@5": 0.13018096906012844, + "f1@10": 0.10858143607705782, + "map@1": 0.0446584938704028, + "map@5": 0.1676444833625219, + "map@10": 0.19508763517082256, + "mrr@1": 0.0446584938704028, + "mrr@5": 0.1676444833625219, + "mrr@10": 0.19508763517082256, + "num_pred": 1142, + "num_data": 1142 }, - "EgoSchema": { - "hit@1": 0.218, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.218, - "ndcg_linear@5": 0.5992262804502428, - "ndcg_linear@10": 0.5992262804502428, - "ndcg_exponential@1": 0.218, - "ndcg_exponential@5": 0.5992262804502428, - "ndcg_exponential@10": 0.5992262804502428, - "precision@1": 0.218, - "precision@5": 0.2, - "precision@10": 0.1, - "recall@1": 0.218, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.218, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.218, - "map@5": 0.46926666666666667, - "map@10": 0.46926666666666667, - "mrr@1": 0.218, - "mrr@5": 0.46926666666666667, - "mrr@10": 0.46926666666666667, - "num_pred": 500, - "num_data": 500 + "ViDoSeek-doc": { + "hit@1": 0.968476357267951, + "hit@5": 0.9912434325744308, + "hit@10": 0.9947460595446584, + "ndcg_linear@1": 0.6625802685347344, + "ndcg_linear@5": 0.7780718841793594, + "ndcg_linear@10": 0.7502728417738824, + "ndcg_exponential@1": 0.44408306229672245, + "ndcg_exponential@5": 0.6673711755462026, + "ndcg_exponential@10": 0.6819104114803104, + "precision@1": 0.968476357267951, + "precision@5": 0.8481611208406304, + "precision@10": 0.750875656742557, + "recall@1": 0.05711879676554371, + "recall@5": 0.24781081127789126, + "recall@10": 0.4314558336731064, + "f1@1": 0.10713328875583096, + "f1@5": 0.375858198266283, + "f1@10": 0.5355661581210087, + "map@1": 0.968476357267951, + "map@5": 0.831468184471687, + "map@10": 0.7255596248277478, + "mrr@1": 0.968476357267951, + "mrr@5": 0.9776999416228838, + "mrr@10": 0.9781680010007505, + "num_pred": 1142, + "num_data": 1142 }, - "ActivityNetQA": { - "hit@1": 0.514, - "hit@5": 1.0, - "hit@10": 1.0, - "ndcg_linear@1": 0.514, - "ndcg_linear@5": 0.8206318602357284, - "ndcg_linear@10": 0.8206318602357284, - "ndcg_exponential@1": 0.514, - "ndcg_exponential@5": 0.8206318602357284, - "ndcg_exponential@10": 0.8206318602357284, - "precision@1": 0.514, - "precision@5": 0.20000000000000004, - "precision@10": 0.10000000000000002, - "recall@1": 0.514, - "recall@5": 1.0, - "recall@10": 1.0, - "f1@1": 0.514, - "f1@5": 0.3333333333333333, - "f1@10": 0.18181818181818182, - "map@1": 0.514, - "map@5": 0.757, - "map@10": 0.757, - "mrr@1": 0.514, - "mrr@5": 0.757, - "mrr@10": 0.757, - "num_pred": 1000, - "num_data": 1000 + "MMLongBench-page": { + "hit@1": 0.06801909307875895, + "hit@5": 0.2386634844868735, + "hit@10": 0.3305489260143198, + "ndcg_linear@1": 0.06801909307875895, + "ndcg_linear@5": 0.11822134474246979, + "ndcg_linear@10": 0.15022040843373538, + "ndcg_exponential@1": 0.06801909307875895, + "ndcg_exponential@5": 0.11822134474246979, + "ndcg_exponential@10": 0.15022040843373538, + "precision@1": 0.06801909307875895, + "precision@5": 0.056324582338902154, + "precision@10": 0.04379474940334129, + "recall@1": 0.038664905102852594, + "recall@5": 0.16723964844489903, + "recall@10": 0.25455198143145635, + "f1@1": 0.04688230756249849, + "f1@5": 0.07989331008424089, + "f1@10": 0.07145515842987549, + "map@1": 0.06801909307875895, + "map@5": 0.0867468841156192, + "map@10": 0.10167219917667411, + "mrr@1": 0.06801909307875895, + "mrr@5": 0.12738663484486873, + "mrr@10": 0.13961955904080012, + "num_pred": 838, + "num_data": 838 + }, + "MMLongBench-doc": { + "hit@1": 0.5668257756563246, + "hit@5": 0.6885441527446301, + "hit@10": 0.747016706443914, + "ndcg_linear@1": 0.3981702466189339, + "ndcg_linear@5": 0.40462422635020384, + "ndcg_linear@10": 0.38948040985289195, + "ndcg_exponential@1": 0.27770201159222635, + "ndcg_exponential@5": 0.33626701840937107, + "ndcg_exponential@10": 0.34529122789195027, + "precision@1": 0.5668257756563246, + "precision@5": 0.4470167064439141, + "precision@10": 0.3953460620525059, + "recall@1": 0.017686024380438876, + "recall@5": 0.06659662623490364, + "recall@10": 0.11644775779833906, + "f1@1": 0.03396785786854894, + "f1@5": 0.11131694335938004, + "f1@10": 0.16871497861256565, + "map@1": 0.5668257756563246, + "map@5": 0.41111774065234685, + "map@10": 0.34533373426273184, + "mrr@1": 0.5668257756563246, + "mrr@5": 0.6124105011933175, + "mrr@10": 0.6201973709133614, + "num_pred": 838, + "num_data": 838 } } }