Shiyu Zhao
commited on
Commit
·
928143b
1
Parent(s):
031d82b
Update space
Browse files
app.py
CHANGED
@@ -2,8 +2,40 @@ import gradio as gr
|
|
2 |
import pandas as pd
|
3 |
import numpy as np
|
4 |
|
5 |
-
# Sample data based on your table
|
6 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
7 |
'Method': ['BM25', 'DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)', 'ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b', 'multi-ada-002', 'ColBERTv2', 'Claude3 Reranker', 'GPT4 Reranker'],
|
8 |
'STARK-AMAZON_Hit@1': [27.16, 16.05, 25.93, 22.22, 39.50, 35.80, 29.63, 40.74, 46.91, 33.33, 53.09, 50.62],
|
9 |
'STARK-AMAZON_Hit@5': [51.85, 39.51, 54.32, 49.38, 64.19, 62.96, 46.91, 71.60, 72.84, 55.56, 74.07, 75.31],
|
@@ -19,7 +51,9 @@ data = {
|
|
19 |
'STARK-PRIME_MRR': [30.37, 7.05, 10.07, 9.39, 26.35, 24.33, 15.24, 34.28, 32.98, 19.67, 36.32, 34.82]
|
20 |
}
|
21 |
|
22 |
-
|
|
|
|
|
23 |
|
24 |
def format_dataframe(df, dataset):
|
25 |
# Filter the dataframe for the selected dataset
|
@@ -34,6 +68,18 @@ def format_dataframe(df, dataset):
|
|
34 |
|
35 |
return filtered_df
|
36 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
37 |
css = """
|
38 |
table > thead {
|
39 |
white-space: normal
|
@@ -52,25 +98,77 @@ with gr.Blocks(css=css) as demo:
|
|
52 |
gr.Markdown("# Semi-structured Retrieval Benchmark (STaRK) Leaderboard")
|
53 |
gr.Markdown("Refer to the [STaRK paper](https://arxiv.org/pdf/2404.13207) for details on metrics, tasks and models.")
|
54 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
55 |
with gr.Tabs() as outer_tabs:
|
56 |
-
with gr.TabItem("Synthesized"):
|
57 |
-
gr.
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
|
|
|
|
65 |
|
66 |
with gr.TabItem("Human-Generated"):
|
67 |
-
gr.Markdown("## Human-Generated Data Results")
|
68 |
with gr.Tabs() as inner_tabs_human:
|
69 |
-
|
70 |
-
gr.
|
71 |
-
|
72 |
-
|
73 |
-
|
74 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
75 |
|
76 |
demo.launch()
|
|
|
2 |
import pandas as pd
|
3 |
import numpy as np
|
4 |
|
5 |
+
# Sample data based on your table (you'll need to update this with the full dataset)
|
6 |
+
data_synthesized_full = {
|
7 |
+
'Method': ['BM25', 'DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)', 'ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b', 'multi-ada-002', 'ColBERTv2'],
|
8 |
+
'STARK-AMAZON_Hit@1': [44.94, 15.29, 30.96, 26.56, 39.16, 40.93, 21.74, 42.08, 40.07, 46.10],
|
9 |
+
'STARK-AMAZON_Hit@5': [67.42, 47.93, 51.06, 50.01, 62.73, 64.37, 41.65, 66.87, 64.98, 66.02],
|
10 |
+
'STARK-AMAZON_R@20': [53.77, 44.49, 41.95, 52.05, 53.29, 54.28, 33.22, 56.52, 55.12, 53.44],
|
11 |
+
'STARK-AMAZON_MRR': [55.30, 30.20, 40.66, 37.75, 50.35, 51.60, 31.47, 53.46, 51.55, 55.51],
|
12 |
+
'STARK-MAG_Hit@1': [25.85, 10.51, 21.96, 12.88, 29.08, 30.06, 18.01, 37.90, 25.92, 31.18],
|
13 |
+
'STARK-MAG_Hit@5': [45.25, 35.23, 36.50, 39.01, 49.61, 50.58, 34.85, 56.74, 50.43, 46.42],
|
14 |
+
'STARK-MAG_R@20': [45.69, 42.11, 35.32, 46.97, 48.36, 50.49, 35.46, 46.40, 50.80, 43.94],
|
15 |
+
'STARK-MAG_MRR': [34.91, 21.34, 29.14, 29.12, 38.62, 39.66, 26.10, 47.25, 36.94, 38.39],
|
16 |
+
'STARK-PRIME_Hit@1': [12.75, 4.46, 6.53, 8.85, 12.63, 10.85, 10.10, 15.57, 15.10, 11.75],
|
17 |
+
'STARK-PRIME_Hit@5': [27.92, 21.85, 15.67, 21.35, 31.49, 30.23, 22.49, 33.42, 33.56, 23.85],
|
18 |
+
'STARK-PRIME_R@20': [31.25, 30.13, 16.52, 29.63, 36.00, 37.83, 26.34, 39.09, 38.05, 25.04],
|
19 |
+
'STARK-PRIME_MRR': [19.84, 12.38, 11.05, 14.73, 21.41, 19.99, 16.12, 24.11, 23.49, 17.39]
|
20 |
+
}
|
21 |
+
|
22 |
+
data_synthesized_10 = {
|
23 |
+
'Method': ['BM25', 'DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)', 'ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b', 'multi-ada-002', 'ColBERTv2', 'Claude3 Reranker', 'GPT4 Reranker'],
|
24 |
+
'STARK-AMAZON_Hit@1': [42.68, 16.46, 30.09, 25.00, 39.02, 43.29, 18.90, 43.29, 40.85, 44.31, 45.49, 44.79],
|
25 |
+
'STARK-AMAZON_Hit@5': [67.07, 50.00, 49.27, 48.17, 64.02, 67.68, 37.80, 71.34, 62.80, 65.24, 71.13, 71.17],
|
26 |
+
'STARK-AMAZON_R@20': [54.48, 42.15, 41.91, 51.65, 49.30, 56.04, 34.73, 56.14, 52.47, 51.00, 53.77, 55.35],
|
27 |
+
'STARK-AMAZON_MRR': [54.02, 30.20, 39.30, 36.87, 50.32, 54.20, 28.76, 55.07, 51.54, 55.07, 55.91, 55.69],
|
28 |
+
'STARK-MAG_Hit@1': [27.81, 11.65, 22.89, 12.03, 28.20, 34.59, 19.17, 38.35, 25.56, 31.58, 36.54, 40.90],
|
29 |
+
'STARK-MAG_Hit@5': [45.48, 36.84, 37.26, 37.97, 52.63, 50.75, 33.46, 58.64, 50.37, 47.36, 53.17, 58.18],
|
30 |
+
'STARK-MAG_R@20': [44.59, 42.30, 44.16, 47.98, 49.25, 50.75, 29.85, 46.38, 53.03, 45.72, 48.36, 48.60],
|
31 |
+
'STARK-MAG_MRR': [35.97, 21.82, 30.00, 28.70, 38.55, 42.90, 26.06, 48.25, 36.82, 38.98, 44.15, 49.00],
|
32 |
+
'STARK-PRIME_Hit@1': [13.93, 5.00, 6.78, 7.14, 15.36, 12.14, 9.29, 16.79, 15.36, 15.00, 17.79, 18.28],
|
33 |
+
'STARK-PRIME_Hit@5': [31.07, 23.57, 16.15, 17.14, 31.07, 31.42, 20.7, 34.29, 32.86, 26.07, 36.90, 37.28],
|
34 |
+
'STARK-PRIME_R@20': [32.84, 30.50, 17.07, 32.95, 37.88, 37.34, 25.54, 41.11, 40.99, 27.78, 35.57, 34.05],
|
35 |
+
'STARK-PRIME_MRR': [21.68, 13.50, 11.42, 16.27, 23.50, 21.23, 15.00, 24.99, 23.70, 19.98, 26.27, 26.55]
|
36 |
+
}
|
37 |
+
|
38 |
+
data_human_generated = {
|
39 |
'Method': ['BM25', 'DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)', 'ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b', 'multi-ada-002', 'ColBERTv2', 'Claude3 Reranker', 'GPT4 Reranker'],
|
40 |
'STARK-AMAZON_Hit@1': [27.16, 16.05, 25.93, 22.22, 39.50, 35.80, 29.63, 40.74, 46.91, 33.33, 53.09, 50.62],
|
41 |
'STARK-AMAZON_Hit@5': [51.85, 39.51, 54.32, 49.38, 64.19, 62.96, 46.91, 71.60, 72.84, 55.56, 74.07, 75.31],
|
|
|
51 |
'STARK-PRIME_MRR': [30.37, 7.05, 10.07, 9.39, 26.35, 24.33, 15.24, 34.28, 32.98, 19.67, 36.32, 34.82]
|
52 |
}
|
53 |
|
54 |
+
df_synthesized_full = pd.DataFrame(data_synthesized_full)
|
55 |
+
df_synthesized_10 = pd.DataFrame(data_synthesized_10)
|
56 |
+
df_human_generated = pd.DataFrame(data_human_generated)
|
57 |
|
58 |
def format_dataframe(df, dataset):
|
59 |
# Filter the dataframe for the selected dataset
|
|
|
68 |
|
69 |
return filtered_df
|
70 |
|
71 |
+
model_types = {
|
72 |
+
'Sparse Retriever': ['BM25'],
|
73 |
+
'Small Dense Retrievers': ['DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)'],
|
74 |
+
'LLM-based Dense Retrievers': ['ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b'],
|
75 |
+
'Multivector Retrievers': ['multi-ada-002', 'ColBERTv2'],
|
76 |
+
'LLM Rerankers': ['Claude3 Reranker', 'GPT4 Reranker']
|
77 |
+
}
|
78 |
+
|
79 |
+
def filter_by_model_type(df, selected_types):
|
80 |
+
selected_models = [model for type in selected_types for model in model_types[type]]
|
81 |
+
return df[df['Method'].isin(selected_models)]
|
82 |
+
|
83 |
css = """
|
84 |
table > thead {
|
85 |
white-space: normal
|
|
|
98 |
gr.Markdown("# Semi-structured Retrieval Benchmark (STaRK) Leaderboard")
|
99 |
gr.Markdown("Refer to the [STaRK paper](https://arxiv.org/pdf/2404.13207) for details on metrics, tasks and models.")
|
100 |
|
101 |
+
with gr.Row():
|
102 |
+
model_type_filter = gr.CheckboxGroup(
|
103 |
+
choices=list(model_types.keys()),
|
104 |
+
value=list(model_types.keys()),
|
105 |
+
label="Model types",
|
106 |
+
interactive=True
|
107 |
+
)
|
108 |
+
|
109 |
with gr.Tabs() as outer_tabs:
|
110 |
+
with gr.TabItem("Synthesized (full)"):
|
111 |
+
with gr.Tabs() as inner_tabs_synthesized_full:
|
112 |
+
for dataset in ['AMAZON', 'MAG', 'PRIME']:
|
113 |
+
with gr.TabItem(dataset):
|
114 |
+
gr.DataFrame(format_dataframe(df_synthesized_full, f"STARK-{dataset}"))
|
115 |
+
|
116 |
+
with gr.TabItem("Synthesized (10%)"):
|
117 |
+
with gr.Tabs() as inner_tabs_synthesized_10:
|
118 |
+
for dataset in ['AMAZON', 'MAG', 'PRIME']:
|
119 |
+
with gr.TabItem(dataset):
|
120 |
+
gr.DataFrame(format_dataframe(df_synthesized_10, f"STARK-{dataset}"))
|
121 |
|
122 |
with gr.TabItem("Human-Generated"):
|
|
|
123 |
with gr.Tabs() as inner_tabs_human:
|
124 |
+
for dataset in ['AMAZON', 'MAG', 'PRIME']:
|
125 |
+
with gr.TabItem(dataset):
|
126 |
+
gr.DataFrame(format_dataframe(df_human_generated, f"STARK-{dataset}"))
|
127 |
+
|
128 |
+
def update_tables(selected_types):
|
129 |
+
filtered_df_full = filter_by_model_type(df_synthesized_full, selected_types)
|
130 |
+
filtered_df_10 = filter_by_model_type(df_synthesized_10, selected_types)
|
131 |
+
filtered_df_human = filter_by_model_type(df_human_generated, selected_types)
|
132 |
+
|
133 |
+
outputs = []
|
134 |
+
for df in [filtered_df_full, filtered_df_10, filtered_df_human]:
|
135 |
+
for dataset in ['AMAZON', 'MAG', 'PRIME']:
|
136 |
+
outputs.append(format_dataframe(df, f"STARK-{dataset}"))
|
137 |
+
|
138 |
+
return outputs
|
139 |
+
|
140 |
+
# Create DataFrame components for each tab and dataset
|
141 |
+
dataframes = []
|
142 |
+
for _ in range(3): # 3 main tabs
|
143 |
+
for _ in range(3): # 3 datasets per tab
|
144 |
+
df_component = gr.DataFrame(visible=False)
|
145 |
+
dataframes.append(df_component)
|
146 |
+
|
147 |
+
# Update visibility and content of DataFrames based on selected tab
|
148 |
+
def update_visibility(tab):
|
149 |
+
visibilities = [False] * 9
|
150 |
+
start_index = tab * 3
|
151 |
+
visibilities[start_index:start_index+3] = [True, True, True]
|
152 |
+
return visibilities
|
153 |
+
|
154 |
+
outer_tabs.select(
|
155 |
+
update_visibility,
|
156 |
+
None,
|
157 |
+
dataframes
|
158 |
+
)
|
159 |
+
|
160 |
+
# Connect model type filter to update_tables function
|
161 |
+
model_type_filter.change(
|
162 |
+
update_tables,
|
163 |
+
inputs=[model_type_filter],
|
164 |
+
outputs=dataframes
|
165 |
+
)
|
166 |
+
|
167 |
+
# Initial load of tables
|
168 |
+
demo.load(
|
169 |
+
update_tables,
|
170 |
+
inputs=[model_type_filter],
|
171 |
+
outputs=dataframes
|
172 |
+
)
|
173 |
|
174 |
demo.launch()
|