Shiyu Zhao commited on
Commit
928143b
·
1 Parent(s): 031d82b

Update space

Browse files
Files changed (1) hide show
  1. app.py +117 -19
app.py CHANGED
@@ -2,8 +2,40 @@ import gradio as gr
2
  import pandas as pd
3
  import numpy as np
4
 
5
- # Sample data based on your table
6
- data = {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  'Method': ['BM25', 'DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)', 'ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b', 'multi-ada-002', 'ColBERTv2', 'Claude3 Reranker', 'GPT4 Reranker'],
8
  'STARK-AMAZON_Hit@1': [27.16, 16.05, 25.93, 22.22, 39.50, 35.80, 29.63, 40.74, 46.91, 33.33, 53.09, 50.62],
9
  'STARK-AMAZON_Hit@5': [51.85, 39.51, 54.32, 49.38, 64.19, 62.96, 46.91, 71.60, 72.84, 55.56, 74.07, 75.31],
@@ -19,7 +51,9 @@ data = {
19
  'STARK-PRIME_MRR': [30.37, 7.05, 10.07, 9.39, 26.35, 24.33, 15.24, 34.28, 32.98, 19.67, 36.32, 34.82]
20
  }
21
 
22
- df = pd.DataFrame(data)
 
 
23
 
24
  def format_dataframe(df, dataset):
25
  # Filter the dataframe for the selected dataset
@@ -34,6 +68,18 @@ def format_dataframe(df, dataset):
34
 
35
  return filtered_df
36
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  css = """
38
  table > thead {
39
  white-space: normal
@@ -52,25 +98,77 @@ with gr.Blocks(css=css) as demo:
52
  gr.Markdown("# Semi-structured Retrieval Benchmark (STaRK) Leaderboard")
53
  gr.Markdown("Refer to the [STaRK paper](https://arxiv.org/pdf/2404.13207) for details on metrics, tasks and models.")
54
 
 
 
 
 
 
 
 
 
55
  with gr.Tabs() as outer_tabs:
56
- with gr.TabItem("Synthesized"):
57
- gr.Markdown("## Synthesized Data Results")
58
- with gr.Tabs() as inner_tabs_synthesized:
59
- with gr.TabItem("Amazon"):
60
- gr.DataFrame(format_dataframe(df, "STARK-AMAZON"))
61
- with gr.TabItem("MAG"):
62
- gr.DataFrame(format_dataframe(df, "STARK-MAG"))
63
- with gr.TabItem("Prime"):
64
- gr.DataFrame(format_dataframe(df, "STARK-PRIME"))
 
 
65
 
66
  with gr.TabItem("Human-Generated"):
67
- gr.Markdown("## Human-Generated Data Results")
68
  with gr.Tabs() as inner_tabs_human:
69
- with gr.TabItem("Amazon"):
70
- gr.DataFrame(format_dataframe(df, "STARK-AMAZON"))
71
- with gr.TabItem("MAG"):
72
- gr.DataFrame(format_dataframe(df, "STARK-MAG"))
73
- with gr.TabItem("Prime"):
74
- gr.DataFrame(format_dataframe(df, "STARK-PRIME"))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
75
 
76
  demo.launch()
 
2
  import pandas as pd
3
  import numpy as np
4
 
5
+ # Sample data based on your table (you'll need to update this with the full dataset)
6
+ data_synthesized_full = {
7
+ 'Method': ['BM25', 'DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)', 'ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b', 'multi-ada-002', 'ColBERTv2'],
8
+ 'STARK-AMAZON_Hit@1': [44.94, 15.29, 30.96, 26.56, 39.16, 40.93, 21.74, 42.08, 40.07, 46.10],
9
+ 'STARK-AMAZON_Hit@5': [67.42, 47.93, 51.06, 50.01, 62.73, 64.37, 41.65, 66.87, 64.98, 66.02],
10
+ 'STARK-AMAZON_R@20': [53.77, 44.49, 41.95, 52.05, 53.29, 54.28, 33.22, 56.52, 55.12, 53.44],
11
+ 'STARK-AMAZON_MRR': [55.30, 30.20, 40.66, 37.75, 50.35, 51.60, 31.47, 53.46, 51.55, 55.51],
12
+ 'STARK-MAG_Hit@1': [25.85, 10.51, 21.96, 12.88, 29.08, 30.06, 18.01, 37.90, 25.92, 31.18],
13
+ 'STARK-MAG_Hit@5': [45.25, 35.23, 36.50, 39.01, 49.61, 50.58, 34.85, 56.74, 50.43, 46.42],
14
+ 'STARK-MAG_R@20': [45.69, 42.11, 35.32, 46.97, 48.36, 50.49, 35.46, 46.40, 50.80, 43.94],
15
+ 'STARK-MAG_MRR': [34.91, 21.34, 29.14, 29.12, 38.62, 39.66, 26.10, 47.25, 36.94, 38.39],
16
+ 'STARK-PRIME_Hit@1': [12.75, 4.46, 6.53, 8.85, 12.63, 10.85, 10.10, 15.57, 15.10, 11.75],
17
+ 'STARK-PRIME_Hit@5': [27.92, 21.85, 15.67, 21.35, 31.49, 30.23, 22.49, 33.42, 33.56, 23.85],
18
+ 'STARK-PRIME_R@20': [31.25, 30.13, 16.52, 29.63, 36.00, 37.83, 26.34, 39.09, 38.05, 25.04],
19
+ 'STARK-PRIME_MRR': [19.84, 12.38, 11.05, 14.73, 21.41, 19.99, 16.12, 24.11, 23.49, 17.39]
20
+ }
21
+
22
+ data_synthesized_10 = {
23
+ 'Method': ['BM25', 'DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)', 'ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b', 'multi-ada-002', 'ColBERTv2', 'Claude3 Reranker', 'GPT4 Reranker'],
24
+ 'STARK-AMAZON_Hit@1': [42.68, 16.46, 30.09, 25.00, 39.02, 43.29, 18.90, 43.29, 40.85, 44.31, 45.49, 44.79],
25
+ 'STARK-AMAZON_Hit@5': [67.07, 50.00, 49.27, 48.17, 64.02, 67.68, 37.80, 71.34, 62.80, 65.24, 71.13, 71.17],
26
+ 'STARK-AMAZON_R@20': [54.48, 42.15, 41.91, 51.65, 49.30, 56.04, 34.73, 56.14, 52.47, 51.00, 53.77, 55.35],
27
+ 'STARK-AMAZON_MRR': [54.02, 30.20, 39.30, 36.87, 50.32, 54.20, 28.76, 55.07, 51.54, 55.07, 55.91, 55.69],
28
+ 'STARK-MAG_Hit@1': [27.81, 11.65, 22.89, 12.03, 28.20, 34.59, 19.17, 38.35, 25.56, 31.58, 36.54, 40.90],
29
+ 'STARK-MAG_Hit@5': [45.48, 36.84, 37.26, 37.97, 52.63, 50.75, 33.46, 58.64, 50.37, 47.36, 53.17, 58.18],
30
+ 'STARK-MAG_R@20': [44.59, 42.30, 44.16, 47.98, 49.25, 50.75, 29.85, 46.38, 53.03, 45.72, 48.36, 48.60],
31
+ 'STARK-MAG_MRR': [35.97, 21.82, 30.00, 28.70, 38.55, 42.90, 26.06, 48.25, 36.82, 38.98, 44.15, 49.00],
32
+ 'STARK-PRIME_Hit@1': [13.93, 5.00, 6.78, 7.14, 15.36, 12.14, 9.29, 16.79, 15.36, 15.00, 17.79, 18.28],
33
+ 'STARK-PRIME_Hit@5': [31.07, 23.57, 16.15, 17.14, 31.07, 31.42, 20.7, 34.29, 32.86, 26.07, 36.90, 37.28],
34
+ 'STARK-PRIME_R@20': [32.84, 30.50, 17.07, 32.95, 37.88, 37.34, 25.54, 41.11, 40.99, 27.78, 35.57, 34.05],
35
+ 'STARK-PRIME_MRR': [21.68, 13.50, 11.42, 16.27, 23.50, 21.23, 15.00, 24.99, 23.70, 19.98, 26.27, 26.55]
36
+ }
37
+
38
+ data_human_generated = {
39
  'Method': ['BM25', 'DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)', 'ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b', 'multi-ada-002', 'ColBERTv2', 'Claude3 Reranker', 'GPT4 Reranker'],
40
  'STARK-AMAZON_Hit@1': [27.16, 16.05, 25.93, 22.22, 39.50, 35.80, 29.63, 40.74, 46.91, 33.33, 53.09, 50.62],
41
  'STARK-AMAZON_Hit@5': [51.85, 39.51, 54.32, 49.38, 64.19, 62.96, 46.91, 71.60, 72.84, 55.56, 74.07, 75.31],
 
51
  'STARK-PRIME_MRR': [30.37, 7.05, 10.07, 9.39, 26.35, 24.33, 15.24, 34.28, 32.98, 19.67, 36.32, 34.82]
52
  }
53
 
54
+ df_synthesized_full = pd.DataFrame(data_synthesized_full)
55
+ df_synthesized_10 = pd.DataFrame(data_synthesized_10)
56
+ df_human_generated = pd.DataFrame(data_human_generated)
57
 
58
  def format_dataframe(df, dataset):
59
  # Filter the dataframe for the selected dataset
 
68
 
69
  return filtered_df
70
 
71
+ model_types = {
72
+ 'Sparse Retriever': ['BM25'],
73
+ 'Small Dense Retrievers': ['DPR (roberta)', 'ANCE (roberta)', 'QAGNN (roberta)'],
74
+ 'LLM-based Dense Retrievers': ['ada-002', 'voyage-l2-instruct', 'LLM2Vec', 'GritLM-7b'],
75
+ 'Multivector Retrievers': ['multi-ada-002', 'ColBERTv2'],
76
+ 'LLM Rerankers': ['Claude3 Reranker', 'GPT4 Reranker']
77
+ }
78
+
79
+ def filter_by_model_type(df, selected_types):
80
+ selected_models = [model for type in selected_types for model in model_types[type]]
81
+ return df[df['Method'].isin(selected_models)]
82
+
83
  css = """
84
  table > thead {
85
  white-space: normal
 
98
  gr.Markdown("# Semi-structured Retrieval Benchmark (STaRK) Leaderboard")
99
  gr.Markdown("Refer to the [STaRK paper](https://arxiv.org/pdf/2404.13207) for details on metrics, tasks and models.")
100
 
101
+ with gr.Row():
102
+ model_type_filter = gr.CheckboxGroup(
103
+ choices=list(model_types.keys()),
104
+ value=list(model_types.keys()),
105
+ label="Model types",
106
+ interactive=True
107
+ )
108
+
109
  with gr.Tabs() as outer_tabs:
110
+ with gr.TabItem("Synthesized (full)"):
111
+ with gr.Tabs() as inner_tabs_synthesized_full:
112
+ for dataset in ['AMAZON', 'MAG', 'PRIME']:
113
+ with gr.TabItem(dataset):
114
+ gr.DataFrame(format_dataframe(df_synthesized_full, f"STARK-{dataset}"))
115
+
116
+ with gr.TabItem("Synthesized (10%)"):
117
+ with gr.Tabs() as inner_tabs_synthesized_10:
118
+ for dataset in ['AMAZON', 'MAG', 'PRIME']:
119
+ with gr.TabItem(dataset):
120
+ gr.DataFrame(format_dataframe(df_synthesized_10, f"STARK-{dataset}"))
121
 
122
  with gr.TabItem("Human-Generated"):
 
123
  with gr.Tabs() as inner_tabs_human:
124
+ for dataset in ['AMAZON', 'MAG', 'PRIME']:
125
+ with gr.TabItem(dataset):
126
+ gr.DataFrame(format_dataframe(df_human_generated, f"STARK-{dataset}"))
127
+
128
+ def update_tables(selected_types):
129
+ filtered_df_full = filter_by_model_type(df_synthesized_full, selected_types)
130
+ filtered_df_10 = filter_by_model_type(df_synthesized_10, selected_types)
131
+ filtered_df_human = filter_by_model_type(df_human_generated, selected_types)
132
+
133
+ outputs = []
134
+ for df in [filtered_df_full, filtered_df_10, filtered_df_human]:
135
+ for dataset in ['AMAZON', 'MAG', 'PRIME']:
136
+ outputs.append(format_dataframe(df, f"STARK-{dataset}"))
137
+
138
+ return outputs
139
+
140
+ # Create DataFrame components for each tab and dataset
141
+ dataframes = []
142
+ for _ in range(3): # 3 main tabs
143
+ for _ in range(3): # 3 datasets per tab
144
+ df_component = gr.DataFrame(visible=False)
145
+ dataframes.append(df_component)
146
+
147
+ # Update visibility and content of DataFrames based on selected tab
148
+ def update_visibility(tab):
149
+ visibilities = [False] * 9
150
+ start_index = tab * 3
151
+ visibilities[start_index:start_index+3] = [True, True, True]
152
+ return visibilities
153
+
154
+ outer_tabs.select(
155
+ update_visibility,
156
+ None,
157
+ dataframes
158
+ )
159
+
160
+ # Connect model type filter to update_tables function
161
+ model_type_filter.change(
162
+ update_tables,
163
+ inputs=[model_type_filter],
164
+ outputs=dataframes
165
+ )
166
+
167
+ # Initial load of tables
168
+ demo.load(
169
+ update_tables,
170
+ inputs=[model_type_filter],
171
+ outputs=dataframes
172
+ )
173
 
174
  demo.launch()