Add Humanity's Last Exam, LiveBench and LiveCodeBench; Remove Codeforces; Update Simple Bench
b605a32
| import json | |
| from datetime import datetime, date | |
| import gradio as gr | |
| import plotly.graph_objects as go | |
| def create_big_five_capex_plot() -> go.Figure: | |
| # Read data from the JSON Lines file. | |
| with open("big_five_capex.jsonl", "r") as file: | |
| data = [json.loads(line) for line in file if line.strip()] | |
| quarters: list[str] = [entry["Quarter"] for entry in data] | |
| companies = ['Microsoft', 'Google', 'Meta', 'Amazon'] | |
| colors = ['#80bb00', '#ee161f', '#0065e3', '#ff6200'] | |
| x_positions = list(range(len(quarters))) | |
| traces = [] | |
| for company, color in zip(companies, colors): | |
| y_data = [entry[company] for entry in data] | |
| traces.append(go.Bar( | |
| name=company, | |
| x=x_positions, | |
| y=y_data, | |
| marker_color=color | |
| )) | |
| fig = go.Figure(data=traces) | |
| fig.update_layout( | |
| barmode="stack", | |
| title="Capital Expenditures of Amazon, Meta, Google and Microsoft in Millions of USD per Quarter", | |
| xaxis_title="Quarter", | |
| yaxis_title="Capital Expenditures (Millions USD)", | |
| xaxis=dict( | |
| tickmode='array', | |
| tickvals=x_positions, | |
| ticktext=quarters | |
| ), | |
| height=800 | |
| ) | |
| # Calculate the x position for the vertical dotted line. | |
| # We want the line drawn between "2023 Q1" and "2023 Q2". | |
| try: | |
| idx_q1 = quarters.index("2023 Q1") | |
| idx_q2 = quarters.index("2023 Q2") | |
| vline_x = (idx_q1 + idx_q2) / 2 # position midway between the two quarters | |
| except ValueError: | |
| # Fall back if quarters not found. | |
| vline_x = 0 | |
| # Add a vertical dotted line spanning the full height | |
| fig.add_shape( | |
| type="line", | |
| xref="x", | |
| yref="paper", | |
| x0=vline_x, | |
| y0=0, | |
| x1=vline_x, | |
| y1=1, | |
| line=dict( | |
| color="black", | |
| dash="dot", | |
| width=2 | |
| ) | |
| ) | |
| # Add an annotation label above the vertical line. | |
| fig.add_annotation( | |
| x=vline_x, | |
| y=1.05, # place just above the top of the plotting area | |
| xref="x", | |
| yref="paper", | |
| text="AI arms race begins", | |
| showarrow=False, | |
| font=dict( | |
| color="black", | |
| size=12 | |
| ), | |
| align="center" | |
| ) | |
| return fig | |
| def create_simple_plot(data_path: str, | |
| name: str, | |
| subtitle: str, | |
| start_date: datetime, end_date: datetime, | |
| min_value: int = 0, max_value: int = 100, | |
| labeled_horizontal_lines: dict[str, float] = None) -> go.Figure: | |
| leaderboard = [] | |
| with open(data_path, 'r') as file: | |
| for line in file: | |
| leaderboard.append(json.loads(line)) | |
| models = [] | |
| with open("models.jsonl", 'r') as file: | |
| for line in file: | |
| models.append(json.loads(line)) | |
| data = [] | |
| for entry in leaderboard: | |
| model_name = entry['model'] | |
| score = entry['score'] | |
| model_info = next((m for m in models if m['Name'] == model_name), None) | |
| if model_info: | |
| release_date = datetime.strptime(model_info['Release Date'], "%Y-%m-%d") | |
| data.append({'model': model_name, 'score': score, 'release_date': release_date}) | |
| else: | |
| print(f"[WARNING] Model '{model_name}' not found in models.jsonl") | |
| data.sort(key=lambda x: x['release_date']) | |
| x_dates = [d['release_date'] for d in data] | |
| y_scores = [] | |
| max_score = 0 | |
| for entry in data: | |
| if entry['score'] > max_score: | |
| max_score = entry['score'] | |
| y_scores.append(max_score) | |
| fig = go.Figure() | |
| fig.add_trace(go.Scatter( | |
| x=x_dates, | |
| y=y_scores, | |
| mode='lines', | |
| line=dict(shape='hv', width=2), | |
| name='Best Score to Date' | |
| )) | |
| for i, entry in enumerate(data): | |
| if i == 0 or y_scores[i] > y_scores[i - 1]: | |
| fig.add_trace(go.Scatter( | |
| x=[entry['release_date']], | |
| y=[entry['score']], | |
| mode='markers+text', | |
| marker=dict(size=10), | |
| text=[entry['model']], | |
| textposition="top center", | |
| name=entry['model'] | |
| )) | |
| fig.update_layout( | |
| title=f'{name} Over Time<br><sup>{subtitle}</sup>', | |
| xaxis_title='Publication or Release Date', | |
| yaxis_title=name, | |
| hovermode='x unified', | |
| xaxis=dict( | |
| range=[start_date, end_date], | |
| type='date' | |
| ), | |
| yaxis=dict( | |
| range=[min_value, max_value] | |
| ), | |
| height=800 | |
| ) | |
| if labeled_horizontal_lines: | |
| for label, y_value in labeled_horizontal_lines.items(): | |
| fig.add_hline( | |
| y=y_value, | |
| line_dash="dot", | |
| line_color="black", | |
| annotation_text=label, | |
| annotation_position="right", | |
| annotation=dict( | |
| font_size=12, | |
| font_color="black", | |
| xanchor="left", | |
| yanchor="middle", | |
| xshift=10 | |
| ) | |
| ) | |
| return fig | |
| with gr.Blocks() as demo: | |
| with gr.Tab("System Performance Over Time"): | |
| with gr.Tab("Legend"): | |
| legend_markdown: gr.Markdown = gr.Markdown( | |
| value=""" | |
| ## Benchmarks and Top Scores | |
| | Benchmark | Top Score | | |
| |-----------|-----------| | |
| | Humanity's Last Exam | 🔴 7% | | |
| | BigCodeBench | 🟠 36% | | |
| | Simple Bench | 🟠 42% | | |
| | PlanBench | 🟠 53% | | |
| | GAIA | 🟡 65% | | |
| | LiveBench Language | 🟡 65% | | |
| | LiveBench Data Analysis | 🟡 71% | | |
| | LiveCodeBench | 🟡 73% | | |
| | ARC-AGI-Pub (Semi-Private Eval) | 🟡 76% | | |
| | LiveBench | 🟡 76% | | |
| | GPQA | 🟡 76% | | |
| | LiveBench Mathematics | 🟡 81% | | |
| | ZebraLogic | 🟡 81% | | |
| | LiveBench Coding | 🟡 83% | | |
| | ARC-AGI-Pub (Public Eval) | 🟡 83% | | |
| | LiveBench IF | 🟡 86% | | |
| | ZeroEval | 🟡 86% | | |
| | MATH-L5 | 🟡 89% | | |
| | LiveBench Reasoning | 🟢 92% | | |
| | MMLU-Redux | 🟢 93% | | |
| | CRUX | 🟢 96% | | |
| ## Colors | |
| | Color | Score Range | | |
| |-------|------------| | |
| | 🔴 Red | Below 30% | | |
| | 🟠 Orange | 30% to 60% | | |
| | 🟡 Yellow | 60% to 90% | | |
| | 🟢 Green | Above 90% |""" | |
| ) | |
| with gr.Tab("🔴 Humanity's Last Exam") as humanitys_last_exam_tab: | |
| humanitys_last_exam_plot: gr.Plot = gr.Plot() | |
| humanitys_last_exam_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [Humanity's Last Exam Quantitative Results](https://lastexam.ai/)""" | |
| ) | |
| with gr.Tab("🟠 BigCodeBench") as bigcodebench_tab: | |
| bigcodebench_plot: gr.Plot = gr.Plot() | |
| bigcodebench_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [BigCodeBench Leaderboard](https://bigcode-bench.github.io/)""" | |
| ) | |
| with gr.Tab("🟠 Simple Bench") as simple_bench_tab: | |
| simple_bench_plot: gr.Plot = gr.Plot() | |
| simple_bench_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [SimpleBench Leaderboard](https://simple-bench.com/)""" | |
| ) | |
| with gr.Tab("🟠 PlanBench") as planbench_tab: | |
| planbench_plot: gr.Plot = gr.Plot() | |
| planbench_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [Valmeekam et al. 2024](https://arxiv.org/abs/2409.13373)""" | |
| ) | |
| with gr.Tab("🟡 GAIA") as gaia_tab: | |
| gaia_plot: gr.Plot = gr.Plot() | |
| gaia_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [GAIA Leaderboard](https://huggingface.co/spaces/gaia-benchmark/leaderboard)""" | |
| ) | |
| with gr.Tab("🟡 LiveBench Language") as livebench_language_tab: | |
| livebench_language_plot: gr.Plot = gr.Plot() | |
| livebench_language_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" | |
| ) | |
| with gr.Tab("🟡 LiveBench Data Analysis") as livebench_data_analysis_tab: | |
| livebench_data_analysis_plot: gr.Plot = gr.Plot() | |
| livebench_data_analysis_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" | |
| ) | |
| with gr.Tab("🟡 LiveCodeBench") as livecodebench_tab: | |
| livecodebench_plot: gr.Plot = gr.Plot() | |
| livecodebench_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [LiveCodeBench Leaderboard](https://livecodebench.github.io/leaderboard.html)""" | |
| ) | |
| with gr.Tab("🟡 ARC-AGI-Pub") as arc_agi_tab: | |
| with gr.Tab("🟡 Semi-Private Eval") as arc_agi_semi_private_eval_tab: | |
| arc_agi_semi_private_eval_plot: gr.Plot = gr.Plot() | |
| with gr.Tab("🟡 Public Eval") as arc_agi_public_eval_tab: | |
| arc_agi_public_eval_plot: gr.Plot = gr.Plot() | |
| arc_agi_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [ARC Prize 2024](https://arcprize.org/2024-results)""" | |
| ) | |
| with gr.Tab("🟡 LiveBench") as livebench_tab: | |
| livebench_plot: gr.Plot = gr.Plot() | |
| livebench_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" | |
| ) | |
| with gr.Tab("🟡 GPQA") as gpqa_tab: | |
| gpqa_plot: gr.Plot = gr.Plot() | |
| gpqa_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [Epoch AI Benchmarking Dashboard](https://epoch.ai/data/ai-benchmarking-dashboard)""" | |
| ) | |
| with gr.Tab("🟡 LiveBench Mathematics") as livebench_mathematics_tab: | |
| livebench_mathematics_plot: gr.Plot = gr.Plot() | |
| livebench_mathematics_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" | |
| ) | |
| with gr.Tab("🟡 ZebraLogic") as zeroeval_zebralogic_tab: | |
| zeroeval_zebralogic_plot: gr.Plot = gr.Plot() | |
| zeroeval_zebralogic_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)""" | |
| ) | |
| with gr.Tab("🟡 LiveBench Coding") as livebench_coding_tab: | |
| livebench_coding_plot: gr.Plot = gr.Plot() | |
| livebench_coding_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" | |
| ) | |
| with gr.Tab("🟡 LiveBench IF") as livebench_if_tab: | |
| livebench_if_plot: gr.Plot = gr.Plot() | |
| livebench_if_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [LiveBench IF](https://livebench.ai/)""" | |
| ) | |
| with gr.Tab("🟡 ZeroEval") as zeroeval_average_tab: | |
| zeroeval_average_plot: gr.Plot = gr.Plot() | |
| zeroeval_average_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)""" | |
| ) | |
| with gr.Tab("🟡 MATH-L5") as zeroeval_math_l5_tab: | |
| zeroeval_math_l5_plot: gr.Plot = gr.Plot() | |
| zeroeval_math_l5_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)""" | |
| ) | |
| with gr.Tab("🟢 LiveBench Reasoning") as livebench_reasoning_tab: | |
| livebench_reasoning_plot: gr.Plot = gr.Plot() | |
| livebench_reasoning_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [LiveBench Leaderboard](https://livebench.ai/)""" | |
| ) | |
| with gr.Tab("🟢 MMLU-Redux") as zeroeval_mmlu_redux_tab: | |
| zeroeval_mmlu_redux_plot: gr.Plot = gr.Plot() | |
| zeroeval_mmlu_redux_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)""" | |
| ) | |
| with gr.Tab("🟢 CRUX") as zeroeval_crux_tab: | |
| zeroeval_crux_plot: gr.Plot = gr.Plot() | |
| zeroeval_crux_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [ZeroEval Leaderboard](https://huggingface.co/spaces/allenai/ZeroEval)""" | |
| ) | |
| with gr.Tab("OpenCompass", visible=False): | |
| opencompass_plot: gr.Plot = gr.Plot() | |
| opencompass_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [OpenCompass LLM Leaderboard](https://huggingface.co/spaces/opencompass/opencompass-llm-leaderboard)""" | |
| ) | |
| with gr.Tab("SWE-bench", visible=False): | |
| swe_bench_plot: gr.Plot = gr.Plot() | |
| swe_bench_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [SWE-bench Leaderboard](https://www.swebench.com/)""" | |
| ) | |
| with gr.Tab("WebArena", visible=False): | |
| webarena_plot: gr.Plot = gr.Plot() | |
| webarena_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [X-WebArena-Leaderboard](https://docs.google.com/spreadsheets/d/1M801lEpBbKSNwP-vDBkC_pF7LdyGU1f_ufZb_NWNBZQ)""" | |
| ) | |
| with gr.Tab("OSWorld", visible=False): | |
| osworld_plot: gr.Plot = gr.Plot() | |
| osworld_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [OSWorld Benchmark](https://os-world.github.io/)""" | |
| ) | |
| with gr.Tab("EMMA-Mini", visible=False): | |
| emma_plot: gr.Plot = gr.Plot() | |
| emma_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [EMMA Leaderboard](https://emma-benchmark.github.io/#leaderboard)""" | |
| ) | |
| with gr.Tab("MathVista", visible=False): | |
| mathvista_plot: gr.Plot = gr.Plot() | |
| mathvista_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [Leaderboard on MathVista](https://mathvista.github.io/#leaderboard)""" | |
| ) | |
| with gr.Tab("DABStep", visible=False): | |
| dabstep_plot: gr.Plot = gr.Plot() | |
| dabstep_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [DABStep Leaderboard](https://huggingface.co/spaces/adyen/DABstep)""" | |
| ) | |
| with gr.Tab("lineage-bench", visible=False): | |
| lineage_bench_plot: gr.Plot = gr.Plot() | |
| lineage_bench_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [lineage-bench Results](https://github.com/fairydreaming/lineage-bench)""" | |
| ) | |
| with gr.Tab("Step-Game", visible=False): | |
| step_game_plot: gr.Plot = gr.Plot() | |
| step_game_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [Step-Game TrueSkill Leaderboard](https://github.com/lechmazur/step_game)""" | |
| ) | |
| with gr.Tab("HHEM", visible=False): | |
| hhem_plot: gr.Plot = gr.Plot() | |
| hhem_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [Vectara Hallucination Leaderboard](https://github.com/vectara/hallucination-leaderboard)""" | |
| ) | |
| with gr.Tab("NYT Connections", visible=False): | |
| nyt_connections_exam_plot: gr.Plot = gr.Plot() | |
| nyt_connections_exam_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [NYT Connections Leaderboard](https://github.com/lechmazur/nyt-connections)""" | |
| ) | |
| with gr.Tab("USACO", visible=False): | |
| usaco_plot: gr.Plot = gr.Plot() | |
| usaco_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [USACO Leaderboard](https://hal.cs.princeton.edu/usaco)""" | |
| ) | |
| with gr.Tab("AppWorld", visible=False): | |
| appworld_plot: gr.Plot = gr.Plot() | |
| appworld_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [AppWorld Agent Scores](https://appworld.dev/leaderboard)""" | |
| ) | |
| with gr.Tab("CORE-Bench", visible=False): | |
| core_bench_plot: gr.Plot = gr.Plot() | |
| core_bench_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [HAL Leaderboards](https://hal.cs.princeton.edu/#leaderboards)""" | |
| ) | |
| with gr.Tab("Cybench", visible=False): | |
| cybench_plot: gr.Plot = gr.Plot() | |
| cybench_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [Cybench Leaderboard](https://hal.cs.princeton.edu/cybench)""" | |
| ) | |
| with gr.Tab("MultiChallenge", visible=False): | |
| multichallenge_plot: gr.Plot = gr.Plot() | |
| multichallenge_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [SEAL Leaderboard: MultiChallenge](https://scale.com/leaderboard/multichallenge)""" | |
| ) | |
| with gr.Tab("VISTA", visible=False): | |
| vista_plot: gr.Plot = gr.Plot() | |
| vista_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [SEAL Leaderboard: Visual-Language Understanding](https://scale.com/leaderboard/visual_language_understanding)""" | |
| ) | |
| with gr.Tab("ToolComp", visible=False): | |
| with gr.Tab("Enterprise"): | |
| toolcomp_enterprise_plot: gr.Plot = gr.Plot() | |
| toolcomp_enterprise_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [SEAL Leaderboard: Agentic Tool Use (Enterprise)](https://scale.com/leaderboard/tool_use_enterprise)""" | |
| ) | |
| with gr.Tab("Chat"): | |
| toolcomp_chat_plot: gr.Plot = gr.Plot() | |
| toolcomp_chat_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [SEAL Leaderboard: Agentic Tool Use (Chat)](https://scale.com/leaderboard/tool_use_chat)""" | |
| ) | |
| with gr.Tab("BFCL", visible=False): | |
| bfcl_plot: gr.Plot = gr.Plot() | |
| bfcl_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [BFCL Leaderboard](https://gorilla.cs.berkeley.edu/leaderboard.html)""" | |
| ) | |
| with gr.Tab("EvalPlus", visible=False): | |
| evalplus_plot: gr.Plot = gr.Plot() | |
| evalplus_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [EvalPlus Leaderboard](https://evalplus.github.io/leaderboard.html)""" | |
| ) | |
| with gr.Tab("Aider Polyglot", visible=False): | |
| aider_plot: gr.Plot = gr.Plot() | |
| aider_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [Aider LLM Leaderboards](https://aider.chat/docs/leaderboards/)""" | |
| ) | |
| with gr.Tab("QuALITY", visible=False): | |
| quality_plot: gr.Plot = gr.Plot() | |
| quality_markdown: gr.Markdown = gr.Markdown( | |
| value="""Source: [QuALITY Leaderboard](https://nyu-mll.github.io/quality/)""" | |
| ) | |
| with gr.Tab("Finance") as finance_tab: | |
| with gr.Tab("Big Tech Capex") as big_five_capex_tab: | |
| big_five_capex_plot: gr.Plot = gr.Plot() | |
| with gr.Tab("NVIDIA Revenue", visible=False) as nvidia_revenue: | |
| nvidia_revenue_plot: gr.Plot = gr.Plot() | |
| big_five_capex_tab.select(fn=create_big_five_capex_plot, outputs=big_five_capex_plot) | |
| arc_agi_public_eval_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("arc_agi_leaderboard.jsonl"), | |
| gr.State( | |
| "ARC-AGI-Pub Score (Public Eval, $20 Compute Budget per Task, General-Purpose Systems)"), | |
| gr.State( | |
| "\"ARC can be seen as a general artificial intelligence benchmark, as a program synthesis benchmark, or as a psychometric intelligence test.\" (Chollet, 2019)"), | |
| gr.State(date(2024, 5, 1)), gr.State(date(2025, 1, 1)), | |
| gr.State(0), gr.State(100), | |
| gr.State({"Humans\n(LeGris et al. 2024)": 64.2})], | |
| outputs=arc_agi_public_eval_plot) | |
| arc_agi_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("arc_agi_semi_private_eval_leaderboard.jsonl"), | |
| gr.State( | |
| "ARC-AGI-Pub Score (Semi-Private Eval, $20 Compute Budget per Task, General-Purpose Systems)"), | |
| gr.State( | |
| "\"ARC can be seen as a general artificial intelligence benchmark, as a program synthesis benchmark, or as a psychometric intelligence test.\" (Chollet, 2019)"), | |
| gr.State(date(2024, 5, 1)), gr.State(date(2025, 1, 1)), | |
| gr.State(0), gr.State(100), | |
| gr.State({"MTurkers": 77})], | |
| outputs=arc_agi_semi_private_eval_plot) | |
| arc_agi_semi_private_eval_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("arc_agi_semi_private_eval_leaderboard.jsonl"), | |
| gr.State( | |
| "ARC-AGI-Pub Score (Semi-Private Eval, $20 Compute Budget per Task, General-Purpose Systems)"), | |
| gr.State( | |
| "\"ARC can be seen as a general artificial intelligence benchmark, as a program synthesis benchmark, or as a psychometric intelligence test.\" (Chollet, 2019)"), | |
| gr.State(date(2024, 5, 1)), gr.State(date(2025, 1, 1)), | |
| gr.State(0), gr.State(100), | |
| gr.State({"MTurkers": 77})], | |
| outputs=arc_agi_semi_private_eval_plot) | |
| finance_tab.select(fn=create_big_five_capex_plot, outputs=big_five_capex_plot) | |
| simple_bench_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("simple_bench_leaderboard.jsonl"), | |
| gr.State("Simple Bench Score"), | |
| gr.State( | |
| "\"multiple-choice text benchmark [...] [including] over 200 questions covering spatio-temporal reasoning, social intelligence, and what we call linguistic adversarial robustness\" (Philip & Hemang, 2024)"), | |
| gr.State(date(2024, 4, 9)), gr.State(date(2025, 2, 1)), | |
| gr.State(0), gr.State(100), | |
| gr.State({"Humans": 83.7})], | |
| outputs=simple_bench_plot) | |
| planbench_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("planbench_leaderboard.jsonl"), | |
| gr.State("PlanBench Score (Mystery Blocksworld, 0-shot)"), | |
| gr.State( | |
| "\"benchmark suite based on the kinds of domains used in the automated planning community [...] to test the capabilities of LLMs in planning or reasoning about actions and change.\" (Valmeekam et al. 2022)"), | |
| gr.State(date(2023, 3, 1)), gr.State(date(2024, 9, 20))], | |
| outputs=planbench_plot) | |
| bigcodebench_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("bigcodebench_hard_average_leaderboard.jsonl"), | |
| gr.State("BigCodeBench Score (Hard, Average of Complete and Instruct)"), | |
| gr.State( | |
| "\"benchmark that challenges LLMs to invoke multiple function calls as tools from 139 libraries and 7 domains for 1,140 fine-grained tasks\" (Zhuo et al. 2024)"), | |
| gr.State(date(2023, 6, 1)), gr.State(date(2025, 1, 1))], | |
| outputs=bigcodebench_plot) | |
| gaia_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("gaia_leaderboard.jsonl"), | |
| gr.State("General AI Assistants (GAIA) Benchmark Score (Test Set, Average)"), | |
| gr.State( | |
| "\"real-world questions that require a set of fundamental abilities such as reasoning, multi-modality handling, web browsing, and generally tool-use proficiency\" (Mialon et al. 2023)"), | |
| gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1)), | |
| gr.State(0), gr.State(100), | |
| gr.State({"Humans": 92})], | |
| outputs=gaia_plot) | |
| gpqa_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("gpqa_leaderboard.jsonl"), | |
| gr.State("Graduate-Level Google-Proof Q&A (GPQA) Benchmark Score"), | |
| gr.State( | |
| "\"challenging dataset of 448 multiple-choice questions written by domain experts in biology, physics, and chemistry [that] are high-quality and extremely difficult\" (Rein et al. 2023)"), | |
| gr.State(date(2023, 6, 1)), gr.State(date(2025, 1, 1)), | |
| gr.State(25), gr.State(100), | |
| gr.State({"Highly skilled non-expert validators": 34, "PhD-level domain experts": 65})], | |
| outputs=gpqa_plot) | |
| zeroeval_average_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("zeroeval_average_leaderboard.jsonl"), | |
| gr.State("ZeroEval Average (MMLU-Redux, ZebraLogic, CRUX and MATH-5) Score"), | |
| gr.State( | |
| "\"a simple unified framework for evaluating language models on various tasks\" (Ai2, 2024)"), | |
| gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))], | |
| outputs=zeroeval_average_plot) | |
| zeroeval_mmlu_redux_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("zeroeval_mmlu_redux_leaderboard.jsonl"), | |
| gr.State( | |
| "ZeroEval MMLU-Redux (Massive Multitask Language Understanding) Score"), | |
| gr.State( | |
| "\"knowledge reasoning\" (Ai2, 2024); \"subset of 3,000 manually re-annotated questions across 30 MMLU subjects\" (Gema et al. 2024)"), | |
| gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))], | |
| outputs=zeroeval_mmlu_redux_plot) | |
| zeroeval_zebralogic_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("zeroeval_zebralogic_leaderboard.jsonl"), | |
| gr.State("ZeroEval ZebraLogic Score"), | |
| gr.State( | |
| "\"logical reasoning\" (Ai2, 2024); \"Each example is a Logic Grid Puzzle [...] often used to test humans' logical reasoning abilities\" (Lin, 2024)"), | |
| gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))], | |
| outputs=zeroeval_zebralogic_plot) | |
| zeroeval_crux_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("zeroeval_crux_leaderboard.jsonl"), | |
| gr.State( | |
| "ZeroEval CRUX (Code Reasoning, Understanding, and eXecution Evaluation) Score"), | |
| gr.State( | |
| "\"code reasoning\" (Ai2, 2024); \"benchmark consisting of 800 Python functions (3-13 lines). Each function comes with [...] two natural tasks: input prediction and output prediction.\" (Gu et al. 2024)"), | |
| gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))], | |
| outputs=zeroeval_crux_plot) | |
| zeroeval_math_l5_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("zeroeval_math_l5_leaderboard.jsonl"), | |
| gr.State("ZeroEval MATH-L5 (Difficulty Level 5 of MATH) Score"), | |
| gr.State( | |
| "\"math reasoning\" (Ai2, 2024); \"dataset of 12,500 challenging competition mathematics problems. [...] a subject’s hardest problems are assigned a difficulty level of ‘5.’\" (Hendrycks et al. 2021)"), | |
| gr.State(date(2023, 3, 1)), gr.State(date(2025, 1, 1))], | |
| outputs=zeroeval_math_l5_plot) | |
| livebench_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("livebench.jsonl"), | |
| gr.State("LiveBench-2024-11-25: Global Average Score"), | |
| gr.State( | |
| "\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), | |
| gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], | |
| outputs=livebench_plot) | |
| livebench_reasoning_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("livebench_reasoning.jsonl"), | |
| gr.State("LiveBench-2024-11-25: Reasoning Average Score"), | |
| gr.State( | |
| "\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), | |
| gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], | |
| outputs=livebench_reasoning_plot) | |
| livebench_coding_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("livebench_coding.jsonl"), | |
| gr.State("LiveBench-2024-11-25: Coding Average Score"), | |
| gr.State( | |
| "\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), | |
| gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], | |
| outputs=livebench_coding_plot) | |
| livebench_mathematics_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("livebench_mathematics.jsonl"), | |
| gr.State("LiveBench-2024-11-25: Mathematics Average Score"), | |
| gr.State( | |
| "\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), | |
| gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], | |
| outputs=livebench_mathematics_plot) | |
| livebench_data_analysis_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("livebench_data_analysis.jsonl"), | |
| gr.State("LiveBench-2024-11-25: Data Analysis Average Score"), | |
| gr.State( | |
| "\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), | |
| gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], | |
| outputs=livebench_data_analysis_plot) | |
| livebench_language_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("livebench_language.jsonl"), | |
| gr.State("LiveBench-2024-11-25: Language Average Score"), | |
| gr.State( | |
| "\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), | |
| gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], | |
| outputs=livebench_language_plot) | |
| livebench_if_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("livebench_if.jsonl"), | |
| gr.State("LiveBench-2024-11-25: IF Average Score"), | |
| gr.State( | |
| "\"LiveBench is designed to limit potential contamination by releasing new questions regularly [...] Each question has verifiable, objective ground-truth answers\" (White et al. 2024)"), | |
| gr.State(date(2024, 2, 29)), gr.State(date(2025, 2, 1))], | |
| outputs=livebench_if_plot) | |
| humanitys_last_exam_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("humanitys_last_exam.jsonl"), | |
| gr.State("Humanity's Last Exam (Multi-Modal Models Only) Score"), | |
| gr.State( | |
| "\"multi-modal benchmark at the frontier of human knowledge, designed to be the final closed-ended academic benchmark of its kind with broad subject coverage\" (Phan et al. 2025)"), | |
| gr.State(date(2024, 5, 13)), gr.State(date(2025, 2, 11))], | |
| outputs=humanitys_last_exam_plot) | |
| livecodebench_tab.select(fn=create_simple_plot, | |
| inputs=[gr.State("livecodebench.jsonl"), | |
| gr.State("LiveCodeBench (7/1/2024 to 2/1/2025) Score"), | |
| gr.State( | |
| "\"comprehensive and contamination-free evaluation of LLMs for code, which continuously collects new problems over time from contests across three competition platforms\" (Jain et al. 2024)"), | |
| gr.State(date(2024, 4, 9)), gr.State(date(2025, 2, 1))], | |
| outputs=livecodebench_plot) | |
| if __name__ == "__main__": | |
| demo.launch() | |