import sys
import gradio as gr
import numpy as np
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
# fmt: off
type_emoji = {
"RTL-Specific": "🔴",
"General": "🟢",
"Coding": "🔵"
}
# fmt: on
def model_hyperlink(link, model_name, release):
if release == "V1":
return f'{model_name}'
else:
return f'{model_name} new'
def handle_special_cases(benchmark, metric):
if metric == "Exact Matching (EM)":
benchmark = "RTL-Repo"
elif benchmark == "RTL-Repo":
metric = "Exact Matching (EM)"
return benchmark, metric
def filter_RTLRepo(subset: pd.DataFrame) -> pd.DataFrame:
subset = subset.drop(subset[subset.Score < 0.0].index)
details = subset[
["Model", "Model URL", "Model Type", "Params", "Release"]
].drop_duplicates("Model")
filtered_df = subset[["Model", "Score"]].rename(
columns={"Score": "Exact Matching (EM)"}
)
filtered_df = pd.merge(filtered_df, details, on="Model", how="left")
filtered_df["Model"] = filtered_df.apply(
lambda row: model_hyperlink(row["Model URL"], row["Model"], row["Release"]),
axis=1,
)
filtered_df["Type"] = filtered_df["Model Type"].map(lambda x: type_emoji.get(x, ""))
filtered_df = filtered_df[["Type", "Model", "Params", "Exact Matching (EM)"]]
filtered_df = filtered_df.sort_values(
by="Exact Matching (EM)", ascending=False
).reset_index(drop=True)
return filtered_df
def filter_bench(subset: pd.DataFrame, df_agg=None, agg_column=None) -> pd.DataFrame:
details = subset[
["Model", "Model URL", "Model Type", "Params", "Release"]
].drop_duplicates("Model")
if "RTLLM" in subset["Benchmark"].unique():
pivot_df = (
subset.pivot_table(
index="Model", columns="Metric", values="Score", aggfunc=custom_agg_s2r
)
.reset_index()
.round(2)
)
else:
pivot_df = (
subset.pivot_table(
index="Model", columns="Metric", values="Score", aggfunc=custom_agg_cc
)
.reset_index()
.round(2)
)
# if df_agg is not None and agg_column is not None and agg_column in df_agg.columns:
# agg_data = df_agg[["Model", agg_column]].rename(
# columns={agg_column: "Aggregated ⬆️"}
# )
# pivot_df = pd.merge(pivot_df, agg_data, on="Model", how="left")
# else: # fallback
# pivot_df["Aggregated ⬆️"] = pivot_df.mean(axis=1, numeric_only=True).round(2)
pivot_df = pd.merge(pivot_df, details, on="Model", how="left")
pivot_df["Model"] = pivot_df.apply(
lambda row: model_hyperlink(row["Model URL"], row["Model"], row["Release"]),
axis=1,
)
pivot_df["Type"] = pivot_df["Model Type"].map(lambda x: type_emoji.get(x, ""))
pivot_df["Post-Synthesis (PSQ)"] = (
pivot_df[["Power", "Performance", "Area"]].mean(axis=1).round(2)
)
pivot_df.rename(
columns={
"Params": "Parameters (B)",
"Syntax (STX)": "Syntax",
"Functionality (FNC)": "Functionality",
"Synthesis (SYN)": "Synthesis",
"Post-Synthesis (PSQ)": "Post-Synthesis",
},
inplace=True,
)
columns_order = [
"Type",
"Model",
"Parameters (B)",
"Syntax",
"Functionality",
"Synthesis",
"Post-Synthesis",
]
pivot_df = pivot_df[[col for col in columns_order if col in pivot_df.columns]]
pivot_df = pivot_df.sort_values(by="Functionality", ascending=False).reset_index(
drop=True
)
return pivot_df
def custom_agg_s2r(vals):
if len(vals) == 2:
s2r_val = vals.iloc[0]
rtllm_val = vals.iloc[1]
w1 = 155
w2 = 47
result = (w1 * s2r_val + w2 * rtllm_val) / (w1 + w2)
else:
result = vals.iloc[0]
return round(result, 2)
def custom_agg_cc(vals):
if len(vals) == 2:
veval_val = vals.iloc[0]
vgen_val = vals.iloc[1]
w1 = 155
w2 = 17
result = (w1 * veval_val + w2 * vgen_val) / (w1 + w2)
else:
result = vals.iloc[0]
return round(result, 2)
def filter_bench_all(
subset: pd.DataFrame, df_agg=None, agg_column=None
) -> pd.DataFrame:
details = subset[
["Model", "Model URL", "Model Type", "Params", "Release"]
].drop_duplicates("Model")
if "RTLLM" in subset["Benchmark"].unique():
pivot_df = (
subset.pivot_table(
index="Model", columns="Metric", values="Score", aggfunc=custom_agg_s2r
)
.reset_index()
.round(2)
)
else:
pivot_df = (
subset.pivot_table(
index="Model", columns="Metric", values="Score", aggfunc=custom_agg_cc
)
.reset_index()
.round(2)
)
pivot_df = pd.merge(pivot_df, details, on="Model", how="left")
pivot_df["Model"] = pivot_df.apply(
lambda row: model_hyperlink(row["Model URL"], row["Model"], row["Release"]),
axis=1,
)
pivot_df["Type"] = pivot_df["Model Type"].map(lambda x: type_emoji.get(x, ""))
pivot_df["Post-Synthesis Quality"] = (
pivot_df[["Power", "Performance", "Area"]].mean(axis=1).round(2)
)
pivot_df.rename(
columns={
"Params": "Parameters (B)",
"Exact Matching (EM)": "EM",
"Syntax (STX)": "Syntax",
"Functionality (FNC)": "Functionality",
"Synthesis (SYN)": "Synthesis",
"Post-Synthesis Quality": "Post-Synthesis",
},
inplace=True,
)
columns_order = [
"Type",
"Model",
"Parameters (B)",
"Syntax",
"Functionality",
"Synthesis",
"Post-Synthesis",
]
pivot_df = pivot_df[[col for col in columns_order if col in pivot_df.columns]]
pivot_df = pivot_df.sort_values(by="Functionality", ascending=False).reset_index(
drop=True
)
return pivot_df