|
import pandas as pd |
|
import gradio as gr |
|
import matplotlib.pyplot as plt |
|
import seaborn as sns |
|
from typing import Tuple |
|
import plotly.express as px |
|
import numpy as np |
|
|
|
VOLUME_FACTOR_REGULARIZATION = 0.25 |
|
UNSCALED_WEIGHTED_ACCURACY_INTERVAL = (-0.5, 80.5) |
|
SCALED_WEIGHTED_ACCURACY_INTERVAL = (0, 1) |
|
|
|
|
|
tools_palette = { |
|
"prediction-request-reasoning": "darkorchid", |
|
"claude-prediction-offline": "rebeccapurple", |
|
"prediction-request-reasoning-claude": "slateblue", |
|
"prediction-request-rag-claude": "steelblue", |
|
"prediction-online": "darkcyan", |
|
"prediction-offline": "mediumaquamarine", |
|
"claude-prediction-online": "mediumseagreen", |
|
"prediction-online-sme": "yellowgreen", |
|
"prediction-url-cot-claude": "gold", |
|
"prediction-offline-sme": "orange", |
|
"prediction-request-rag": "chocolate", |
|
} |
|
|
|
HEIGHT = 400 |
|
WIDTH = 1100 |
|
|
|
|
|
def scale_value( |
|
value: float, |
|
min_max_bounds: Tuple[float, float], |
|
scale_bounds: Tuple[float, float] = (0, 1), |
|
) -> float: |
|
"""Perform min-max scaling on a value.""" |
|
min_, max_ = min_max_bounds |
|
current_range = max_ - min_ |
|
|
|
std = (value - min_) / current_range |
|
|
|
min_bound, max_bound = scale_bounds |
|
target_range = max_bound - min_bound |
|
return std * target_range + min_bound |
|
|
|
|
|
def get_weighted_accuracy(row, global_requests: int): |
|
"""Function to compute the weighted accuracy of a tool""" |
|
return scale_value( |
|
( |
|
row["tool_accuracy"] |
|
+ (row["total_requests"] / global_requests) * VOLUME_FACTOR_REGULARIZATION |
|
), |
|
UNSCALED_WEIGHTED_ACCURACY_INTERVAL, |
|
SCALED_WEIGHTED_ACCURACY_INTERVAL, |
|
) |
|
|
|
|
|
def compute_weighted_accuracy(tools_accuracy: pd.DataFrame): |
|
global_requests = tools_accuracy.total_requests.sum() |
|
tools_accuracy["weighted_accuracy"] = tools_accuracy.apply( |
|
lambda x: get_weighted_accuracy(x, global_requests), axis=1 |
|
) |
|
return tools_accuracy |
|
|
|
|
|
def plot_tools_accuracy_rotated_graph(tools_accuracy_info: pd.DataFrame): |
|
tools_accuracy_info = tools_accuracy_info.sort_values( |
|
by="tool_accuracy", ascending=False |
|
) |
|
fig = px.bar( |
|
tools_accuracy_info, |
|
x="tool", |
|
y="tool_accuracy", |
|
color="tool", |
|
color_discrete_map=tools_palette, |
|
) |
|
fig.update_layout( |
|
xaxis_title="Tool", |
|
yaxis_title="Mech tool_accuracy (%)", |
|
) |
|
fig.update_layout(width=WIDTH, height=HEIGHT) |
|
|
|
fig.update_xaxes(showticklabels=False) |
|
return gr.Plot( |
|
value=fig, |
|
) |
|
|
|
|
|
def plot_tools_weighted_accuracy_graph(tools_accuracy_info: pd.DataFrame): |
|
tools_accuracy_info = tools_accuracy_info.sort_values( |
|
by="weighted_accuracy", ascending=False |
|
) |
|
|
|
|
|
plt.figure(figsize=(25, 10)) |
|
plot = sns.barplot( |
|
tools_accuracy_info, |
|
x="weighted_accuracy", |
|
y="tool", |
|
hue="tool", |
|
dodge=False, |
|
palette=tools_palette, |
|
) |
|
plt.xlabel("Weighted accuracy metric", fontsize=20) |
|
plt.ylabel("tool", fontsize=20) |
|
plt.tick_params(axis="y", labelsize=12) |
|
return gr.Plot(value=plot.get_figure()) |
|
|
|
|
|
def plot_tools_weighted_accuracy_rotated_graph( |
|
tools_accuracy_info: pd.DataFrame, |
|
) -> gr.Plot: |
|
tools_accuracy_info = tools_accuracy_info.sort_values( |
|
by="weighted_accuracy", ascending=False |
|
) |
|
fig = px.bar( |
|
tools_accuracy_info, |
|
x="tool", |
|
y="weighted_accuracy", |
|
color="tool", |
|
color_discrete_map=tools_palette, |
|
) |
|
fig.update_layout( |
|
xaxis_title="Tool", |
|
yaxis_title="Weighted accuracy metric", |
|
) |
|
fig.update_layout(width=WIDTH, height=HEIGHT) |
|
|
|
fig.update_xaxes(showticklabels=False) |
|
return gr.Plot( |
|
value=fig, |
|
) |
|
|
|
|
|
def plot_mech_requests_topthree_tools( |
|
daily_mech_requests: pd.DataFrame, tools_accuracy_info: pd.DataFrame, top: int |
|
): |
|
"""Function to plot the percentage of mech requests from the top three tools""" |
|
|
|
top_tools = tools_accuracy_info.sort_values( |
|
by="tool_accuracy", ascending=False |
|
).head(top) |
|
top_tools = top_tools.tool.tolist() |
|
|
|
daily_mech_requests_local_copy = daily_mech_requests.copy() |
|
daily_mech_requests_local_copy = daily_mech_requests_local_copy[ |
|
daily_mech_requests_local_copy["market_creator"] == "all" |
|
] |
|
|
|
total_daily_mech_requests = ( |
|
daily_mech_requests_local_copy.groupby(["request_date"]) |
|
.agg({"total_mech_requests": "sum"}) |
|
.reset_index() |
|
) |
|
print("total_daily_mech_requests", total_daily_mech_requests.head()) |
|
total_daily_mech_requests.rename( |
|
columns={"total_mech_requests": "total_daily_mech_requests"}, |
|
inplace=True, |
|
) |
|
|
|
daily_mech_requests_local_copy = pd.merge( |
|
daily_mech_requests_local_copy, |
|
total_daily_mech_requests, |
|
on="request_date", |
|
how="left", |
|
) |
|
|
|
daily_mech_requests_local_copy["percentage"] = ( |
|
daily_mech_requests_local_copy["total_mech_requests"] |
|
/ daily_mech_requests_local_copy["total_daily_mech_requests"] |
|
) * 100 |
|
|
|
daily_mech_requests_local_copy = daily_mech_requests_local_copy[ |
|
daily_mech_requests_local_copy.tool.isin(top_tools) |
|
] |
|
|
|
|
|
daily_mech_requests_local_copy = daily_mech_requests_local_copy.drop( |
|
columns=["market_creator"] |
|
) |
|
|
|
|
|
pivoted = daily_mech_requests_local_copy.pivot( |
|
index="request_date", columns="tool", values="percentage" |
|
) |
|
|
|
sorted_values = np.sort(pivoted.values, axis=1)[ |
|
:, ::-1 |
|
] |
|
sorted_columns = np.argsort(pivoted.values, axis=1)[:, ::-1] |
|
|
|
sorted_df = pd.DataFrame( |
|
sorted_values, |
|
index=pivoted.index, |
|
columns=[ |
|
pivoted.columns[i] for i in sorted_columns[0] |
|
], |
|
) |
|
|
|
sorted_long = sorted_df.reset_index().melt( |
|
id_vars=["request_date"], var_name="tool", value_name="percentage" |
|
) |
|
|
|
fig = px.bar( |
|
sorted_long, |
|
x="request_date", |
|
y="percentage", |
|
color="tool", |
|
color_discrete_map=tools_palette, |
|
) |
|
fig.update_layout( |
|
xaxis_title="Day of the request", |
|
yaxis_title="Percentage of Total daily mech requests", |
|
legend=dict(yanchor="top", y=0.5), |
|
) |
|
fig.update_layout(width=WIDTH, height=HEIGHT) |
|
return gr.Plot(value=fig) |
|
|