import pandas as pd import gradio as gr import matplotlib.pyplot as plt import seaborn as sns from typing import Tuple import plotly.express as px import numpy as np VOLUME_FACTOR_REGULARIZATION = 0.25 UNSCALED_WEIGHTED_ACCURACY_INTERVAL = (-0.5, 80.5) SCALED_WEIGHTED_ACCURACY_INTERVAL = (0, 1) # tools palette as dictionary tools_palette = { "prediction-request-reasoning": "darkorchid", "claude-prediction-offline": "rebeccapurple", "prediction-request-reasoning-claude": "slateblue", "prediction-request-rag-claude": "steelblue", "prediction-online": "darkcyan", "prediction-offline": "mediumaquamarine", "claude-prediction-online": "mediumseagreen", "prediction-online-sme": "yellowgreen", "prediction-url-cot-claude": "gold", "prediction-offline-sme": "orange", "prediction-request-rag": "chocolate", } HEIGHT = 400 WIDTH = 1100 def scale_value( value: float, min_max_bounds: Tuple[float, float], scale_bounds: Tuple[float, float] = (0, 1), ) -> float: """Perform min-max scaling on a value.""" min_, max_ = min_max_bounds current_range = max_ - min_ # normalize between 0-1 std = (value - min_) / current_range # scale between min_bound and max_bound min_bound, max_bound = scale_bounds target_range = max_bound - min_bound return std * target_range + min_bound def get_weighted_accuracy(row, global_requests: int): """Function to compute the weighted accuracy of a tool""" return scale_value( ( row["tool_accuracy"] + (row["total_requests"] / global_requests) * VOLUME_FACTOR_REGULARIZATION ), UNSCALED_WEIGHTED_ACCURACY_INTERVAL, SCALED_WEIGHTED_ACCURACY_INTERVAL, ) def compute_weighted_accuracy(tools_accuracy: pd.DataFrame): global_requests = tools_accuracy.total_requests.sum() tools_accuracy["weighted_accuracy"] = tools_accuracy.apply( lambda x: get_weighted_accuracy(x, global_requests), axis=1 ) return tools_accuracy def plot_tools_accuracy_rotated_graph(tools_accuracy_info: pd.DataFrame): tools_accuracy_info = tools_accuracy_info.sort_values( by="tool_accuracy", ascending=False ) fig = px.bar( tools_accuracy_info, x="tool", y="tool_accuracy", color="tool", color_discrete_map=tools_palette, ) fig.update_layout( xaxis_title="Tool", yaxis_title="Mech tool_accuracy (%)", ) fig.update_layout(width=WIDTH, height=HEIGHT) # fig.update_xaxes(tickangle=45) fig.update_xaxes(showticklabels=False) return gr.Plot( value=fig, ) def plot_tools_weighted_accuracy_graph(tools_accuracy_info: pd.DataFrame): tools_accuracy_info = tools_accuracy_info.sort_values( by="weighted_accuracy", ascending=False ) # Create the Seaborn bar plot # sns.set_theme(palette="viridis") plt.figure(figsize=(25, 10)) plot = sns.barplot( tools_accuracy_info, x="weighted_accuracy", y="tool", hue="tool", dodge=False, palette=tools_palette, ) plt.xlabel("Weighted accuracy metric", fontsize=20) plt.ylabel("tool", fontsize=20) plt.tick_params(axis="y", labelsize=12) return gr.Plot(value=plot.get_figure()) def plot_tools_weighted_accuracy_rotated_graph( tools_accuracy_info: pd.DataFrame, ) -> gr.Plot: tools_accuracy_info = tools_accuracy_info.sort_values( by="weighted_accuracy", ascending=False ) fig = px.bar( tools_accuracy_info, x="tool", y="weighted_accuracy", color="tool", color_discrete_map=tools_palette, ) fig.update_layout( xaxis_title="Tool", yaxis_title="Weighted accuracy metric", ) fig.update_layout(width=WIDTH, height=HEIGHT) # fig.update_xaxes(tickangle=45) fig.update_xaxes(showticklabels=False) return gr.Plot( value=fig, ) def plot_mech_requests_topthree_tools( daily_mech_requests_by_pearl_agents: pd.DataFrame, tools_accuracy_info: pd.DataFrame, top: int, ): """Function to plot the percentage of mech requests from the top three tools only for pearl agents""" # Get the top three tools top_tools = tools_accuracy_info.sort_values( by="tool_accuracy", ascending=False ).head(top) top_tools = top_tools.tool.tolist() # Filter the daily mech requests for the top three tools # Get the daily total of mech requests no matter the tool total_daily_mech_requests = ( daily_mech_requests_by_pearl_agents.groupby(["request_date"]) .agg({"total_mech_requests": "sum"}) .reset_index() ) print("total_daily_mech_requests", total_daily_mech_requests.head()) total_daily_mech_requests.rename( columns={"total_mech_requests": "total_daily_mech_requests"}, inplace=True, ) # Merge the total daily mech requests with the daily mech requests daily_mech_requests_local_copy = pd.merge( daily_mech_requests_by_pearl_agents, total_daily_mech_requests, on="request_date", how="left", ) # Compute the percentage of mech requests for each tool daily_mech_requests_local_copy["percentage"] = ( daily_mech_requests_local_copy["total_mech_requests"] / daily_mech_requests_local_copy["total_daily_mech_requests"] ) * 100 daily_mech_requests_local_copy = daily_mech_requests_local_copy[ daily_mech_requests_local_copy.tool.isin(top_tools) ] # Create a pivot table to get the total mech requests per tool pivoted = daily_mech_requests_local_copy.pivot( index="request_date", columns="tool", values="percentage" ) # Sort the columns for each row independently sorted_values = np.sort(pivoted.values, axis=1)[ :, ::-1 ] # sort and reverse (descending) sorted_columns = np.argsort(pivoted.values, axis=1)[:, ::-1] # get sorting indices sorted_df = pd.DataFrame( sorted_values, index=pivoted.index, columns=[ pivoted.columns[i] for i in sorted_columns[0] ], # use first row's order ) sorted_long = sorted_df.reset_index().melt( id_vars=["request_date"], var_name="tool", value_name="percentage" ) fig = px.bar( sorted_long, x="request_date", y="percentage", color="tool", color_discrete_map=tools_palette, ) fig.update_layout( xaxis_title="Day of the request", yaxis_title="Percentage of Total daily mech requests", legend=dict(yanchor="top", y=0.5), ) fig.update_layout(width=WIDTH, height=HEIGHT) return gr.Plot(value=fig)