tabs/tool_accuracy.py

import pandas as pd
import gradio as gr
import matplotlib.pyplot as plt
import seaborn as sns
from typing import Tuple
import plotly.express as px
import numpy as np

VOLUME_FACTOR_REGULARIZATION = 0.25
UNSCALED_WEIGHTED_ACCURACY_INTERVAL = (-0.5, 80.5)
SCALED_WEIGHTED_ACCURACY_INTERVAL = (0, 1)

# tools palette as dictionary
tools_palette = {
    "prediction-request-reasoning": "darkorchid",
    "claude-prediction-offline": "rebeccapurple",
    "prediction-request-reasoning-claude": "slateblue",
    "prediction-request-rag-claude": "steelblue",
    "prediction-online": "darkcyan",
    "prediction-offline": "mediumaquamarine",
    "claude-prediction-online": "mediumseagreen",
    "prediction-online-sme": "yellowgreen",
    "prediction-url-cot-claude": "gold",
    "prediction-offline-sme": "orange",
    "prediction-request-rag": "chocolate",
}

HEIGHT = 400
WIDTH = 1100


def scale_value(
    value: float,
    min_max_bounds: Tuple[float, float],
    scale_bounds: Tuple[float, float] = (0, 1),
) -> float:
    """Perform min-max scaling on a value."""
    min_, max_ = min_max_bounds
    current_range = max_ - min_
    # normalize between 0-1
    std = (value - min_) / current_range
    # scale between min_bound and max_bound
    min_bound, max_bound = scale_bounds
    target_range = max_bound - min_bound
    return std * target_range + min_bound


def get_weighted_accuracy(row, global_requests: int):
    """Function to compute the weighted accuracy of a tool"""
    return scale_value(
        (
            row["tool_accuracy"]
            + (row["total_requests"] / global_requests) * VOLUME_FACTOR_REGULARIZATION
        ),
        UNSCALED_WEIGHTED_ACCURACY_INTERVAL,
        SCALED_WEIGHTED_ACCURACY_INTERVAL,
    )


def compute_weighted_accuracy(tools_accuracy: pd.DataFrame):
    global_requests = tools_accuracy.total_requests.sum()
    tools_accuracy["weighted_accuracy"] = tools_accuracy.apply(
        lambda x: get_weighted_accuracy(x, global_requests), axis=1
    )
    return tools_accuracy


def plot_tools_accuracy_rotated_graph(tools_accuracy_info: pd.DataFrame):
    tools_accuracy_info = tools_accuracy_info.sort_values(
        by="tool_accuracy", ascending=False
    )
    fig = px.bar(
        tools_accuracy_info,
        x="tool",
        y="tool_accuracy",
        color="tool",
        color_discrete_map=tools_palette,
    )
    fig.update_layout(
        xaxis_title="Tool",
        yaxis_title="Mech tool_accuracy (%)",
    )
    fig.update_layout(width=WIDTH, height=HEIGHT)
    # fig.update_xaxes(tickangle=45)
    fig.update_xaxes(showticklabels=False)
    return gr.Plot(
        value=fig,
    )


def plot_tools_weighted_accuracy_graph(tools_accuracy_info: pd.DataFrame):
    tools_accuracy_info = tools_accuracy_info.sort_values(
        by="weighted_accuracy", ascending=False
    )
    # Create the Seaborn bar plot
    # sns.set_theme(palette="viridis")
    plt.figure(figsize=(25, 10))
    plot = sns.barplot(
        tools_accuracy_info,
        x="weighted_accuracy",
        y="tool",
        hue="tool",
        dodge=False,
        palette=tools_palette,
    )
    plt.xlabel("Weighted accuracy metric", fontsize=20)
    plt.ylabel("tool", fontsize=20)
    plt.tick_params(axis="y", labelsize=12)
    return gr.Plot(value=plot.get_figure())


def plot_tools_weighted_accuracy_rotated_graph(
    tools_accuracy_info: pd.DataFrame,
) -> gr.Plot:
    tools_accuracy_info = tools_accuracy_info.sort_values(
        by="weighted_accuracy", ascending=False
    )
    fig = px.bar(
        tools_accuracy_info,
        x="tool",
        y="weighted_accuracy",
        color="tool",
        color_discrete_map=tools_palette,
    )
    fig.update_layout(
        xaxis_title="Tool",
        yaxis_title="Weighted accuracy metric",
    )
    fig.update_layout(width=WIDTH, height=HEIGHT)
    # fig.update_xaxes(tickangle=45)
    fig.update_xaxes(showticklabels=False)
    return gr.Plot(
        value=fig,
    )


def plot_mech_requests_topthree_tools(
    daily_mech_requests_by_pearl_agents: pd.DataFrame,
    tools_accuracy_info: pd.DataFrame,
    top: int,
):
    """Function to plot the percentage of mech requests from the top three tools only for pearl agents"""
    # Get the top three tools
    top_tools = tools_accuracy_info.sort_values(
        by="tool_accuracy", ascending=False
    ).head(top)
    top_tools = top_tools.tool.tolist()

    # Filter the daily mech requests for the top three tools

    # Get the daily total of mech requests no matter the tool
    total_daily_mech_requests = (
        daily_mech_requests_by_pearl_agents.groupby(["request_date"])
        .agg({"total_mech_requests": "sum"})
        .reset_index()
    )
    print("total_daily_mech_requests", total_daily_mech_requests.head())
    total_daily_mech_requests.rename(
        columns={"total_mech_requests": "total_daily_mech_requests"},
        inplace=True,
    )
    # Merge the total daily mech requests with the daily mech requests
    daily_mech_requests_local_copy = pd.merge(
        daily_mech_requests_by_pearl_agents,
        total_daily_mech_requests,
        on="request_date",
        how="left",
    )
    # Compute the percentage of mech requests for each tool
    daily_mech_requests_local_copy["percentage"] = (
        daily_mech_requests_local_copy["total_mech_requests"]
        / daily_mech_requests_local_copy["total_daily_mech_requests"]
    ) * 100

    daily_mech_requests_local_copy = daily_mech_requests_local_copy[
        daily_mech_requests_local_copy.tool.isin(top_tools)
    ]

    # Create a pivot table to get the total mech requests per tool
    pivoted = daily_mech_requests_local_copy.pivot(
        index="request_date", columns="tool", values="percentage"
    )
    # Sort the columns for each row independently
    sorted_values = np.sort(pivoted.values, axis=1)[
        :, ::-1
    ]  # sort and reverse (descending)
    sorted_columns = np.argsort(pivoted.values, axis=1)[:, ::-1]  # get sorting indices

    sorted_df = pd.DataFrame(
        sorted_values,
        index=pivoted.index,
        columns=[
            pivoted.columns[i] for i in sorted_columns[0]
        ],  # use first row's order
    )

    sorted_long = sorted_df.reset_index().melt(
        id_vars=["request_date"], var_name="tool", value_name="percentage"
    )

    fig = px.bar(
        sorted_long,
        x="request_date",
        y="percentage",
        color="tool",
        color_discrete_map=tools_palette,
    )
    fig.update_layout(
        xaxis_title="Day of the request",
        yaxis_title="Percentage of Total daily mech requests",
        legend=dict(yanchor="top", y=0.5),
    )
    fig.update_layout(width=WIDTH, height=HEIGHT)
    return gr.Plot(value=fig)