app.py

import gradio as gr
import pandas as pd
import seaborn as sns
import duckdb
import logging


from scripts.metrics import (
    compute_weekly_metrics_by_market_creator,
    compute_daily_metrics_by_market_creator,
    compute_winning_metrics_by_trader,
)
from scripts.retention_metrics import (
    prepare_retention_dataset,
    calculate_wow_retention_by_type,
    calculate_cohort_retention,
)
from tabs.trader_plots import (
    plot_trader_metrics_by_market_creator,
    default_trader_metric,
    trader_metric_choices,
    get_metrics_text,
    plot_winning_metric_per_trader,
    get_interpretation_text,
    plot_total_bet_amount,
    plot_active_traders,
)
from tabs.daily_graphs import (
    get_current_week_data,
    plot_daily_metrics,
    trader_daily_metric_choices,
    default_daily_metric,
)
from scripts.utils import get_traders_family
from tabs.market_plots import (
    plot_kl_div_per_market,
    plot_total_bet_amount_per_trader_per_market,
)
from tabs.retention_plots import (
    plot_wow_retention_by_type,
    plot_cohort_retention_heatmap,
)


def get_logger():
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    # stream handler and formatter
    stream_handler = logging.StreamHandler()
    stream_handler.setLevel(logging.DEBUG)
    formatter = logging.Formatter(
        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
    )
    stream_handler.setFormatter(formatter)
    logger.addHandler(stream_handler)
    return logger


logger = get_logger()


def get_all_data():
    """
    Get parquet files from weekly stats and new generated
    """
    logger.info("Getting traders data")
    con = duckdb.connect(":memory:")
    # Query to fetch data from all_trades_profitability.parquet
    query1 = f"""
    SELECT *
    FROM read_parquet('./data/all_trades_profitability.parquet')
    """
    df1 = con.execute(query1).fetchdf()
    logger.info("Got all data from all_trades_profitability.parquet")

    # Query to fetch data from closed_markets_div.parquet
    query2 = f"""
    SELECT *
    FROM read_parquet('./data/closed_markets_div.parquet')
    """
    df2 = con.execute(query2).fetchdf()
    logger.info("Got all data from closed_markets_div.parquet")

    # Query to fetch daily live data
    query3 = f"""
    SELECT *
    FROM read_parquet('./data/daily_info.parquet')
    """
    df3 = con.execute(query3).fetchdf()

    # Query to fetch daily live data of unknown daily traders
    query4 = f"""
    SELECT *
    FROM read_parquet('./data/unknown_traders.parquet')
    """
    df4 = con.execute(query4).fetchdf()

    # Query to fetch retention activity data
    query5 = f"""
    SELECT *
    FROM read_parquet('./data/retention_activity.parquet')
    """
    df5 = con.execute(query5).fetchdf()
    con.close()
    return df1, df2, df3, df4, df5


def prepare_data():

    all_trades, closed_markets, daily_info, unknown_traders, retention_df = (
        get_all_data()
    )

    all_trades["creation_date"] = all_trades["creation_timestamp"].dt.date

    # nr-trades variable
    volume_trades_per_trader_and_market = (
        all_trades.groupby(["trader_address", "title"])["roi"]
        .count()
        .reset_index(name="nr_trades_per_market")
    )

    traders_data = pd.merge(
        all_trades, volume_trades_per_trader_and_market, on=["trader_address", "title"]
    )
    daily_info["creation_date"] = daily_info["creation_timestamp"].dt.date
    unknown_traders["creation_date"] = unknown_traders["creation_timestamp"].dt.date
    # adding the trader family column
    traders_data["trader_family"] = traders_data.apply(
        lambda x: get_traders_family(x), axis=1
    )
    # print(traders_data.head())

    traders_data = traders_data.sort_values(by="creation_timestamp", ascending=True)
    unknown_traders = unknown_traders.sort_values(
        by="creation_timestamp", ascending=True
    )
    traders_data["month_year_week"] = (
        traders_data["creation_timestamp"]
        .dt.to_period("W")
        .dt.start_time.dt.strftime("%b-%d-%Y")
    )
    unknown_traders["month_year_week"] = (
        unknown_traders["creation_timestamp"]
        .dt.to_period("W")
        .dt.start_time.dt.strftime("%b-%d-%Y")
    )
    closed_markets["month_year_week"] = (
        closed_markets["opening_datetime"]
        .dt.to_period("W")
        .dt.start_time.dt.strftime("%b-%d-%Y")
    )
    return traders_data, closed_markets, daily_info, unknown_traders, retention_df


traders_data, closed_markets, daily_info, unknown_traders, raw_retention_df = (
    prepare_data()
)
retention_df = prepare_retention_dataset(
    retention_df=raw_retention_df, unknown_df=unknown_traders
)
print("max date of retention df")
print(max(retention_df.creation_timestamp))

demo = gr.Blocks()
# get weekly metrics by market creator: qs, pearl or all.
weekly_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(
    traders_data
)
weekly_o_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(
    traders_data, trader_filter="Olas"
)
weekly_non_olas_metrics_by_market_creator = compute_weekly_metrics_by_market_creator(
    traders_data, trader_filter="non_Olas"
)
weekly_unknown_trader_metrics_by_market_creator = None
if len(unknown_traders) > 0:
    weekly_unknown_trader_metrics_by_market_creator = (
        compute_weekly_metrics_by_market_creator(
            unknown_traders, trader_filter=None, unknown_trader=True
        )
    )

weekly_winning_metrics = compute_winning_metrics_by_trader(traders_data=traders_data)
weekly_non_olas_winning_metrics = compute_winning_metrics_by_trader(
    traders_data=traders_data, trader_filter="non_Olas"
)

with demo:
    gr.HTML("<h1>Traders monitoring dashboard </h1>")
    gr.Markdown("This app shows the weekly performance of the traders in Olas Predict.")

    with gr.Tabs():
        with gr.TabItem("🔥 Weekly metrics"):
            with gr.Row():
                gr.Markdown("# Weekly metrics of all traders")
            with gr.Row():
                trader_details_selector = gr.Dropdown(
                    label="Select a weekly trader metric",
                    choices=trader_metric_choices,
                    value=default_trader_metric,
                )

            with gr.Row():
                with gr.Column(scale=3):
                    trader_markets_plot = plot_trader_metrics_by_market_creator(
                        metric_name=default_trader_metric,
                        traders_df=weekly_metrics_by_market_creator,
                    )
                with gr.Column(scale=1):
                    trade_details_text = get_metrics_text(trader_type=None)

            def update_trader_details(trader_detail):
                return plot_trader_metrics_by_market_creator(
                    metric_name=trader_detail,
                    traders_df=weekly_metrics_by_market_creator,
                )

            trader_details_selector.change(
                update_trader_details,
                inputs=trader_details_selector,
                outputs=trader_markets_plot,
            )

            with gr.Row():
                gr.Markdown("# Weekly metrics of 🌊 Olas traders")
            with gr.Row():
                trader_o_details_selector = gr.Dropdown(
                    label="Select a weekly trader metric",
                    choices=trader_metric_choices,
                    value=default_trader_metric,
                )

            with gr.Row():
                with gr.Column(scale=3):
                    o_trader_markets_plot = plot_trader_metrics_by_market_creator(
                        metric_name=default_trader_metric,
                        traders_df=weekly_o_metrics_by_market_creator,
                    )
                with gr.Column(scale=1):
                    trade_details_text = get_metrics_text(trader_type="Olas")

            def update_a_trader_details(trader_detail):
                return plot_trader_metrics_by_market_creator(
                    metric_name=trader_detail,
                    traders_df=weekly_o_metrics_by_market_creator,
                )

            trader_o_details_selector.change(
                update_a_trader_details,
                inputs=trader_o_details_selector,
                outputs=o_trader_markets_plot,
            )

            # Non-Olas traders graph
            with gr.Row():
                gr.Markdown("# Weekly metrics of Non-Olas traders")
            with gr.Row():
                trader_no_details_selector = gr.Dropdown(
                    label="Select a weekly trader metric",
                    choices=trader_metric_choices,
                    value=default_trader_metric,
                )

            with gr.Row():
                with gr.Column(scale=3):
                    trader_no_markets_plot = plot_trader_metrics_by_market_creator(
                        metric_name=default_trader_metric,
                        traders_df=weekly_non_olas_metrics_by_market_creator,
                    )
                with gr.Column(scale=1):
                    trade_details_text = get_metrics_text(trader_type="non_Olas")

            def update_no_trader_details(trader_detail):
                return plot_trader_metrics_by_market_creator(
                    metric_name=trader_detail,
                    traders_df=weekly_non_olas_metrics_by_market_creator,
                )

            trader_no_details_selector.change(
                update_no_trader_details,
                inputs=trader_no_details_selector,
                outputs=trader_no_markets_plot,
            )
            # Unknown traders graph
            if weekly_unknown_trader_metrics_by_market_creator is not None:
                with gr.Row():
                    gr.Markdown("# Weekly metrics of Unclassified traders")
                with gr.Row():
                    trader_u_details_selector = gr.Dropdown(
                        label="Select a weekly trader metric",
                        choices=trader_metric_choices,
                        value=default_trader_metric,
                    )

                with gr.Row():
                    with gr.Column(scale=3):
                        trader_u_markets_plot = plot_trader_metrics_by_market_creator(
                            metric_name=default_trader_metric,
                            traders_df=weekly_unknown_trader_metrics_by_market_creator,
                        )
                    with gr.Column(scale=1):
                        trade_details_text = get_metrics_text(
                            trader_type="unclassified"
                        )

                def update_u_trader_details(trader_detail):
                    return plot_trader_metrics_by_market_creator(
                        metric_name=trader_detail,
                        traders_df=weekly_unknown_trader_metrics_by_market_creator,
                    )

                trader_u_details_selector.change(
                    update_u_trader_details,
                    inputs=trader_u_details_selector,
                    outputs=trader_u_markets_plot,
                )
        with gr.TabItem("📅 Daily metrics"):
            live_trades_current_week = get_current_week_data(trades_df=daily_info)
            if len(live_trades_current_week) > 0:
                live_metrics_by_market_creator = (
                    compute_daily_metrics_by_market_creator(
                        live_trades_current_week, trader_filter=None, live_metrics=True
                    )
                )
            else:
                live_metrics_by_market_creator = pd.DataFrame()
            with gr.Row():
                gr.Markdown("# Daily live metrics for all trades")
            with gr.Row():
                trade_live_details_selector = gr.Dropdown(
                    label="Select a daily live metric",
                    choices=trader_daily_metric_choices,
                    value=default_daily_metric,
                )

            with gr.Row():
                with gr.Column(scale=3):
                    trade_live_details_plot = plot_daily_metrics(
                        metric_name=default_daily_metric,
                        trades_df=live_metrics_by_market_creator,
                    )
                with gr.Column(scale=1):
                    trade_details_text = get_metrics_text(daily=True)

            def update_trade_live_details(trade_detail, trade_live_details_plot):
                new_a_plot = plot_daily_metrics(
                    metric_name=trade_detail, trades_df=live_metrics_by_market_creator
                )
                return new_a_plot

            trade_live_details_selector.change(
                update_trade_live_details,
                inputs=[trade_live_details_selector, trade_live_details_plot],
                outputs=[trade_live_details_plot],
            )
            # Olas traders
            with gr.Row():
                gr.Markdown("# Daily live metrics for 🌊 Olas traders")
            with gr.Row():
                o_trader_live_details_selector = gr.Dropdown(
                    label="Select a daily live metric",
                    choices=trader_daily_metric_choices,
                    value=default_daily_metric,
                )

            with gr.Row():
                with gr.Column(scale=3):
                    o_trader_live_details_plot = plot_daily_metrics(
                        metric_name=default_daily_metric,
                        trades_df=live_metrics_by_market_creator,
                        trader_filter="Olas",
                    )
                with gr.Column(scale=1):
                    trade_details_text = get_metrics_text(daily=True)

            def update_a_trader_live_details(trade_detail, a_trader_live_details_plot):
                o_trader_plot = plot_daily_metrics(
                    metric_name=trade_detail,
                    trades_df=live_metrics_by_market_creator,
                    trader_filter="Olas",
                )
                return o_trader_plot

            o_trader_live_details_selector.change(
                update_a_trader_live_details,
                inputs=[o_trader_live_details_selector, o_trader_live_details_plot],
                outputs=[o_trader_live_details_plot],
            )
            with gr.Row():
                gr.Markdown("# Daily live metrics for Non-Olas traders")
            with gr.Row():
                no_trader_live_details_selector = gr.Dropdown(
                    label="Select a daily live metric",
                    choices=trader_daily_metric_choices,
                    value=default_daily_metric,
                )

            with gr.Row():
                with gr.Column(scale=3):
                    no_trader_live_details_plot = plot_daily_metrics(
                        metric_name=default_daily_metric,
                        trades_df=live_metrics_by_market_creator,
                        trader_filter="non_Olas",
                    )
                with gr.Column(scale=1):
                    trade_details_text = get_metrics_text(daily=True)

            def update_na_trader_live_details(
                trade_detail, no_trader_live_details_plot
            ):
                no_trader_plot = plot_daily_metrics(
                    metric_name=trade_detail,
                    trades_df=live_metrics_by_market_creator,
                    trader_filter="non_Olas",
                )
                return no_trader_plot

            no_trader_live_details_selector.change(
                update_na_trader_live_details,
                inputs=[no_trader_live_details_selector, no_trader_live_details_plot],
                outputs=[no_trader_live_details_plot],
            )
        with gr.TabItem("🪝 Retention metrics (WIP)"):
            with gr.Row():
                gr.Markdown("# Wow retention by trader type")
            with gr.Row():
                gr.Markdown(
                    """
                    Activity based on mech interactions for Olas and non_Olas traders and based on trading acitivity for the unclassified ones.
                    - Olas trader: agent using Mech, with a service ID and the corresponding safe in the registry
                    - Non-Olas trader: agent using Mech, with no service ID
                    - Unclassified trader: agent (safe/EOAs) not using Mechs
                    """
                )

            with gr.Row():
                with gr.Column(scale=1):
                    gr.Markdown("## Wow retention in Pearl markets")
                    wow_retention = calculate_wow_retention_by_type(
                        retention_df, market_creator="pearl"
                    )
                    wow_retention_plot = plot_wow_retention_by_type(
                        wow_retention=wow_retention
                    )
                with gr.Column(scale=1):
                    gr.Markdown("## Wow retention in Quickstart markets")
                    wow_retention = calculate_wow_retention_by_type(
                        retention_df, market_creator="quickstart"
                    )
                    wow_retention_plot = plot_wow_retention_by_type(
                        wow_retention=wow_retention
                    )

            with gr.Row():
                gr.Markdown("# Cohort retention in pearl traders")
            with gr.Row():
                gr.Markdown(
                    "The Cohort groups are organized by cohort weeks. A trader is part of a cohort group/week where it was detected the FIRST activity ever of that trader."
                )
            with gr.Row():
                gr.Markdown(
                    """
                    Week 0 for a cohort group is the same cohort week of the FIRST detected activity ever of that trader. 
                    Only two values are possible for this Week 0:

                    1. 100% if the cohort size is > 0, meaning all traders active that first cohort week
                    2. 0% if the cohort size = 0, meaning no totally new traders started activity that cohort week.
                    """
                )
            with gr.Row():
                with gr.Column(scale=1):
                    gr.Markdown("## Cohort retention of 🌊 Olas traders")
                    cohort_retention_olas_pearl = calculate_cohort_retention(
                        df=retention_df, market_creator="pearl", trader_type="Olas"
                    )
                    cohort_retention_plot1 = plot_cohort_retention_heatmap(
                        retention_matrix=cohort_retention_olas_pearl, cmap="Purples"
                    )
                with gr.Column(scale=1):
                    gr.Markdown("## Cohort retention of Non-Olas traders")
                    # non_Olas
                    cohort_retention_non_olas_pearl = calculate_cohort_retention(
                        df=retention_df, market_creator="pearl", trader_type="non_Olas"
                    )
                    cohort_retention_plot2 = plot_cohort_retention_heatmap(
                        retention_matrix=cohort_retention_non_olas_pearl,
                        cmap=sns.color_palette("light:goldenrod", as_cmap=True),
                    )
            with gr.Row():
                with gr.Column(scale=1):
                    gr.Markdown("## Cohort retention of unclassified traders")
                    cohort_retention_unclassified_pearl = calculate_cohort_retention(
                        df=retention_df,
                        market_creator="pearl",
                        trader_type="unclassified",
                    )
                    cohort_retention_plot3 = plot_cohort_retention_heatmap(
                        retention_matrix=cohort_retention_unclassified_pearl,
                        cmap="Greens",
                    )
                with gr.Column(scale=1):
                    print("Adding explanatory text")
            with gr.Row():
                gr.Markdown("# Cohort retention in quickstart traders")
            with gr.Row():
                with gr.Column(scale=1):
                    gr.Markdown("## Cohort retention of 🌊 Olas traders")
                    cohort_retention_olas_qs = calculate_cohort_retention(
                        df=retention_df, market_creator="quickstart", trader_type="Olas"
                    )
                    cohort_retention_plot4 = plot_cohort_retention_heatmap(
                        retention_matrix=cohort_retention_olas_qs,
                        cmap="Purples",
                    )
                with gr.Column(scale=1):
                    gr.Markdown("## Cohort retention of Non-Olas traders")
                    # non_Olas
                    cohort_retention_non_olas_qs = calculate_cohort_retention(
                        df=retention_df,
                        market_creator="quickstart",
                        trader_type="non_Olas",
                    )
                    cohort_retention_plot5 = plot_cohort_retention_heatmap(
                        retention_matrix=cohort_retention_non_olas_qs,
                        cmap=sns.color_palette("light:goldenrod", as_cmap=True),
                    )
            with gr.Row():
                with gr.Column(scale=1):
                    gr.Markdown("## Cohort retention of unclassified traders")
                    cohort_retention_unclassified_qs = calculate_cohort_retention(
                        df=retention_df,
                        market_creator="quickstart",
                        trader_type="unclassified",
                    )
                    cohort_retention_plot6 = plot_cohort_retention_heatmap(
                        retention_matrix=cohort_retention_unclassified_qs,
                        cmap="Greens",
                    )
                with gr.Column(scale=1):
                    print("Adding explanatory text")
        with gr.TabItem("⚙️ Active traders"):
            with gr.Row():
                gr.Markdown("# Active traders for all markets by trader categories")
            with gr.Row():
                active_traders_plot = plot_active_traders(traders_data, unknown_traders)

            with gr.Row():
                gr.Markdown("# Active traders for Pearl markets by trader categories")
            with gr.Row():
                active_traders_plot = plot_active_traders(
                    traders_data, unknown_traders, market_creator="pearl"
                )

            with gr.Row():
                gr.Markdown(
                    "# Active traders for Quickstart markets by trader categories"
                )
            with gr.Row():
                active_traders_plot = plot_active_traders(
                    traders_data, unknown_traders, market_creator="quickstart"
                )

        with gr.TabItem("📉 Markets Kullback–Leibler divergence"):
            with gr.Row():
                gr.Markdown(
                    "# Weekly Market Prediction Accuracy for Closed Markets (Kullback-Leibler Divergence)"
                )
            with gr.Row():
                gr.Markdown(
                    "Aka, how much off is the market prediction’s accuracy from the real outcome of the event. Values capped at 20 for market outcomes completely opposite to the real outcome."
                )
            with gr.Row():
                trade_details_text = get_metrics_text()
            with gr.Row():
                with gr.Column(scale=3):
                    kl_div_plot = plot_kl_div_per_market(closed_markets=closed_markets)
                with gr.Column(scale=1):
                    interpretation = get_interpretation_text()

        with gr.TabItem("💰 Money invested per trader type"):
            with gr.Row():
                gr.Markdown("# Weekly total bet amount per trader type for all markets")
            with gr.Row():
                total_bet_amount = plot_total_bet_amount(
                    traders_data, market_filter="all"
                )

            with gr.Row():
                gr.Markdown(
                    "# Weekly total bet amount per trader type for Pearl markets"
                )
            with gr.Row():
                o_trader_total_bet_amount = plot_total_bet_amount(
                    traders_data, market_filter="pearl"
                )

            with gr.Row():
                gr.Markdown(
                    "# Weekly total bet amount per trader type for Quickstart markets"
                )
            with gr.Row():
                no_trader_total_bet_amount = plot_total_bet_amount(
                    traders_data, market_filter="quickstart"
                )

        with gr.TabItem("💰 Money invested per market"):
            with gr.Row():
                gr.Markdown("# Weekly bet amounts per market for all traders")
            with gr.Row():
                bet_amounts = plot_total_bet_amount_per_trader_per_market(traders_data)

            with gr.Row():
                gr.Markdown("# Weekly bet amounts per market for 🌊 Olas traders")
            with gr.Row():
                o_trader_bet_amounts = plot_total_bet_amount_per_trader_per_market(
                    traders_data, trader_filter="Olas"
                )

            with gr.Row():
                gr.Markdown("# Weekly bet amounts per market for Non-Olas traders")
            with gr.Row():
                no_trader_bet_amounts = plot_total_bet_amount_per_trader_per_market(
                    traders_data, trader_filter="non_Olas"
                )

        with gr.TabItem("🎖️Weekly winning trades % per trader"):
            with gr.Row():
                gr.Markdown("# Weekly winning trades percentage from all traders")
            with gr.Row():
                metrics_text = get_metrics_text()
            with gr.Row():
                winning_metric = plot_winning_metric_per_trader(weekly_winning_metrics)

            with gr.Row():
                gr.Markdown("# Weekly winning trades percentage from 🌊 Olas traders")
            with gr.Row():
                metrics_text = get_metrics_text()
            with gr.Row():
                winning_metric = plot_winning_metric_per_trader(weekly_winning_metrics)

            # non_Olasic traders
            with gr.Row():
                gr.Markdown("# Weekly winning trades percentage from Non-Olas traders")
            with gr.Row():
                metrics_text = get_metrics_text()
            with gr.Row():
                winning_metric = plot_winning_metric_per_trader(
                    weekly_non_olas_winning_metrics
                )

demo.queue(default_concurrency_limit=40).launch()