Add theoretical performance of a model that scores the highest on every individual task

app.py

@@ -17,6 +17,32 @@ import plotly.express as px
 import plotly.graph_objects as go
 
 
+def mean_of_max_per_field(df):
+    """
+    Calcola il massimo per ciascun campo e poi la media dei massimi.
+
+    Args:
+        df (pd.DataFrame): DataFrame con colonne TE, SA, HS, AT, WIC, FAQ, LS, SU, NER, REL
+
+    Returns:
+        float: media dei valori massimi dei campi
+    """
+    fields = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
+
+    # Controlla che tutte le colonne esistano nel DataFrame
+    missing = [f for f in fields if f not in df.columns]
+    if missing:
+        raise ValueError(f"Le seguenti colonne mancano nel DataFrame: {missing}")
+
+    # Calcola il massimo per ciascun campo
+    max_values = df[fields].max()
+
+    # Calcola la media dei massimi
+    mean_max = max_values.mean()
+
+    return mean_max
+
+
 def line_chart(dataframe):
     # Separiamo i dati in base a IS_FS
     df_true = dataframe[dataframe['IS_FS'] == True]
@@ -44,7 +70,7 @@ def line_chart(dataframe):
         x=x_true,
         y=y_true,
         mode='markers',  # solo marker, niente testo
-        name='5-Few-Shot',
+        name='5-Shot',
         marker=dict(color='red', size=10),
         hovertemplate='<b>%{customdata}</b><br>#Params: %{x}<br>Performance: %{y}<extra></extra>',
         customdata=labels_true  # tutte le informazioni sul hover
@@ -78,6 +104,8 @@ def line_chart(dataframe):
 
 
 
+
+
 # Define task metadata (icons, names, descriptions)
 TASK_METADATA_MULTIPLECHOICE = {
     "TE": {"icon": "📊", "name": "Textual Entailment", "tooltip": ""},
@@ -109,6 +137,8 @@ def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
 
+    #print("????????????????????????????????", mean_of_max_per_field(dataframe))
+
     sorted_dataframe = dataframe.sort_values(by="Avg. Comb. Perf. ⬆️", ascending=False)
 
     sorted_dataframe = sorted_dataframe.reset_index(drop=True)
@@ -168,10 +198,10 @@ def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
         search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
         hide_columns=hidden_columns or [c.name for c in field_list if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)"),
+            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Shot Learning (FS)"),
             #ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)",
             #             default=[["0️⃣", "0️⃣"]]),
-        #   ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=150, label="Select the number of parameters (B)"),
+            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max = 100, default = [0,100], label="Select the number of parameters (B)"),
         ],
         #filter_columns=[
         #    ColumnFilter("IS_FS", type="checkbox", default=False, label="5-Few-Shot")
@@ -195,13 +225,46 @@ def update_task_leaderboard(dataframe, default_selection=None, hidden_columns=No
     sorted_dataframe = sorted_dataframe.reset_index(drop=True)
     sorted_dataframe["rank"] = sorted_dataframe.index + 1
 
-    # aggiungi la corona accanto al nome del modello se il rank è 1
-    sorted_dataframe["Model"] = sorted_dataframe.apply(
-        lambda row: f"{row['Model']} 🥇" if row["rank"] == 1 else
-        (f"{row['Model']} 🥈" if row["rank"] == 2 else
-         (f"{row['Model']} 🥉" if row["rank"] == 3 else row["Model"])),
-        axis=1
-    )
+    # Flag per sapere se la medaglia è già stata assegnata per categoria e tipo
+    large_medal_fs_assigned = False
+    medium_medal_fs_assigned = False
+    small_medal_fs_assigned = False
+
+    large_medal_0shot_assigned = False
+    medium_medal_0shot_assigned = False
+    small_medal_0shot_assigned = False
+
+    # Lista temporanea per salvare i nuovi valori della colonna Model
+    new_model_column = []
+
+    for _, row in sorted_dataframe.iterrows():
+        if row['IS_FS']:  # 5-Few-Shot
+            if row["#Params (B)"] > 30 and not large_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 7️⃣0️⃣🅱️🏆")
+                large_medal_fs_assigned = True
+            elif 10 < row["#Params (B)"] <= 30 and not medium_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 3️⃣0️⃣🅱️🏆")
+                medium_medal_fs_assigned = True
+            elif row["#Params (B)"] <= 10 and not small_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 1️⃣0️⃣🅱️🏆")
+                small_medal_fs_assigned = True
+            else:
+                new_model_column.append(row["Model"])
+        else:  # 0-Shot
+            if row["#Params (B)"] > 30 and not large_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 7️⃣0️⃣🅱️🎖️")
+                large_medal_0shot_assigned = True
+            elif 10 < row["#Params (B)"] <= 30 and not medium_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 3️⃣0️⃣🅱️🎖️")
+                medium_medal_0shot_assigned = True
+            elif row["#Params (B)"] <= 10 and not small_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 1️⃣0️⃣🅱️🎖️")
+                small_medal_0shot_assigned = True
+            else:
+                new_model_column.append(row["Model"])
+
+    # Aggiorna la colonna Model
+    sorted_dataframe["Model"] = new_model_column
 
     pd.set_option('display.max_colwidth', None)
     #print("========================", dataframe['Model'])
@@ -222,7 +285,9 @@ def update_task_leaderboard(dataframe, default_selection=None, hidden_columns=No
         search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
         hide_columns=hidden_columns or [c.name for c in field_list if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Few-Shot Learning (FS)"),
+            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Shot Learning (FS)"),
+            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=100, default=[0, 100],
+                         label="Select the number of parameters (B)"),
         ],
         bool_checkboxgroup_label="Evaluation Mode",
         interactive=False
@@ -273,6 +338,8 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
 finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 #print(LEADERBOARD_DF.columns.tolist())
 
+theoretical_max_combined_perf = mean_of_max_per_field(LEADERBOARD_DF)
+
 # Prepare the main interface
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -306,6 +373,22 @@ with demo:
                 hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['rank', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]]
             )
 
+            gr.HTML(
+                f"""
+                        <div style="
+                            border: 2px solid #1f77b4;
+                            border-radius: 10px;
+                            padding: 10px;
+                            background-color: #f0f8ff;
+                            font-weight: bold;
+                            font-size: 14px;
+                            display: inline-block;
+                        ">
+                            Theoretical performance of a model that scores the highest on every individual task: <span style="color:#d62728; font-size:18px;">{theoretical_max_combined_perf:.2f}</span>
+                        </div>
+                        """
+            )
+
         with gr.TabItem("📈 Charts"):
             #gr.Plot(value=line_chart(LEADERBOARD_DF), label="Andamento di esempio")
             #gr.Plot(value=line_chart_interactive_test(), label="Andamento interattivo")
@@ -319,6 +402,7 @@ with demo:
         with gr.TabItem("║", interactive=False):
             gr.Markdown("", elem_classes="markdown-text")
 
+
         # Task-specific leaderboards
         for task, metadata in TASK_METADATA_MULTIPLECHOICE.items():
 

src/display/utils.py

@@ -48,6 +48,7 @@ auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B
 auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
 auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+#auto_eval_column_dict.append(["submitted_time", ColumnContent, ColumnContent("Submitted time", "date", False)])
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)

src/leaderboard/read_evals.py

@@ -7,6 +7,7 @@ from dataclasses import dataclass, field
 import dateutil
 import numpy as np
 from typing import Dict, Union
+from datetime import datetime
 
 #from get_model_info import num_params
 from src.display.formatting import make_clickable_model
@@ -23,6 +24,7 @@ class EvalResult:
     org: str 
     model: str
     revision: str # commit hash, "" if main
+    #submitted_time: datetime
     results: Dict[str, Union[float, int]]  # float o int
     average_CPS: float
     is_5fewshot: bool
@@ -119,7 +121,8 @@ class EvalResult:
             still_on_hub=still_on_hub,
             architecture=architecture,
             num_params=num_params,
-            rank = 0
+            rank = 0,
+            #submitted_time=config.get("submitted_time", ""),
         )
 
     '''