Top size + Top by Skill

app.py

@@ -59,22 +59,86 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
+def hide_skill_columns(dataframe, exceptions=[]):
+    return dataframe[[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default or c.name in exceptions]]
+
+
+def perform_cell_formatting(dataframe):
+    return dataframe.style.format({'Contamination Score': "{:.2f}",'Benchmark Score': "{:.2f}",'Speed (words/sec)': "{:.2f}"}).apply(
+        lambda rows: [
+            "background-color: red;" if (value >0) else "background-color: green;" for value in rows
+        ],
+        subset=["Contamination Score"],
+    )
+
 def init_leaderboard(dataframe):
-    #if dataframe is None or dataframe.empty:
-        #raise ValueError("Leaderboard DataFrame is empty or None.")
-     dataframe = dataframe[[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default]]
+
+    dataframe = hide_skill_columns(dataframe)
+
      
-     styler = dataframe.style.format({'Contamination Score': "{:.2f}",'Benchmark Score': "{:.2f}",'Speed (words/sec)': "{:.2f}"}).apply(
-            lambda rows: [
-                "background-color: red;" if (value >0) else "background-color: green;" for value in rows
-            ],
-            subset=["Contamination Score"],
+    styler = perform_cell_formatting(dataframe)
 
+    return gr.Dataframe(
+            value=styler,
+            datatype="markdown",
+            wrap=True,
+            show_fullscreen_button=False,
+            interactive=False,
+            column_widths=[30,50,50,150,60,60,60],
+            max_height=420,
+            elem_classes="leaderboard_col_style"
         )
 
 
-        
-     return gr.Dataframe(
+def init_skill_leaderboard(dataframe):
+
+    
+
+    ## create selector for model skills, based on the selector filter the dataframe
+    skills = ['MMLU', 'General Knowledge', 'Reasoning & Math', 'Translation (incl Dialects)', 'Trust & Safety', 'Writing (incl Dialects)', 'RAG QA', 'Reading Comprehension', 'Arabic Language & Grammar', 'Diacritization', 'Dialect Detection', 'Sentiment Analysis', 'Summarization', 'Instruction Following', 'Transliteration', 'Paraphrasing', 'Entity Extraction', 'Long Context', 'Coding', 'Hallucination', 'Function Calling', 'Structuring']
+
+    skills_dropdown = gr.Dropdown(choices=skills, label="Select Skill", value=skills[0])
+
+    def filter_dataframe(skill):
+        filtered_df = dataframe.sort_values(by=[skill], ascending=False).reset_index(drop=True)
+        filtered_df = hide_skill_columns(filtered_df, exceptions=[skill])
+        filtered_df["Rank"] = range(1, len(filtered_df) + 1)
+        styler = perform_cell_formatting(filtered_df)
+        return gr.Dataframe(
+            value=styler,
+            datatype="markdown",
+            wrap=True,
+            show_fullscreen_button=False,
+            interactive=False,
+            column_widths=[30,50,50,150,60,60,60,80],
+            max_height=420,
+            elem_classes="leaderboard_col_style"
+        )
+
+    leaderboard_by_skill = filter_dataframe(skills[0])
+    skills_dropdown.change(filter_dataframe, inputs=skills_dropdown, outputs=leaderboard_by_skill)
+    return leaderboard_by_skill
+
+
+
+def init_size_leaderboard(dataframe):
+
+    dataframe = hide_skill_columns(dataframe)
+
+    size_keys = ["Large","Medium","Small","Nano"]
+
+    size_names = ["Large (More than 30B Parameter)","Medium (~30B)","Small (~10B)","Nano (~3B)"]
+    sizes_dropdown = gr.Dropdown(choices=size_names, label="Select Model Size", value=size_names[0])
+
+    def filter_dataframe(size_name):
+        ##map size name to size key
+        size_name_mapped_to_key = size_keys[size_names.index(size_name)]
+        ##slice array from 0 to index of size
+        size_list = size_keys[size_keys.index(size_name_mapped_to_key):]
+        filtered_df = dataframe[dataframe["Category"].isin(size_list)].reset_index(drop=True)
+        filtered_df["Rank"] = range(1, len(filtered_df) + 1)
+        styler = perform_cell_formatting(filtered_df)
+        return gr.Dataframe(
             value=styler,
             datatype="markdown",
             wrap=True,
@@ -85,6 +149,9 @@ def init_leaderboard(dataframe):
             elem_classes="leaderboard_col_style"
         )
 
+    leaderboard_by_skill = filter_dataframe(size_names[0])
+    sizes_dropdown.change(filter_dataframe, inputs=sizes_dropdown, outputs=leaderboard_by_skill)
+    return leaderboard_by_skill
 
 demo = gr.Blocks(css=custom_css)
 with demo:
@@ -92,13 +159,20 @@ with demo:
     gr.HTML(INTRODUCTION_TEXT, elem_classes="abl_desc_text")
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
-        with gr.TabItem("🏅 Arabic LLM Leaderboard", elem_id="llm-benchmark-tab-table", id=0):
+        with gr.TabItem("🏅 Leaderboard - Top Models", elem_id="llm-benchmark-tab-table", id=0):
             leaderboard = init_leaderboard(LEADERBOARD_DF)
+        
+        with gr.TabItem("🏅 Top by Size", elem_id="llm-benchmark-tab-size", id=1):
+            leaderboard = init_size_leaderboard(LEADERBOARD_DF)
+
+        with gr.TabItem("🏅 Top by Skill", elem_id="llm-benchmark-tab-skills", id=2):
+            leaderboard = init_skill_leaderboard(LEADERBOARD_DF)
+
 
-        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-about", id=4):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
+        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-submit", id=5):
             with gr.Column():
                 with gr.Row():
                     gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")

src/about.py

@@ -12,6 +12,39 @@ class EvalDimension:
 class EvalDimensions(Enum):
     d0 = EvalDimension("speed",  "Speed (words/sec)")
     d1 = EvalDimension("contamination_score",  "Contamination Score")
+    d2 = EvalDimension("paraphrasing",  "Paraphrasing")
+    d3 = EvalDimension("sentiment analysis",  "Sentiment Analysis")
+    d4 = EvalDimension("coding",  "Coding")
+    d5 = EvalDimension("function calling",  "Function Calling")
+    d6 = EvalDimension("rag qa",  "RAG QA")
+    d7 = EvalDimension("reading comprehension",  "Reading Comprehension")
+    d8 = EvalDimension("entity extraction",  "Entity Extraction")
+    d9 = EvalDimension("summarization",  "Summarization")
+    d10 = EvalDimension("long context",  "Long Context")
+    d11 = EvalDimension("mmlu",  "MMLU")
+    d12 = EvalDimension("arabic language & grammar",  "Arabic Language & Grammar")
+    d13 = EvalDimension("general knowledge",  "General Knowledge")
+    d14 = EvalDimension("translation (incl dialects)",  "Translation (incl Dialects)")
+    d15 = EvalDimension("trust & safety","Trust & Safety")
+    d16 = EvalDimension("writing (incl dialects)",  "Writing (incl Dialects)")
+    d17 = EvalDimension("dialect detection",  "Dialect Detection")
+    d18 = EvalDimension("reasoning & math",  "Reasoning & Math")
+    d19 = EvalDimension("diacritization",  "Diacritization")
+    d20 = EvalDimension("instruction following",  "Instruction Following")
+    d21 = EvalDimension("transliteration",  "Transliteration")
+    d22 = EvalDimension("structuring",  "Structuring")
+    d23 = EvalDimension("hallucination",  "Hallucination")
+
+
+
+
+
+    
+
+
+
+
+
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
@@ -23,10 +56,10 @@ TITLE = """<div ><img class='abl_header_image' src='https://huggingface.co/space
 
 # What does your leaderboard evaluate?
 INTRODUCTION_TEXT = """
-<h1 style='width: 100%;text-align: center;' id="space-title">Arabic Board Leaderboard (ABL) - The first comprehensive Leaderboard for Arabic LLMs</h1>
-ABL is the official Leaderboard of <a href='https://huggingface.co/datasets/silma-ai/arabic-broad-benchmark' target='_blank'>Arabic Board Benchmark (ABB)</a>. 
+<h1 style='width: 100%;text-align: center;' id="space-title">Arabic Broad Leaderboard (ABL) - The first comprehensive Leaderboard for Arabic LLMs</h1>
+ABL is the official Leaderboard of <a href='https://huggingface.co/datasets/silma-ai/arabic-broad-benchmark' target='_blank'>Arabic Broad Benchmark (ABB)</a>. 
 With advanced features and innovative visualizations, we provide the community with a comprehensive view of the capabilities of Arabic models, showcasing their speed, diverse skills while also defending against benchmarking contamination. 
-The benchmark consists of <b>450</b> high quality questions sampled from <b>63</b> Arabic benchmarking datasets, evaluating <b>22 categories and skills</b>.
+The benchmark consists of <b>450 high quality human-validated questions</b> sampled from <b>63 Arabic benchmarking datasets</b>, evaluating <b>22 categories and skills</b>.
 Find more details in the about Tab. 
 
 

src/display/utils.py

@@ -30,20 +30,23 @@ auto_eval_column_dict.append(["model_category", ColumnContent, ColumnContent("Ca
 
 
 #auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model Name", "markdown", True, never_hidden=True)])
 #Scores
 auto_eval_column_dict.append(["average_score", ColumnContent, ColumnContent("Benchmark Score", "number", True)])
 for eval_dim in EvalDimensions:
-    auto_eval_column_dict.append([eval_dim.name, ColumnContent, ColumnContent(eval_dim.value.col_name, "number", True)])
+    if eval_dim.value.metric in ["speed", "contamination_score"]:
+         auto_eval_column_dict.append([eval_dim.name, ColumnContent, ColumnContent(eval_dim.value.col_name, "number", True)])
+    else:
+        auto_eval_column_dict.append([eval_dim.name, ColumnContent, ColumnContent(eval_dim.value.col_name, "number", False)])
 # Model information
 
 #auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 #auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 #auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
 #auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Popularity (Likes)", "number", False)])
+#auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("License", "str", False)])
+#auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+#auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Popularity (Likes)", "number", False)])
 #auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
 #auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 

src/leaderboard/read_evals.py

@@ -28,9 +28,9 @@ class EvalResult:
     model_category: str = "" #Nano, Small, Medium, Large
     #weight_type: WeightType = WeightType.Original # Original or Adapter
     #architecture: str = "Unknown" 
-    license: str = "?"
-    likes: int = 0
-    num_params: int = 0
+    #license: str = "?"
+    #likes: int = 0
+    #num_params: int = 0
     date: str = "" # submission date of request file
     still_on_hub: bool = False
 
@@ -81,6 +81,16 @@ class EvalResult:
         results["speed"] = results_obj.get("speed")
         results["contamination_score"] = results_obj.get("contamination_score")
 
+        scores_by_category = results_obj.get("scores_by_category")
+
+        for category_obj in scores_by_category:
+            category = category_obj["category"]
+            average_score = category_obj["average_score"]
+            results[category.lower()] = average_score
+
+            
+        
+
         return self(
             eval_name=result_key,
             full_model=full_model,
@@ -88,9 +98,9 @@ class EvalResult:
             model=model,
             model_source=config.get("model_source", ""),
             model_category=config.get("model_category", ""),
-            num_params=config.get("params", 0),
-            license=config.get("license", "?"),
-            likes=config.get("likes", -1),
+            #num_params=config.get("params", 0),
+            #license=config.get("license", "?"),
+            #likes=config.get("likes", -1),
             results=results,
             #precision=precision,  
             #revision= config.get("model_sha", ""),
@@ -128,15 +138,19 @@ class EvalResult:
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             #AutoEvalColumn.revision.name: self.revision,
             AutoEvalColumn.average_score.name: average_score,
-            AutoEvalColumn.license.name: self.license,
-            AutoEvalColumn.likes.name: self.likes,
-            AutoEvalColumn.params.name: self.num_params,
+            #AutoEvalColumn.license.name: self.license,
+            #AutoEvalColumn.likes.name: self.likes,
+            #AutoEvalColumn.params.name: self.num_params,
             #AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
         for eval_dim in EvalDimensions:
             dimension_name = eval_dim.value.col_name
-            dimension_value = self.results[eval_dim.value.metric]
+            try:
+                dimension_value = self.results[eval_dim.value.metric]
+            except KeyError:
+                dimension_value = 0
+
             if dimension_name == "Contamination Score":
                 dimension_value = 0 if dimension_value < 0 else round(dimension_value,2)
             

src/populate.py

@@ -24,6 +24,8 @@ def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchm
 
         df.insert(0, "Rank", range(1, len(df) + 1))
         df = df[cols].round(decimals=2)
+        print("###############\n\n\n\n\n\n###############")
+
         print(df)