Fixing source/category

app.py

@@ -69,11 +69,13 @@ def init_leaderboard(dataframe):
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        search_columns=[AutoEvalColumn.model.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+          
+            ColumnFilter(AutoEvalColumn.model_source.name, type="checkboxgroup", label="Model Source"),
+            ColumnFilter(AutoEvalColumn.model_category.name, type="checkboxgroup", label="Model Category"),
+            #ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
             ColumnFilter(
                 AutoEvalColumn.params.name,
                 type="slider",
@@ -81,12 +83,12 @@ def init_leaderboard(dataframe):
                 max=150,
                 label="Select the number of parameters (B)",
             ),
-            ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
-            ),
+            #ColumnFilter(
+            #    AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+            #),
         ],
         bool_checkboxgroup_label="Hide models",
-        interactive=False,
+        interactive=True,
     )
 
 

results/Qwen/Qwen2.5-0.5B-Instruct_results_2025-04-21 16:50:28.595317.json

@@ -1,14 +1,8 @@
 {
   "results": {
-    "average": {
-      "score": 3.2
-    },
-    "speed": {
-      "spq": 0.18969469807280515
-    },
-    "contamination": {
-      "score": 0
-    },
+    "average_score":  6.0,
+    "speed": 5,
+    "contamination_score":  0,
     "execution_time": 88.587424,
     "errors": [],
     "scores_by_category": [
@@ -34,6 +28,8 @@
   "config": {
     "model": "Qwen/Qwen2.5-0.5B-Instruct",
     "model_sha": "7ae557604adf67be50417f59c2c2f167def9a775",
+    "model_source": "Hugging Face",
+    "model_category": "Nano",
     "submitted_time": "2025-04-21T14:43:01Z",
     "likes": 310,
     "params": 0.494,

results/openai-community/gpt2_results_2025-04-21 16:59:47.547731.json

@@ -1,14 +1,8 @@
 {
   "results": {
-    "average": {
-      "score": 1.0
-    },
-    "speed": {
-      "spq": 1.1064065631691649
-    },
-    "contamination": {
-      "score": 0
-    },
+    "average_score":  1.0,
+    "speed": 1.1064065631691649,
+    "contamination_score":  0,
     "execution_time": 516.691865,
     "errors": [],
     "scores_by_category": [
@@ -34,6 +28,8 @@
   "config": {
     "model": "openai-community/gpt2",
     "model_sha": "607a30d783dfa663caf39e06633721c8d4cfcd7e",
+    "model_source": "Hugging Face",
+    "model_category": "Nano",
     "submitted_time": "2025-04-21T14:50:23Z",
     "likes": 2679,
     "params": 0.137,

src/about.py

@@ -2,18 +2,16 @@ from dataclasses import dataclass
 from enum import Enum
 
 @dataclass
-class Task:
-    benchmark: str
+class EvalDimension:
     metric: str
     col_name: str
 
 
 # Select your tasks here
 # ---------------------------------------------------
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
-    task0 = Task("speed", "spq", "Speed")
-    task1 = Task("contamination", "score", "Contamination")
+class EvalDimensions(Enum):
+    d0 = EvalDimension("speed",  "Speed")
+    d1 = EvalDimension("contamination_score",  "Contamination Score")
 
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks
+from src.about import EvalDimensions
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -23,14 +23,19 @@ class ColumnContent:
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
+auto_eval_column_dict.append(["model_source", ColumnContent, ColumnContent("Source", "str", True, False)])
+auto_eval_column_dict.append(["model_category", ColumnContent, ColumnContent("Category", "str", True, False)])
+
+
+#auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
 #Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+for eval_dim in EvalDimensions:
+    auto_eval_column_dict.append([eval_dim.name, ColumnContent, ColumnContent(eval_dim.value.col_name, "number", True)])
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
+
+#auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 #auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
 #auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
 #auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
@@ -40,6 +45,7 @@ auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️"
 #auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
 #auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
 
+
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
@@ -108,5 +114,5 @@ COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
+BENCHMARK_COLS = [t.value.col_name for t in EvalDimensions]
 

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, Tasks#, ModelType, Precision, WeightType
+from src.display.utils import AutoEvalColumn, EvalDimensions#, ModelType, Precision, WeightType
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -24,7 +24,8 @@ class EvalResult:
     results: dict
     #precision: Precision = Precision.Unknown
     #model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    model_source: str = "" # HF, private, ...
+    model_source: str = "" # HF, API, ...
+    model_category: str = "" #Nano, Small, Medium, Large
     #weight_type: WeightType = WeightType.Original # Original or Adapter
     #architecture: str = "Unknown" 
     license: str = "?"
@@ -86,16 +87,17 @@ class EvalResult:
             org=org,
             model=model,
             model_source=config.get("model_source", ""),
+            model_category=config.get("model_category", ""),
             results=results,
             #precision=precision,  
-            revision= config.get("model_sha", ""),
+            #revision= config.get("model_sha", ""),
             still_on_hub=still_on_hub,
             #architecture=architecture
         )
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+        request_file = get_request_file_for_model(requests_path, self.full_model) #, self.precision.value.name
         try:
             with open(request_file, "r") as f:
                 request = json.load(f)
@@ -107,34 +109,35 @@ class EvalResult:
             self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
         except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+            print(f"Could not find request file for {self.org}/{self.model}") # with precision {self.precision.value.name}
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = self.results["average"]
+        average_score = self.results["average_score"]
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             #AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model_source.name: self.model_source.value.name,
+            AutoEvalColumn.model_source.name: self.model_source,
+            AutoEvalColumn.model_category.name: self.model_category,
             #AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
             #AutoEvalColumn.weight_type.name: self.weight_type.value.name,
             #AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
             #AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.average.name: average_score,
             AutoEvalColumn.license.name: self.license,
             AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
-            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+            #AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+        for eval_dim in EvalDimensions:
+            data_dict[eval_dim.value.col_name] = self.results[eval_dim.value.metric]
 
         return data_dict
 
 
-def get_request_file_for_model(requests_path, model_name, precision):
+def get_request_file_for_model(requests_path, model_name): #,precision
     """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
     request_files = os.path.join(
         requests_path,