Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -3,6 +3,7 @@ import pandas as pd
|
|
3 |
import os
|
4 |
import shutil
|
5 |
import json
|
|
|
6 |
|
7 |
# Function to load leaderboard data from a CSV file
|
8 |
def load_leaderboard_data(csv_file_path):
|
@@ -11,38 +12,36 @@ def load_leaderboard_data(csv_file_path):
|
|
11 |
return df
|
12 |
except Exception as e:
|
13 |
print(f"Error loading CSV file: {e}")
|
14 |
-
return pd.DataFrame()
|
15 |
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
try:
|
21 |
-
|
22 |
-
|
23 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
24 |
except Exception as e:
|
25 |
-
return
|
26 |
-
|
27 |
-
# Function to save the uploaded JSON file
|
28 |
-
def save_json_file(file):
|
29 |
-
if file is None:
|
30 |
-
return "No file uploaded."
|
31 |
-
|
32 |
-
# Define the directory to save uploaded files
|
33 |
-
save_dir = "uploaded_jsons"
|
34 |
-
os.makedirs(save_dir, exist_ok=True)
|
35 |
-
|
36 |
-
# Get the original filename
|
37 |
-
original_filename = os.path.basename(file.name)
|
38 |
-
|
39 |
-
# Define the path to save the file
|
40 |
-
save_path = os.path.join(save_dir, original_filename)
|
41 |
-
|
42 |
-
# Copy the uploaded file to the save directory
|
43 |
-
shutil.copy2(file.name, save_path)
|
44 |
-
|
45 |
-
return f"File saved to {save_path}"
|
46 |
|
47 |
# Load the leaderboard data
|
48 |
leaderboard1 = load_leaderboard_data("leaderboard_swe.csv")
|
@@ -51,6 +50,7 @@ leaderboard2 = load_leaderboard_data("leaderboard_gaia.csv")
|
|
51 |
# Create the Gradio interface
|
52 |
with gr.Blocks() as demo:
|
53 |
gr.Markdown("# 🏆 TRAIL: Trace Reasoning and Agentic Issue Localization Leaderboard")
|
|
|
54 |
with gr.Row():
|
55 |
with gr.Column():
|
56 |
gr.Markdown("## TRAIL-SWE Leaderboard")
|
@@ -59,250 +59,31 @@ with gr.Blocks() as demo:
|
|
59 |
gr.Markdown("## TRAIL-GAIA Leaderboard")
|
60 |
gr.Dataframe(leaderboard2)
|
61 |
|
62 |
-
with gr.
|
63 |
-
gr.Markdown("## Submit Your
|
64 |
-
file_input = gr.File(label="Upload JSON File", file_types=['.json'])
|
65 |
-
json_preview = gr.JSON(label="JSON Preview")
|
66 |
-
submit_button = gr.Button("Submit", interactive=True)
|
67 |
-
output = gr.Textbox(label="Status")
|
68 |
-
|
69 |
-
def handle_submission(file):
|
70 |
-
if file is None:
|
71 |
-
return None, "Please upload a JSON file."
|
72 |
-
try:
|
73 |
-
# Process and preview the JSON
|
74 |
-
with open(file.name, 'r') as f:
|
75 |
-
data = json.load(f)
|
76 |
-
# Save the file
|
77 |
-
save_result = save_json_file(file)
|
78 |
-
return data, save_result
|
79 |
-
except Exception as e:
|
80 |
-
return None, f"Error: {str(e)}"
|
81 |
-
|
82 |
-
submit_button.click(
|
83 |
-
fn=handle_submission,
|
84 |
-
inputs=[file_input],
|
85 |
-
outputs=[json_preview, output]
|
86 |
-
)
|
87 |
-
|
88 |
-
if __name__ == "__main__":
|
89 |
-
demo.launch()
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
"""
|
94 |
-
import gradio as gr
|
95 |
-
import pandas as pd
|
96 |
-
import os
|
97 |
-
import json
|
98 |
-
import uuid
|
99 |
-
import hashlib
|
100 |
-
from datetime import datetime
|
101 |
-
from huggingface_hub import HfApi, login, HfFolder
|
102 |
-
|
103 |
-
# Configuration
|
104 |
-
LEADERBOARD_CSV = "leaderboard.csv"
|
105 |
-
SUBMISSIONS_FOLDER = "submissions"
|
106 |
-
CONFIG_FILE = "config.json"
|
107 |
-
DEFAULT_COLUMNS = ["rank", "submission_name", "score", "user", "timestamp"]
|
108 |
-
VERIFY_USERS = False # Set to True to enable HF authentication
|
109 |
-
|
110 |
-
# Default configuration
|
111 |
-
DEFAULT_CONFIG = {
|
112 |
-
"title": "Hugging Face Competition Leaderboard",
|
113 |
-
"description": "Submit your results for the competition",
|
114 |
-
"metric_name": "Score",
|
115 |
-
"higher_is_better": True,
|
116 |
-
"max_submissions_per_user": 5,
|
117 |
-
"allow_submission_edits": True
|
118 |
-
}
|
119 |
-
|
120 |
-
# Ensure submissions folder exists
|
121 |
-
os.makedirs(SUBMISSIONS_FOLDER, exist_ok=True)
|
122 |
-
|
123 |
-
# Load or create config
|
124 |
-
if os.path.exists(CONFIG_FILE):
|
125 |
-
with open(CONFIG_FILE, "r") as f:
|
126 |
-
config = json.load(f)
|
127 |
-
else:
|
128 |
-
config = DEFAULT_CONFIG
|
129 |
-
with open(CONFIG_FILE, "w") as f:
|
130 |
-
json.dump(config, f, indent=2)
|
131 |
-
|
132 |
-
# Initialize leaderboard if it doesn't exist
|
133 |
-
if not os.path.exists(LEADERBOARD_CSV):
|
134 |
-
pd.DataFrame(columns=DEFAULT_COLUMNS).to_csv(LEADERBOARD_CSV, index=False)
|
135 |
-
|
136 |
-
def read_leaderboard():
|
137 |
-
#Read the current leaderboard
|
138 |
-
if os.path.exists(LEADERBOARD_CSV):
|
139 |
-
df = pd.read_csv(LEADERBOARD_CSV)
|
140 |
-
return df
|
141 |
-
return pd.DataFrame(columns=DEFAULT_COLUMNS)
|
142 |
-
|
143 |
-
def verify_user(username, token):
|
144 |
-
#Verify a user with their Hugging Face token
|
145 |
-
if not VERIFY_USERS:
|
146 |
-
return True
|
147 |
-
|
148 |
-
try:
|
149 |
-
api = HfApi(token=token)
|
150 |
-
user_info = api.whoami()
|
151 |
-
return user_info["name"] == username
|
152 |
-
except:
|
153 |
-
return False
|
154 |
-
|
155 |
-
def count_user_submissions(username):
|
156 |
-
#Count how many submissions a user already has
|
157 |
-
df = read_leaderboard()
|
158 |
-
return len(df[df["user"] == username])
|
159 |
-
|
160 |
-
def update_leaderboard():
|
161 |
-
#Update the leaderboard based on submissions
|
162 |
-
# Read all submissions
|
163 |
-
submissions = []
|
164 |
-
for filename in os.listdir(SUBMISSIONS_FOLDER):
|
165 |
-
if filename.endswith(".json"):
|
166 |
-
with open(os.path.join(SUBMISSIONS_FOLDER, filename), "r") as f:
|
167 |
-
try:
|
168 |
-
data = json.load(f)
|
169 |
-
submissions.append(data)
|
170 |
-
except json.JSONDecodeError:
|
171 |
-
print(f"Error decoding {filename}")
|
172 |
-
|
173 |
-
if not submissions:
|
174 |
-
return pd.DataFrame(columns=DEFAULT_COLUMNS)
|
175 |
-
|
176 |
-
# Create dataframe and sort by score
|
177 |
-
df = pd.DataFrame(submissions)
|
178 |
-
|
179 |
-
# Sort based on configuration (higher or lower is better)
|
180 |
-
ascending = not config.get("higher_is_better", True)
|
181 |
-
df = df.sort_values("score", ascending=ascending)
|
182 |
-
|
183 |
-
# Add rank
|
184 |
-
df["rank"] = range(1, len(df) + 1)
|
185 |
-
|
186 |
-
# Save updated leaderboard
|
187 |
-
df.to_csv(LEADERBOARD_CSV, index=False)
|
188 |
-
return df
|
189 |
-
|
190 |
-
def submit(submission_name, score, username, hf_token="", submission_details=None):
|
191 |
-
#Add a new submission to the leaderboard
|
192 |
-
if not submission_name or not username:
|
193 |
-
return "Submission name and username are required", None
|
194 |
-
|
195 |
-
try:
|
196 |
-
score = float(score)
|
197 |
-
except ValueError:
|
198 |
-
return "Score must be a valid number", None
|
199 |
-
|
200 |
-
# Verify user if enabled
|
201 |
-
if VERIFY_USERS and not verify_user(username, hf_token):
|
202 |
-
return "Invalid Hugging Face credentials", None
|
203 |
-
|
204 |
-
# Check submission limit
|
205 |
-
max_submissions = config.get("max_submissions_per_user", 5)
|
206 |
-
if count_user_submissions(username) >= max_submissions:
|
207 |
-
return f"You've reached the maximum of {max_submissions} submissions", None
|
208 |
-
|
209 |
-
# Create submission entry
|
210 |
-
submission_id = str(uuid.uuid4())[:8]
|
211 |
-
submission = {
|
212 |
-
"submission_id": submission_id,
|
213 |
-
"submission_name": submission_name,
|
214 |
-
"score": score,
|
215 |
-
"user": username,
|
216 |
-
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
217 |
-
}
|
218 |
-
|
219 |
-
# Add optional details
|
220 |
-
if submission_details:
|
221 |
-
submission["details"] = submission_details
|
222 |
-
|
223 |
-
# Save submission to file
|
224 |
-
filename = f"{username}_{submission_name.replace(' ', '_')}_{submission_id}.json"
|
225 |
-
with open(os.path.join(SUBMISSIONS_FOLDER, filename), "w") as f:
|
226 |
-
json.dump(submission, f)
|
227 |
-
|
228 |
-
# Update leaderboard
|
229 |
-
leaderboard = update_leaderboard()
|
230 |
-
return f"Submission '{submission_name}' added successfully!", leaderboard
|
231 |
-
|
232 |
-
def render_leaderboard():
|
233 |
-
#Display the current leaderboard
|
234 |
-
df = update_leaderboard()
|
235 |
-
if len(df) == 0:
|
236 |
-
return "No submissions yet."
|
237 |
-
|
238 |
-
# Format the dataframe for display
|
239 |
-
display_df = df[DEFAULT_COLUMNS].copy()
|
240 |
-
return display_df
|
241 |
-
|
242 |
-
# Create the Gradio interface
|
243 |
-
with gr.Blocks(title=config["title"]) as demo:
|
244 |
-
gr.Markdown(f"# {config['title']}")
|
245 |
-
gr.Markdown(f"{config['description']}")
|
246 |
-
|
247 |
-
with gr.Tab("Leaderboard"):
|
248 |
-
gr.Markdown("## Current Rankings")
|
249 |
-
metric_name = config.get("metric_name", "Score")
|
250 |
-
higher_better = "higher is better" if config.get("higher_is_better", True) else "lower is better"
|
251 |
-
gr.Markdown(f"*Ranked by {metric_name} ({higher_better})*")
|
252 |
-
|
253 |
-
leaderboard_output = gr.Dataframe(
|
254 |
-
headers=["Rank", "Submission", metric_name, "User", "Timestamp"],
|
255 |
-
datatype=["number", "str", "number", "str", "str"],
|
256 |
-
interactive=False
|
257 |
-
)
|
258 |
-
refresh_btn = gr.Button("Refresh Leaderboard")
|
259 |
-
refresh_btn.click(render_leaderboard, inputs=[], outputs=[leaderboard_output])
|
260 |
-
|
261 |
-
with gr.Tab("Submit"):
|
262 |
-
gr.Markdown("## Submit Your Results")
|
263 |
-
with gr.Row():
|
264 |
-
with gr.Column():
|
265 |
-
submission_name = gr.Textbox(label="Submission Name", placeholder="MyAwesomeModel v1.0")
|
266 |
-
score = gr.Number(label=metric_name, precision=4)
|
267 |
-
username = gr.Textbox(label="Username", placeholder="Your Hugging Face username")
|
268 |
-
|
269 |
-
# Only show token field if verification is enabled
|
270 |
-
if VERIFY_USERS:
|
271 |
-
hf_token = gr.Textbox(
|
272 |
-
label="Hugging Face Token",
|
273 |
-
placeholder="hf_...",
|
274 |
-
type="password"
|
275 |
-
)
|
276 |
-
else:
|
277 |
-
hf_token = gr.Textbox(visible=False)
|
278 |
-
|
279 |
-
submission_details = gr.Textbox(
|
280 |
-
label="Additional Details (optional)",
|
281 |
-
placeholder="Model details, training info, etc.",
|
282 |
-
lines=5
|
283 |
-
)
|
284 |
-
submit_btn = gr.Button("Submit to Leaderboard")
|
285 |
-
|
286 |
-
submit_output = gr.Markdown()
|
287 |
-
submission_leaderboard = gr.Dataframe(
|
288 |
-
headers=["Rank", "Submission", metric_name, "User", "Timestamp"],
|
289 |
-
datatype=["number", "str", "number", "str", "str"],
|
290 |
-
interactive=False
|
291 |
-
)
|
292 |
-
|
293 |
-
submit_btn.click(
|
294 |
-
submit,
|
295 |
-
inputs=[submission_name, score, username, hf_token, submission_details],
|
296 |
-
outputs=[submit_output, submission_leaderboard]
|
297 |
-
)
|
298 |
-
|
299 |
-
# Add admin tab if desired
|
300 |
-
with gr.Tab("About"):
|
301 |
-
gr.Markdown("## About This Leaderboard")
|
302 |
|
303 |
-
|
304 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
305 |
|
306 |
if __name__ == "__main__":
|
307 |
demo.launch()
|
308 |
-
"""
|
|
|
3 |
import os
|
4 |
import shutil
|
5 |
import json
|
6 |
+
from zipfile import ZipFile
|
7 |
|
8 |
# Function to load leaderboard data from a CSV file
|
9 |
def load_leaderboard_data(csv_file_path):
|
|
|
12 |
return df
|
13 |
except Exception as e:
|
14 |
print(f"Error loading CSV file: {e}")
|
15 |
+
return pd.DataFrame()
|
16 |
|
17 |
+
def save_zip_and_extract_json(zip_path):
|
18 |
+
if not zip_path:
|
19 |
+
return "Please upload a ZIP file."
|
20 |
+
|
21 |
try:
|
22 |
+
# 1) Determine Space name and persistent base dir
|
23 |
+
cwd = os.getcwd()
|
24 |
+
space_name = os.path.basename(cwd)
|
25 |
+
base_dir = os.path.join("data", space_name, "uploaded_jsons")
|
26 |
+
os.makedirs(base_dir, exist_ok=True)
|
27 |
+
|
28 |
+
# 2) Copy the zip into base_dir
|
29 |
+
zip_dest = os.path.join(base_dir, os.path.basename(zip_path))
|
30 |
+
shutil.copy(zip_path, zip_dest)
|
31 |
+
|
32 |
+
# 3) Extract only .json files
|
33 |
+
extracted = []
|
34 |
+
with ZipFile(zip_dest, 'r') as archive:
|
35 |
+
for member in archive.namelist():
|
36 |
+
if member.lower().endswith(".json"):
|
37 |
+
archive.extract(member, path=base_dir)
|
38 |
+
extracted.append(member)
|
39 |
+
|
40 |
+
if not extracted:
|
41 |
+
return "No JSON files found in the ZIP."
|
42 |
+
return f"Extracted JSON files:\n" + "\n".join(extracted)
|
43 |
except Exception as e:
|
44 |
+
return f"Error processing ZIP file: {str(e)}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
45 |
|
46 |
# Load the leaderboard data
|
47 |
leaderboard1 = load_leaderboard_data("leaderboard_swe.csv")
|
|
|
50 |
# Create the Gradio interface
|
51 |
with gr.Blocks() as demo:
|
52 |
gr.Markdown("# 🏆 TRAIL: Trace Reasoning and Agentic Issue Localization Leaderboard")
|
53 |
+
|
54 |
with gr.Row():
|
55 |
with gr.Column():
|
56 |
gr.Markdown("## TRAIL-SWE Leaderboard")
|
|
|
59 |
gr.Markdown("## TRAIL-GAIA Leaderboard")
|
60 |
gr.Dataframe(leaderboard2)
|
61 |
|
62 |
+
with gr.Row():
|
63 |
+
gr.Markdown("## Submit Your ZIP of JSONs")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
64 |
|
65 |
+
with gr.Row():
|
66 |
+
with gr.Column():
|
67 |
+
file_input = gr.File(
|
68 |
+
label="Upload ZIP File",
|
69 |
+
file_types=['.zip']
|
70 |
+
)
|
71 |
+
submit_button = gr.Button("Submit", interactive=True)
|
72 |
+
output = gr.Textbox(label="Status")
|
73 |
+
|
74 |
+
def process_upload(file):
|
75 |
+
if file is None:
|
76 |
+
return "Please upload a ZIP file."
|
77 |
+
try:
|
78 |
+
return save_zip_and_extract_json(file.name)
|
79 |
+
except Exception as e:
|
80 |
+
return f"Error: {str(e)}"
|
81 |
+
|
82 |
+
submit_button.click(
|
83 |
+
fn=process_upload,
|
84 |
+
inputs=file_input,
|
85 |
+
outputs=output
|
86 |
+
)
|
87 |
|
88 |
if __name__ == "__main__":
|
89 |
demo.launch()
|
|