Spaces:
Running
Running
File size: 3,957 Bytes
e09bf50 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 |
from gradio_client import Client
from datasets import load_dataset
import requests
import json
import time
import random
import os
import re
from datetime import datetime
# Fetch the official Wordle guess list from GitHub
WORD_LIST_URL = "https://raw.githubusercontent.com/tabatkins/wordle-list/main/words"
def load_word_list():
resp = requests.get(WORD_LIST_URL)
resp.raise_for_status()
words = [w.strip().lower() for w in resp.text.splitlines()]
return [w for w in words if len(w) == 5 and w.isalpha()]
WORD_LIST = load_word_list()
def get_last_assistant_content(resp):
if isinstance(resp, tuple): resp = resp[0]
if not isinstance(resp, list): return ""
for turn in reversed(resp):
if turn.get("role") != "assistant": continue
if turn.get("content"): return turn["content"]
fr = turn.get("function_response", {})
out = fr.get("result", {}).get("output")
if out: return out
cont = turn.get("content")
if isinstance(cont, dict):
parts = cont.get("parts", [])
if parts and parts[0].get("text"): return parts[0]["text"]
return ""
def compute_feedback(guess, solution):
feedback = ["B"] * 5
sol = list(solution)
for i, g in enumerate(guess):
if g == sol[i]: feedback[i], sol[i] = "G", None
for i, g in enumerate(guess):
if feedback[i] == "B" and g in sol:
feedback[i] = "Y"
sol[sol.index(g)] = None
return "".join(feedback)
def sanitize_guess(raw):
raw = raw.lower()
m = re.search(r"\b[a-z]{5}\b", raw)
if m: return m.group(0)
cleaned = re.sub(r"[^a-z]", "", raw)
return cleaned[-5:]
def benchmark_wordle(num_games=10, max_guesses=6):
client = Client("http://127.0.0.1:7860/")
os.makedirs("results", exist_ok=True)
out_path = os.path.join("results", f"wordle_benchmark_{datetime.now():%Y%m%d_%H%M%S}.jsonl")
results = []
for gi in range(num_games):
solution = random.choice(WORD_LIST)
print(f"Game {gi+1}/{num_games}, solution: {solution}")
guesses = []
attempts = 0
start_time = time.time()
while attempts < max_guesses:
history = "\n".join(f"Guess: {g}, Feedback: {f}" for g, f in guesses)
prompt = (
f"Wordle game. Guess the 5-letter word.\n" +
(history + "\n" if history else "") +
"Respond with a single 5-letter guess and with ONLY YOUR GUESS. NO FILLER OR PUNCTUATION."
"\n" + "Use the feedback format: G=green, Y=yellow, B=black.\n" +
f"(Green: letter in correct position, Yellow: letter in wrong position, Black: letter not in word)\n" +
f"Use tools and agents to help you guess the word.\n"
)
resp = client.predict(messages=[{"role": "user", "content": prompt}], api_name="/run")
raw = get_last_assistant_content(resp)
guess = sanitize_guess(raw)
# If guess invalid, retry without counting
if len(guess) != 5 or guess not in WORD_LIST:
print(f"Warning: '{guess}' invalid; retrying without using a turn.")
continue
feedback = compute_feedback(guess, solution)
guesses.append((guess, feedback))
attempts += 1
print(f"Attempt {attempts}: {guess} -> {feedback}")
if feedback == "GGGGG":
break
results.append({
"solution": solution,
"guesses": guesses,
"solved": bool(guesses and guesses[-1][1] == "GGGGG"),
"turns": len(guesses),
"time": time.time() - start_time
})
with open(out_path, "a") as f:
f.write(json.dumps(results[-1]) + "\n")
print(f"Benchmark complete, results saved to {out_path}")
return results
if __name__ == "__main__":
benchmark_wordle(num_games=1)
|