Delete agent

agent/checklist.py

@@ -1,18 +0,0 @@
-from .mini_bench.agent import ChecklistGenerationAgent
-
-def generate_checklist(**data):
-    # data: 'intent', 'start_url', 'text_observation'
-    agent_config = {
-        'model_name': 'WPRM/qwen-3b-ar-reward-cot-mtl-checklist-enhanced',
-        'base_url': 'http://165.132.144.84:7701/v1',
-        'api_key': 'empty',
-        'temperature': 0.7,
-        'use_log_probs': True,
-        'use_checklist': True,
-        'use_multimodal': False,
-        'num_generate': 1,
-    }
-    checklist_generation_agent = ChecklistGenerationAgent(agent_config)
-    response_list, cost = checklist_generation_agent.generate_response(data, prompt_type='ours', constraint_str_list=["<think>", "</think>", "<answer>", "</answer>"])
-    response = response_list[0]
-    return response.split("<answer>")[-1].split("</answer>")[0].strip()

agent/mini_bench/agent.py

@@ -1,467 +0,0 @@
-from abc import ABC, abstractmethod
-import time
-import requests
-import json
-import math
-from langsmith import Client
-from langchain_openai import ChatOpenAI
-
-from .prompts import get_messages
-from .prompts.judge_prompt import (
-    JUDGE_OURS_BT_MODELING_PROMPT_TEMPLATE,
-    JUDGE_OURS_BT_MODELING_WO_CHECKLIST_PROMPT_TEMPLATE,
-    JUDGE_OURS_BT_MODELING_MULTIMODAL_PROMPT_TEMPLATE,
-    JUDGE_OURS_BT_MODELING_MULTIMODAL_WO_CHECKLIST_PROMPT_TEMPLATE
-)
-from .prompts.image_utils import image_to_base64_url
-
-MAX_RETRY = 3
-RETRY_SLEEP = 5
-MODEL_COST_MAPPING = {
-    "gpt-4o-mini": {
-        "input_token_cost": 0.15,
-        "output_token_cost": 0.6
-    },
-    "gpt-4o": {
-        "input_token_cost": 2.5,
-        "output_token_cost": 10
-    },
-}
-
-
-class Agent(ABC):
-    @abstractmethod
-    def generate_response(self, inputs: dict) -> str:
-        pass
-
-class BaseAgent(Agent):
-    def __init__(self, agent_config: dict):
-        self.agent_config = agent_config
-        self._setup()
-    
-    def _setup(self):
-        use_log_probs = self.agent_config.get("use_log_probs", False)
-        if use_log_probs:
-            self.llm = ChatOpenAI(
-                model=self.agent_config["model_name"], 
-                base_url=self.agent_config["base_url"], 
-                api_key=self.agent_config["api_key"], 
-                temperature=self.agent_config["temperature"],
-                timeout=300,
-                logprobs=True,
-                top_logprobs=10
-            )
-        else:
-            self.llm = ChatOpenAI(
-                model=self.agent_config["model_name"], 
-                base_url=self.agent_config["base_url"], 
-                api_key=self.agent_config["api_key"], 
-                temperature=self.agent_config["temperature"],
-                timeout=300
-            )
-        self.temperature = self.agent_config["temperature"]
-        self.num_generate = self.agent_config["num_generate"]
-        self.use_checklist = self.agent_config.get("use_checklist", False)
-        self.use_multimodal = self.agent_config.get("use_multimodal", False)
-
-        # setup cost
-        model_cost = MODEL_COST_MAPPING.get(self.agent_config["model_name"], None)
-        if model_cost and "api" in self.agent_config["base_url"]:
-            self.input_token_cost = model_cost["input_token_cost"]
-            self.output_token_cost = model_cost["output_token_cost"]
-        else:
-            self.input_token_cost = 0.0
-            self.output_token_cost = 0.0
-
-    def generate_with_retry(self, model_input, constraint_str_list: list = None):
-        total_input_tokens = 0
-        total_output_tokens = 0
-        if self.temperature == 0:
-            response = self.llm.invoke(model_input)
-            total_input_tokens += response.response_metadata["token_usage"]["prompt_tokens"]
-            total_output_tokens += response.response_metadata["token_usage"]["completion_tokens"]
-        else:
-            for i in range(MAX_RETRY):
-                try:
-                    response = self.llm.invoke(model_input)
-                    total_input_tokens += response.response_metadata["token_usage"]["prompt_tokens"]
-                    total_output_tokens += response.response_metadata["token_usage"]["completion_tokens"]
-                    if constraint_str_list:
-                        pass_constraint_num = 0
-                        for constraint_str in constraint_str_list:
-                            if constraint_str in response.content:
-                                pass_constraint_num += 1
-                        if pass_constraint_num == len(constraint_str_list):
-                            break
-                        else:
-                            print(f"Agent has fomat issue, retry... {i+1}/{MAX_RETRY}")
-                            print(response.content)
-                    else:
-                        break
-                except Exception as e:
-                    print(f"Agent returned an Error: {e}")
-                    response = None
-                    time.sleep(RETRY_SLEEP)
-        
-        cost = self.input_token_cost * total_input_tokens / 1000000 + self.output_token_cost * total_output_tokens / 1000000
-        
-        if response is None:
-            return "", cost
-        else:
-            return response.content, cost
-    
-    def prepare_message(self, model_input: dict, prompt_type: str):
-        message = []
-        return message
-
-    def generate_response(self, model_input: dict, prompt_type: str, constraint_str_list: list = None,):
-        total_cost = 0
-        response_list = []
-        # prepare message
-        message = self.prepare_message(model_input, prompt_type)
-        # print(message)
-
-        # n sampling
-        for i in range(self.num_generate):
-            response, cost = self.generate_with_retry(message, constraint_str_list)
-            response_list.append(response)
-            total_cost += cost
-        
-        return response_list, total_cost
-
-
-class GroundingJudgeAgent(BaseAgent):
-    def __init__(self, agent_config: dict):
-        super().__init__(agent_config)
-        self._setup()
-    
-    def prepare_message(self, model_input: dict, prompt_type):
-        message = get_messages(
-            input_info=model_input,
-            inference_mode="judge_grounding",
-            prompt_type=prompt_type,
-            use_multimodal=self.use_multimodal,
-            text_obs=self.agent_config["text_obs_type"],
-            image_obs=self.agent_config["image_obs_type"]
-        )
-        return message
-
-
-class ProgressJudgeAgent(BaseAgent):
-    def __init__(self, agent_config: dict):
-        super().__init__(agent_config)
-        self._setup()
-    
-    def prepare_message(self, model_input: dict, prompt_type):
-        if self.agent_config["input_type"]=="text_only":
-            use_multimodal = False
-            text_obs = self.agent_config["text_obs_type"]
-            image_obs = None
-        elif self.agent_config["input_type"]=="image_only":
-            use_multimodal = True
-            text_obs = None
-            image_obs = self.agent_config["image_obs_type"]
-        elif self.agent_config["input_type"]=="text_image":
-            use_multimodal = True
-            text_obs = self.agent_config["text_obs_type"]
-            image_obs = self.agent_config["image_obs_type"]
-        else:
-            raise ValueError(f"Invalid input type: {self.agent_config['input_type']}")
-        
-        if self.agent_config["use_in_progress"]:
-            use_in_progress = True
-        else:
-            use_in_progress = False
-
-        message = get_messages(
-            input_info=model_input,
-            inference_mode="judge_progress",
-            prompt_type=prompt_type,
-            use_checklist=self.use_checklist,
-            use_multimodal=use_multimodal,
-            text_obs=text_obs,
-            image_obs=image_obs,
-            use_in_progress=use_in_progress
-        )
-        return message
-    
-    def add_logprob(self, ori_logprob: float, add_logprob: float):
-        if ori_logprob is None:
-            return add_logprob
-        else:
-            ori_prob = math.exp(ori_logprob)
-            add_prob = math.exp(add_logprob)
-            return math.log(ori_prob + add_prob)
-    
-    def get_judge_probs(self, logprobs: list):
-        # target_judge = {
-        #     "yes": [" Yes", "Yes"],
-        #     "no": [" No", "No"],
-        #     "in": [" In", "In"]
-        # }
-        target_judge = {
-            "yes": [
-                " Yes", "ĠYes", "Yes", "ĊYes",
-                "Ġyes", "yes", "Ċyes",
-                "ĠYES", "YES", "ĊYES",
-                "ĠDone", "Done", "ĊDone",
-                "ĠCompleted", "Completed", "ĊCompleted",
-                "ĠCorrect", "Correct", "ĊCorrect"
-            ],
-            "no": [
-                " No", "ĠNo", "No", "ĊNo",
-                "ĠNO", "NO", "ĊNO",
-                "ĠNot", "Not", "ĊNot",
-                "ĠNone", "None", "ĊNone",
-                "ĠNope", "Nope", "ĊNope",
-                "ĠUn", "Un", "ĊUn",
-                "ĠWrong", "Wrong", "ĊWrong"
-            ],
-            "in": [
-                " In", "ĠIn", "In", "ĊIn",
-                "ĠPending", "Pending", "ĊPending",
-                "ĠPart", "Part", "ĊPart",
-                "ĠPartial", "Partial", "ĊPartial",
-                "ĠInProgress", "InProgress", "ĊInProgress"
-            ]
-        }
-        response_str = ""
-        judge_probs_list = []
-        # print(logprobs)
-        for i, log_prob in enumerate(logprobs):
-            # Start to find judge string
-            if "<answer>" in response_str:
-                find_judge_str = None
-                for judge_type in target_judge:
-                    if log_prob["token"] in target_judge[judge_type]:
-                        # print(log_prob)
-                        find_judge_str = judge_type
-                        break
-                if find_judge_str:
-                    # print("find judge str")
-                    token_judge_dict = {
-                        "yes": None,
-                        "no": None,
-                        "in": None
-                    }
-                    if "top_logprobs" in log_prob:
-                        for token_info in log_prob["top_logprobs"]:
-                            for judge_type in target_judge:
-                                for judge_str in target_judge[judge_type]:
-                                    # if judge_str in token_info["token"] and token_info["logprob"] > token_judge_dict[judge_type]:
-                                    #     token_judge_dict[judge_type] = token_info["logprob"]
-                                    if judge_str in token_info["token"]:
-                                        # print(token_info["logprob"])
-                                        token_judge_dict[judge_type] = self.add_logprob(token_judge_dict[judge_type], token_info["logprob"])
-                        # for None case
-                        for judge_type in token_judge_dict:
-                            if token_judge_dict[judge_type] is None:
-                                token_judge_dict[judge_type] = float("-inf")
-                        judge_probs_list.append(token_judge_dict)
-                    else:
-                        # for vllm bugs : no top_logprobs
-                        for judge_type in token_judge_dict:
-                            if judge_type == find_judge_str:
-                                token_judge_dict[judge_type] = log_prob["logprob"]
-                            else:
-                                token_judge_dict[judge_type] = float("-inf")
-                        judge_probs_list.append(token_judge_dict)
-                    # print(token_judge_dict)
-            
-            if "</answer>" in response_str:
-                break
-            
-            response_str += log_prob["token"]
-        # print(response_str.replace("Ġ", " ").replace("Ċ", "\n"))
-        # print(judge_probs_list)
-        if len(judge_probs_list) == 0:
-            return [{
-                "yes": 0.0,
-                "no": 0.0,
-                "in": 0.0
-            }]
-        else:
-            # convert with softmax
-            final_judge_probs_list = []
-            for judge_probs in judge_probs_list:
-                exp_logprobs = [math.exp(x) for x in [judge_probs["yes"], judge_probs["no"], judge_probs["in"]]]
-                sum_exp_logprobs = sum(exp_logprobs)
-                softmax_probs = [x / sum_exp_logprobs for x in exp_logprobs]
-                final_judge_probs_list.append({
-                    "yes": softmax_probs[0], 
-                    "no": softmax_probs[1],
-                    "in": softmax_probs[2]
-                })
-            return final_judge_probs_list
-    
-    def generate_probs(self, model_input: dict, prompt_type: str):
-        total_cost = 0
-        response_list = []
-        # prepare message
-        message = self.prepare_message(model_input, prompt_type)
-        # print(message)
-
-        for i in range(self.num_generate):
-            try:
-                response = self.llm.invoke(message)
-                total_input_tokens = response.response_metadata["token_usage"]["prompt_tokens"]
-                total_output_tokens = response.response_metadata["token_usage"]["completion_tokens"]
-                total_cost = self.input_token_cost * total_input_tokens / 1000000 + self.output_token_cost * total_output_tokens / 1000000  
-                logprobs = response.response_metadata["logprobs"]["content"]
-                response_list.append(
-                    {
-                        "response": response.content,
-                        "judge_probs": self.get_judge_probs(logprobs)
-                    }
-                )
-            except Exception as e:
-                print(f"Error: {e}")
-                # print(response.response_metadata["logprobs"])
-                response_list.append(
-                    {
-                        "response": response.content,
-                        "judge_probs": []
-                    }
-                )
-        return response_list, total_cost
-
-
-class ChecklistGenerationAgent(BaseAgent):
-    def __init__(self, agent_config: dict):
-        super().__init__(agent_config)
-        self._setup()
-    
-    def prepare_message(self, model_input: dict, prompt_type):
-        message = get_messages(
-            input_info=model_input,
-            inference_mode="checklist_generation",
-            prompt_type=prompt_type
-        )
-        return message
-    
-
-class ClassifierRewardAgent(Agent):
-    def __init__(self, url: str, use_checklist: bool = False, use_multimodal: bool = False):
-        self.url = url
-        self.use_checklist = use_checklist
-        self.use_multimodal = use_multimodal
-
-    def _process_multimodal_message(self, prompt: str, image_list: list[str]):
-        multimodal_message = []
-        text_prompt_prefix = prompt.split("<IMAGE_PLACEHOLDER>")[0]
-        text_prompt_suffix = prompt.split("<IMAGE_PLACEHOLDER>")[1]
-        multimodal_message = [
-            {"type": "text", "text": text_prompt_prefix},
-            # {"type": "image_url", "image_url": {"url": image_to_base64_url(image_list[0])}},
-            {"type": "image", "image": image_to_base64_url(image_list[0])},
-            {"type": "text", "text": text_prompt_suffix}
-        ]
-        return multimodal_message
-
-    def _make_query(self, user_prompt_template: dict, model_input: dict | list[dict]):
-        if self.use_multimodal:
-            tmp_user_prompt = user_prompt_template["user"].format(
-                **model_input
-            )
-            user_prompt =  self._process_multimodal_message(tmp_user_prompt, model_input["image_list"])
-        else:
-            user_prompt = user_prompt_template["user"].format(
-                **model_input
-            )
-        assistant_prompt = user_prompt_template["assistant"].format(
-            **model_input
-        )
-        query = [
-            {"role": "user", "content": user_prompt},
-            {"role": "assistant", "content": assistant_prompt}
-        ]
-        return query
-
-    def prepare_message(self, model_input: dict | list[dict], batch: bool = False):
-        if self.use_checklist:
-            if self.use_multimodal:
-                user_prompt_template = JUDGE_OURS_BT_MODELING_MULTIMODAL_PROMPT_TEMPLATE
-            else:
-                user_prompt_template = JUDGE_OURS_BT_MODELING_PROMPT_TEMPLATE
-        else:
-            if self.use_multimodal:
-                user_prompt_template = JUDGE_OURS_BT_MODELING_MULTIMODAL_WO_CHECKLIST_PROMPT_TEMPLATE
-            else:
-                user_prompt_template = JUDGE_OURS_BT_MODELING_WO_CHECKLIST_PROMPT_TEMPLATE
-
-        if self.use_multimodal:
-            if batch:
-                message = [self._make_query(user_prompt_template, input) for input in model_input]
-            else:
-                message = [self._make_query(user_prompt_template, model_input)]
-        else:
-            if batch:
-                message = {
-                    "query": [self._make_query(user_prompt_template, input) for input in model_input],
-                    "promptts": []
-                }
-            else:
-                message = {
-                    "query": self._make_query(user_prompt_template, model_input),
-                    "prompts": []
-                }
-
-        return message
-    
-    def get_rm_scroe(self, message: dict | list):
-        headers = {"Content-Type": "application/json"}
-        
-        try:
-            if self.use_multimodal:
-                response = requests.post(
-                    self.url,
-                    json={"messages": message},
-                    timeout=600
-                )
-            else:
-                response = requests.post(
-                    self.url,
-                    headers=headers,
-                    data=json.dumps(message),
-                    timeout=300
-                )
-            response.raise_for_status()
-            
-            response_json = response.json()
-
-            if "rewards" not in response_json:
-                print(f"Error: 'rewards' key not found in API response: {response_json}")
-                return []
-            
-            if "get_reward" in self.url:
-                # use openrlhf
-                return response_json["rewards"]
-            elif "pooling" in self.url:
-                # use vllm server
-                return response_json["reward"]
-            else:
-                # error
-                raise ValueError(f"Invalid URL: {self.url}")
-
-        except requests.exceptions.Timeout:
-            print(f"Error: Request timed out to {self.url}")
-            return []
-        except requests.exceptions.RequestException as e:
-            print(f"Error during request to {self.url}: {e}")
-            return []
-        except json.JSONDecodeError:
-            print(f"Error: Failed to decode JSON response from {self.url}")
-            return []
-        except KeyError as e:
-             print(f"Error: Missing key {e} in response from {self.url}")
-             return []
-
-
-    def generate_response(self, model_input: dict | list[dict], batch: bool = False):
-        if batch:
-            message = self.prepare_message(model_input, batch=True)
-        else:
-            message = self.prepare_message(model_input)
-        rewards = self.get_rm_scroe(message)
-        
-        return rewards, 0

agent/mini_bench/checklist_eval.py

@@ -1,95 +0,0 @@
-import re
-
-from langchain_openai import ChatOpenAI
-
-from .agent import BaseAgent
-
-SYSTEM_PROMPT = "You are an expert evaluator. Your task is to assess how well a Web Agent’s generated checklist aligns with the reference checklist for a given user instruction."
-
-USER_PROMPT = """# Task Description
-Use the provided task description, evaluation criteria, and both checklists to assign a score from 1 to 5. Justify your rating with a brief explanation that considers both content overlap and logical structure.
-
-## Score Criteria
-- 5: Checklist covers all subgoals, is correct and clearly expressed
-- 4: Minor omissions or phrasing issues but mostly accurate and complete
-- 3: Partially matches, but with noticeable gaps or errors
-- 2: Incomplete or includes incorrect steps
-- 1: Mostly irrelevant, incorrect, or missing the task goal
-
-## User Instruction:
-{intent}
-
-## Reference Checklist:
-{gt_checklist}
-
-## Agent’s Generated Checklist:
-{generated_checklist}
-
-# Output Format
-Your response should be in the following format:
-REASON: [Write 2–4 sentences explaining how well the generated checklist matches the reference. Mention specific matches, omissions, errors, or strengths.]
-SCORE: [1–5]
-"""
-
-
-class ChecklistEvalAgent(BaseAgent):
-    def __init__(self, agent_config: dict):
-        super().__init__(agent_config)
-        self._setup()
-    
-    def prepare_message(self, model_input: dict, prompt_type):
-        message = [
-            {
-                "role": "system",
-                "content": SYSTEM_PROMPT
-            },
-            {
-                "role": "user",
-                "content": USER_PROMPT.format(
-                    intent=model_input["intent"],
-                    gt_checklist=model_input["gt_checklist"],
-                    generated_checklist=model_input["generated_checklist"]
-                )
-            }
-        ]
-        return message
-    
-    def generate_response(self, model_input: dict):
-        total_cost = 0
-        response_list = []
-        # prepare message
-        message = self.prepare_message(model_input)
-
-        # n sampling
-        for _ in range(self.num_generate):
-            response, cost = self.generate_with_retry(message, ["SCORE"])
-            response_list.append(response)
-            total_cost += cost
-
-        return response_list, total_cost
-
-def parsing_score(response: str):
-    score = response.split("SCORE:")[-1].split("\n")[0].strip()
-    match = re.search(r'\d+', score)
-    
-    if match:
-        return int(match.group())
-    else:
-        return None
-
-def average_score(scores: list[int]):
-    if len(scores) == 0:
-        return 0
-    return sum(scores) / len(scores)
-
-def get_score(results: list[dict]):
-    score_list = []
-    for result in results:
-        tmp_scores = [parsing_score(response) for response in result["response"]]
-        scores = [score for score in tmp_scores if score is not None]
-        result["score_list"] = scores
-        final_score = average_score(scores)
-        result["score"] = final_score
-        score_list.append(result)
-
-    return results, score_list

agent/mini_bench/eval_utils.py

@@ -1,309 +0,0 @@
-import re
-import random
-from collections import Counter
-
-from .utils import load_json, save_json, create_html_report
-
-random.seed(42)
-def get_score(response_list: list, indicator: str) -> int:
-    if len(response_list) == 0:
-        return [-100]
-
-    if isinstance(response_list[0], float):
-        return response_list
-    
-    if indicator == "prob":
-        score_list = []
-        for response in response_list:
-            total_score = 0
-            for judge_probs in response:
-                yes_prob = judge_probs.get("yes", 0)
-                in_progress_prob = judge_probs.get("in", 0)
-                total_score += yes_prob + in_progress_prob * 0.5
-            if len(response) > 0:
-                score_list.append(total_score / len(response))
-            else:
-                score_list.append(0)
-        return score_list
-    else:
-        score_list = []
-        for response in response_list:
-            if indicator == "SCORE":
-                if "SCORE" in response:
-                    try:
-                        score_str = response.split("SCORE:")[1].split("\n")[0].strip()
-                    except:
-                        score_str = response.split("SCORE:")[-1].strip()
-                    # find first integer
-                    try:
-                        score = re.search(r'-?\d+', score_str).group()
-                        score_list.append(int(score))
-                    except:
-                        score_list.append(0)
-                else:
-                    try:
-                        score_str = response.split("<answer>")[1].split("</answer>")[0].strip()
-                    except:
-                        score_str = response.split("<answer>")[-1].split("</answer>")[0].strip()
-                    # find "Yes" or "No"
-                    if "Yes" in score_str:
-                        score_list.append(1)
-                    elif "In Progress" in score_str:
-                        score_list.append(0.5)
-                    elif "No" in score_str:
-                        score_list.append(0)
-                    else:
-                        score_list.append(0)
-            elif indicator == "JUDGE":
-                try:
-                    judge_str = response.split("JUDGE:")[1].split("\n")[0].strip()
-                except:
-                    judge_str = response.split("JUDGE:")[-1].strip()
-                if "Yes" in judge_str:
-                    score_list.append(1)
-                elif "No" in judge_str:
-                    score_list.append(0)
-                else:
-                    score_list.append(0)
-            elif indicator == "CHECKLIST EVALUATION":
-                if "<answer>" in response:
-                    try:
-                        checklist_str = response.split("<answer>")[1].split("</answer>")[0].strip()
-                    except:
-                        checklist_str = response.split("<answer>")[-1].split("</answer>")[0].strip()
-                else:
-                    checklist_str = response.split("CHECKLIST EVALUATION:")[-1].strip()
-                
-                count_yes = checklist_str.count("Yes")
-                count_no = checklist_str.count("No")
-                count_in_progress = checklist_str.count("In Progress")
-                try:
-                    total_score = (count_yes + count_in_progress*0.5) / (count_yes + count_no + count_in_progress)
-                except:
-                    total_score = 0
-                score_list.append(total_score)
-            else:
-                raise ValueError(f"Invalid indicator: {indicator}")
-    return score_list
-
-def get_acc_and_mrr(chosen_score, rejected_scores):
-    if len(rejected_scores) == 0:
-        return 0, False
-    
-    same_score_num = rejected_scores.count(chosen_score)
-    all_scores = rejected_scores + [chosen_score]
-    sorted_scores = sorted(all_scores, reverse=True)
-    rank = sorted_scores.index(chosen_score) + 1 + same_score_num  # draw penalty
-    if all(chosen_score > r for r in rejected_scores):
-        accuracy = True
-    else:
-        accuracy = False
-    return 1 / rank, accuracy
-
-def average_score(score_list: list[float]):
-    if len(score_list) == 0:
-        return -100
-    return sum(score_list) / len(score_list)
-
-def self_consistency_score(score_list: list[float]):
-    if len(score_list) == 0:
-        return -100
-    counter = Counter(score_list)
-    return max(counter.values()) / len(score_list)
-
-def get_chosen_rejected_scores(data: dict, agg_func: str):
-    if len(data["chosen"]) == 0:
-        data["chosen"] = [{"score": [-100]}]
-    if len(data["rejected"]) == 0:
-        data["rejected"] = [{"score": [-100]}]
-    if not isinstance(data["chosen"][0], dict):
-        data["chosen"][0]["score"] = [-100]
-    if not isinstance(data["rejected"][0], dict):
-        data["rejected"][0]["score"] = [-100]
-        
-    if agg_func == "average":
-        chosen_score = average_score(data["chosen"][0]["score"])
-        rejected_scores = [average_score(rejected_score["score"]) for rejected_score in data["rejected"]]
-    elif agg_func == "self_consistency":
-        chosen_score = self_consistency_score(data["chosen"][0]["score"])
-        rejected_scores = [self_consistency_score(rejected_score["score"]) for rejected_score in data["rejected"]]
-    else:
-        raise ValueError(f"Invalid agg_func: {agg_func}")
-    return chosen_score, rejected_scores
-
-def get_score_results(results, agg_func):
-    score_dict = {"mrr": [], "accuracy": [], "traj_accuracy": []}
-    task_accuracy = {}
-    for result in results:
-        chosen_score, rejected_scores = get_chosen_rejected_scores(result, agg_func)
-        mrr, accuracy = get_acc_and_mrr(chosen_score, rejected_scores)
-        score_dict["mrr"].append(mrr)
-        score_dict["accuracy"].append(accuracy)
-        if result["task_id"] not in task_accuracy:
-            task_accuracy[result["task_id"]] = []
-        task_accuracy[result["task_id"]].append(accuracy)
-
-    for task_id in task_accuracy:
-        if sum(task_accuracy[task_id]) == len(task_accuracy[task_id]):
-            score_dict["traj_accuracy"].append(True)
-        else:
-            score_dict["traj_accuracy"].append(False)
-
-    return score_dict
-
-def calculate_stats(results, agg_func: str="average"):
-    if len(results) == 0:
-        return {
-            "MRR": 0,
-            "Accuracy": 0,
-            "Traj_Accuracy": 0,
-        }
-    total_score = get_score_results(results, agg_func)
-    stats = {
-        "MRR": sum(total_score["mrr"]) / len(total_score["mrr"]),
-        "Accuracy": sum(total_score["accuracy"]) / len(total_score["accuracy"]),
-        "Traj_Accuracy": sum(total_score["traj_accuracy"]) / len(total_score["traj_accuracy"]),
-    }
-    
-    return stats
-
-def group_by_task(results, split_indicator: str):
-    # sort results by task_id and step_id
-    results.sort(key=lambda x: (x["task_id"], x["step_id"]))
-    # group by task_name
-    grouped_task_dict = {}
-    for result in results:
-        task_name = "task_" + str(result["task_id"]) + "_step_" + str(result["step_id"])
-        if task_name not in grouped_task_dict:
-            grouped_task_dict[task_name] = {
-                "task_id": result["task_id"],
-                "step_id": result["step_id"],
-                "intent": result["intent"],
-                "start_url": result["start_url"],
-                "gt_checklist": result["gt_checklist"],
-                "generated_checklist": result.get("generated_checklist", None)  ,
-                "trajectory": result["trajectory"],
-                "current_url": result["current_url"],
-                "text_observation": result["text_observation"],
-                # "image_list": result["image_list"],
-                "chosen": [],
-                "rejected": [],
-                "source_name": result["source_name"],
-            }
-        
-        response = result["response"] if "response" in result else []
-        type_data = {
-            "thought": result["thought"],
-            "action": result["action"],
-            "response": response,
-            "score": get_score(response, split_indicator) if split_indicator != "prob" else get_score(result["judge_probs"], split_indicator),
-        }
-        if split_indicator == "prob":
-            type_data["judge_probs"] = result["judge_probs"]
-        if result["type"] == "chosen":
-            grouped_task_dict[task_name]["chosen"].append(type_data)
-        elif result["type"] == "rejected":
-            grouped_task_dict[task_name]["rejected"].append(type_data)
-    
-    return list(grouped_task_dict.values())
-
-
-def processing_results(results, evaluation_mode: str, num_generate: int, use_batch: bool=False):
-    if "judge_probs" in results[0]:
-        split_indicator = "prob"
-    else:
-        if evaluation_mode == "judge_with_checklist_generation" or evaluation_mode == "judge_with_gt_checklist":
-            split_indicator = "CHECKLIST EVALUATION" 
-        else:
-            split_indicator = "SCORE"
-
-    # if use_batch is True, make it flattened
-    if use_batch:
-        tmp_results = []
-        for result in results:
-            for d in result:
-                tmp_results.append(d)
-        grouped_results = group_by_task(tmp_results, split_indicator)
-    else:
-        grouped_results = group_by_task(results, split_indicator)
-
-    mind2web_results = []
-    webarena_results = []
-    mind2web_task_results = []
-    mind2web_website_results = []
-    mind2web_domain_results = []
-    
-    for grouped_result in grouped_results:
-        if "mind2web" in grouped_result["source_name"]:
-            mind2web_results.append(grouped_result)
-            if grouped_result["source_name"] == "mind2web_test_task":
-                mind2web_task_results.append(grouped_result)
-            elif grouped_result["source_name"] == "mind2web_test_website":
-                mind2web_website_results.append(grouped_result)
-            elif grouped_result["source_name"] == "mind2web_test_domain":
-                mind2web_domain_results.append(grouped_result)
-        elif "webarena" in grouped_result["source_name"]:
-            webarena_results.append(grouped_result)
-            
-    try:
-        final_stats = {
-            "mind2web": {
-                "MRR": {},
-                "Accuracy": {},
-                "Traj_Accuracy": {},
-            },
-            "webarena": {
-                "MRR": {},
-                "Accuracy": {},
-                "Traj_Accuracy": {},
-            },
-            "mind2web_task": {
-                "MRR": {},
-                "Accuracy": {},
-                "Traj_Accuracy": {},
-            },
-            "mind2web_website": {
-                "MRR": {},
-                "Accuracy": {},
-                "Traj_Accuracy": {},
-            },
-            "mind2web_domain": {
-                "MRR": {},
-                "Accuracy": {},
-                "Traj_Accuracy": {},
-            },
-        }
-        for source_results in [
-            ("mind2web", mind2web_results), 
-            ("webarena", webarena_results),
-            ("mind2web_task", mind2web_task_results),
-            ("mind2web_website", mind2web_website_results),
-            ("mind2web_domain", mind2web_domain_results)
-        ]:
-            average_stats = calculate_stats(source_results[1], "average")
-            self_consistency_stats = calculate_stats(source_results[1], "self_consistency")
-            for metric in average_stats:
-                final_stats[source_results[0]][metric]["Average"] = average_stats[metric]
-            for metric in self_consistency_stats:
-                final_stats[source_results[0]][metric]["Self_Consistency"] = self_consistency_stats[metric]
-        
-        if num_generate == 1:
-            for source_name in final_stats:
-                for metric in final_stats[source_name]:
-                    print(f"{round(100 * final_stats[source_name][metric]['Average'], 2)}", end=", ")
-            print()
-        else:
-            for agg_func in ["Average", "Self_Consistency"]:
-                print(f"{agg_func}")
-                for source_name in final_stats:
-                    for metric in final_stats[source_name]:
-                        print(f"{round(100 * final_stats[source_name][metric][agg_func], 2)}", end=", ")
-                print()
-    except Exception as e:
-        print(e)
-        return grouped_results, None    
-    
-    # add function to convert json format results to html format results
-    # TODO: implement this function
-    # create_html_report(results, "results.html")
-    return grouped_results, final_stats

agent/mini_bench/inference_utils.py

@@ -1,87 +0,0 @@
-import time
-
-from multiprocessing import Process, Manager
-from tqdm import tqdm
-
-
-def worker_main(work_queue, result_queue, process_func, config):
-    while True:
-        item = work_queue.get()
-        if item is None:
-            result_queue.put(None)
-            break
-        try:
-            results, cost = process_func(config, item)
-            result_queue.put((results, cost))
-        except Exception as e:
-            item_info = item.get('idx', item.get('id', 'unknown item'))
-            print(f"Error processing item {item_info}: {e}")
-            result_queue.put(None)
-        finally:
-            work_queue.task_done()
-
-def run_parallel_evaluation(dataset, process_func, config, num_workers, description):
-    """
-    Runs parallel evaluation on the given dataset and returns the results.
-
-    Args:
-        dataset (list or datasets.Dataset): Data to evaluate.
-        process_func (callable): Function to process each data item.
-        config (dict): Configuration for the process_func.
-        num_workers (int): Number of worker processes to use.
-        description (str): Description to display on the tqdm progress bar.
-
-    Returns:
-        tuple: (list of evaluation results, total cost)
-    """
-    manager = Manager()
-    work_queue = manager.Queue()
-    result_queue = manager.Queue()
-
-    # Add data to the work queue
-    dataset_list = list(dataset) if not isinstance(dataset, list) else dataset
-    for data in dataset_list:
-        work_queue.put(data)
-    
-    # Add termination signals for workers
-    for _ in range(num_workers):
-        work_queue.put(None)
-
-    # Start parallel processing
-    processes = []
-    for _ in range(num_workers):
-        p = Process(target=worker_main, args=(work_queue, result_queue, process_func, config))
-        p.start()
-        processes.append(p)
-    
-    # Show progress bar and collect results
-    process_results = []
-    process_cost = 0
-    completed_workers = 0
-
-    with tqdm(total=len(dataset_list), desc=description) as pbar:
-        while completed_workers < num_workers:
-            result_item = result_queue.get()
-            if result_item is None:
-                completed_workers += 1
-            else:
-                results, cost = result_item
-                if results is not None:
-                    process_results.append(results)
-                    process_cost += cost if cost is not None else 0
-                pbar.update(1)
-
-    # Wait for all processes to finish
-    for p in processes:
-        p.join()
-
-    # Collect remaining results
-    while not result_queue.empty():
-        result_item = result_queue.get_nowait()
-        if result_item is not None:
-            results, cost = result_item
-            if results is not None:
-                process_results.append(results)
-                process_cost += cost if cost is not None else 0
-
-    return process_results, process_cost

agent/mini_bench/prompts/__init__.py

@@ -1 +0,0 @@
-from .construct_messages import get_messages

agent/mini_bench/prompts/action.py

@@ -1,93 +0,0 @@
-ACTION_SPACE_PROMPT = """Note: This action set allows you to interact with your environment. Most of them are python function executing playwright code. The primary way of referring to elements in the page is through bid which are specified in your observations.
-
-15 different types of actions are available.
-
-noop(wait_ms: float = 1000)
-    Examples:
-        noop()
-
-        noop(500)
-
-scroll(delta_x: float, delta_y: float)
-    Examples:
-        scroll(0, 200)
-
-        scroll(-50.2, -100.5)
-
-keyboard_press(key: str)
-    Examples:
-        keyboard_press('Backspace')
-
-        keyboard_press('ControlOrMeta+a')
-
-        keyboard_press('Meta+Shift+t')
-
-click(bid: str, button: Literal['left', 'middle', 'right'] = 'left', modifiers: list[typing.Literal['Alt', 'Control', 'ControlOrMeta', 'Meta', 'Shift']] = [])
-    Examples:
-        click('a51')
-
-        click('b22', button='right')
-
-        click('48', button='middle', modifiers=['Shift'])
-
-fill(bid: str, value: str)
-    Examples:
-        fill('237', 'example value')
-
-        fill('45', 'multi-line\nexample')
-
-        fill('a12', 'example with "quotes"')
-
-hover(bid: str)
-    Examples:
-        hover('b8')
-
-tab_focus(index: int)
-    Examples:
-        tab_focus(2)
-
-new_tab()
-    Examples:
-        new_tab()
-
-go_back()
-    Examples:
-        go_back()
-
-go_forward()
-    Examples:
-        go_forward()
-
-goto(url: str)
-    Examples:
-        goto('http://www.example.com')
-
-tab_close()
-    Examples:
-        tab_close()
-
-select_option(bid: str, options: str | list[str])
-    Examples:
-        select_option('a48', 'blue')
-
-        select_option('c48', ['red', 'green', 'blue'])
-
-send_msg_to_user(text: str)
-    Examples:
-        send_msg_to_user('Based on the results of my search, the city was built in 1751.')
-
-report_infeasible(reason: str)
-    Examples:
-        report_infeasible('I cannot follow these instructions because there is no email field in this form.')
-
-Only a single action can be provided at once. Example:
-fill('a12', 'example with "quotes"')
-
-Note:
-* Some tasks may be game like and may require to interact with the mouse position in x, y coordinates.
-* Some text field might have auto completion. To see it, you have to type a few characters and wait until next step.
-* If you have to cut and paste, don't forget to select the text first.
-* Coordinate inside an SVG are relative to it's top left corner.
-* Make sure to use bid to identify elements when using commands.
-* Interacting with combobox, dropdowns and auto-complete fields can be tricky, sometimes you need to use select_option, while other times you need to use fill or click and wait for the reaction of the page.
-"""

agent/mini_bench/prompts/checklist_prompt.py

@@ -1,50 +0,0 @@
-CHECKLIST_SYSTEM_PROMPT = "You are an AI assistant tasked with generating structured checklists that highlight key subgoals necessary to complete a task."
-
-CHECKLIST_USER_PROMPT = """## Task Description
-User Instruction (Goal): "{intent}"
-Start Website URL: {start_url}
-
-## Guidelines for Checklist Generation
-1. Identify Essential High-Level Subgoals:
-- A subgoal should represent a significant step involving user interaction that leads to noticeable page transitions or meaningful changes in system state.
-- Consolidate closely related user actions (such as applying multiple filters or selecting several options) into a single subgoal, rather than separate checklist items for each action.
-- Prioritize only the most critical interactions necessary for meaningful progression, avoiding the inclusion of minor or unnecessary steps (e.g., scroll, hover).
-2. Provide a Concise Subgoal Analysis:
-- Before creating the checklist, offer a brief paragraph summarizing the main subgoals, emphasizing significant transitions or page-level interactions.
-3. Ensure Clear Goal:
-- If multiple related interactions occur (e.g., setting filters 1, 2, and 3), combine them into one subgoal with clear criteria verifying all required conditions.
-- The checklist should contain only essential steps, explicitly excluding unnecessary actions, and should not exceed five critical subgoals. It is not necessary to use all five checklist items if fewer steps adequately represent the essential subgoals.
-
-### Output Format
-Before generating the checklist, first produce a concise subgoal analysis in a single paragraph summarizing the required interactions. Then, based on this, generate the checklist following the format below:
-[SUBGOAL ANALYSIS]  
-[One-paragraph summary explaining the key subgoals and their logical sequence in task completion.]  
-
-[CHECKLISTS]
-Checklist X: [Short title of the action/goal]
-- Goal: [Brief description of the subgoal at this stage, emphasizing the purpose of the action.]
-"""
-
-# TODO: implement ours
-CHECKLIST_OURS_SYSTEM_PROMPT = ""
-
-CHECKLIST_OURS_USER_PROMPT = """You are an AI assistant tasked with generating structured checklists that highlight key subgoals necessary to complete a task.
-
-# Task Description
-Generate a checklist which are key milestones for achieving the given instruction. Frist, provide a concise 
-subgoal analysis in a single paragraph summarizing the required interactions. Then, based on this, generate the checklist with breif description.
-
-Note: If the target website requires login, assume the user is already logged in and starts from an authenticated session.
-
-# Given Information
-## User Instruction
-{intent}
-
-## Current State
-### Current URL
-{start_url}
-
-### AXTREE
-Note: [bid] is the unique alpha-numeric identifier at the beginning of lines for each element in the AXTree. Always use bid to refer to elements in your actions.
-{text_observation}
-"""

agent/mini_bench/prompts/construct_messages.py

@@ -1,309 +0,0 @@
-from abc import ABC, abstractmethod
-
-from .action import ACTION_SPACE_PROMPT
-from .eval_type import (
-    GROUNDING,
-    PROGRESS_LIKERT_SCALE,
-    PROGRESS_THREE_CLASS,
-    PROGRESS_WITH_CHECKLIST,
-    PROGRESS_WITH_CHECKLIST_IN_PROGRESS,
-    PROGRESS_OURS
-)
-from .input_information import (
-    USER_INSTRUCTION,
-    TRAJECTORY,
-    AGENT_RESPONSE,
-    CHECKLIST,
-    CURRENT_URL,
-    TEXT_OBSERVATION,
-    SOM_IMAGE_OBSERVATION, 
-    COORD_IMAGE_OBSERVATION
-)
-from .judge_prompt import (
-    JUDGE_GROUNDING_PROMPT_TEMPLATE,
-    JUDGE_LIKERT_SCALE_PROMPT_TEMPLATE,
-    JUDGE_THREE_CLASS_PROMPT_TEMPLATE,
-    JUDGE_WITH_CHECKLIST_PROMPT_TEMPLATE,
-    JUDGE_OURS_PROMPT_TEMPLATE,
-    JUDGE_OURS_WO_CHECKLIST_PROMPT_TEMPLATE
-)
-from .checklist_prompt import (
-    CHECKLIST_SYSTEM_PROMPT,
-    CHECKLIST_USER_PROMPT,
-    CHECKLIST_OURS_SYSTEM_PROMPT,
-    CHECKLIST_OURS_USER_PROMPT
-)
-from .image_utils import image_to_base64_url
-
-
-class Message(ABC):    
-    @abstractmethod
-    def get_messages(self):
-        pass
-
-class BaseMessage(Message):
-    def __init__(self, input_info:dict, use_multimodal:bool=False):
-        self.input_info = input_info
-        self.use_multimodal = use_multimodal
-
-    def _get_system_message(self):
-        system_message = {"role": "system", "content": "You are a helpful assistant."}
-        return system_message
-    
-    def _process_multimodal_message(self, prompt: str, image_list: list[str]):
-        multimodal_message = []
-        text_prompt_prefix = prompt.split("<IMAGE_PLACEHOLDER>")[0]
-        text_prompt_suffix = prompt.split("<IMAGE_PLACEHOLDER>")[1]
-        multimodal_message.append({"type": "text", "text": text_prompt_prefix})
-        for i, image in enumerate(image_list):
-            # TODO: text prompt for multiple images
-            # multimodal_message.append({"type": "text", "text": f"IMAGE {i+1}\n"})
-            multimodal_message.append({"type": "image_url", "image_url": {"url": image_to_base64_url(image), "detail": "low"}})
-        multimodal_message.append({"type": "text", "text": text_prompt_suffix})
-        return {"role": "user", "content": multimodal_message}
-    
-    def _get_user_message(self):
-        user_prompt = "What is the capital of France?"
-        if self.use_multimodal:
-            image_list = self.input_info.get("image_list", [])
-            user_message = self._process_multimodal_message(user_prompt, image_list)
-        else:
-            user_message = {"role": "user", "content": user_prompt}
-        return user_message
-
-    def get_messages(self):
-        message = []
-        system_message = self._get_system_message()
-        user_message = self._get_user_message()
-
-        message.append(system_message)
-        # message.append({"role": "system", "content": ""})
-        message.append(user_message)
-        return message
-    
-
-class ProgressMessage(BaseMessage):
-    '''
-    Progress Judge Message
-    '''
-    def __init__(self, input_info:dict, use_multimodal:bool, prompt_type:str, text_obs: str, image_obs: str, use_checklist:bool, use_in_progress:bool):
-        super().__init__(input_info, use_multimodal)
-        self.prompt_type = prompt_type
-        self.text_obs = text_obs
-        self.image_obs = image_obs
-        self.use_checklist = use_checklist
-        self.use_in_progress = use_in_progress
-    
-    def _get_system_message(self):
-        if self.prompt_type == "likert_scale":
-            system_message = {"role": "system", "content": JUDGE_LIKERT_SCALE_PROMPT_TEMPLATE["system"]}
-        elif self.prompt_type == "three_class":
-            system_message = {"role": "system", "content": JUDGE_THREE_CLASS_PROMPT_TEMPLATE["system"]}
-        elif self.prompt_type == "with_checklist":
-            system_message = {"role": "system", "content": JUDGE_WITH_CHECKLIST_PROMPT_TEMPLATE["system"]}
-        elif self.prompt_type == "ours":
-            system_message = {"role": "system", "content": JUDGE_OURS_PROMPT_TEMPLATE["system"]}
-        else:
-            raise ValueError(f"Invalid prompt type: {self.prompt_type}")
-        return system_message
-        
-    def _setup_input_information(self):
-        observation = "## Current State\n"
-
-        observation += CURRENT_URL
-
-        # text observation
-        if self.text_obs:
-            observation += TEXT_OBSERVATION
-        
-        # image observation (som, coord, none)
-        if self.image_obs == "som":
-            observation += SOM_IMAGE_OBSERVATION
-        elif self.image_obs == "coord":
-            observation += COORD_IMAGE_OBSERVATION
-
-
-        if self.use_checklist:
-            input_information = USER_INSTRUCTION + TRAJECTORY + observation + CHECKLIST + AGENT_RESPONSE
-        else:
-            input_information = USER_INSTRUCTION + TRAJECTORY + observation + AGENT_RESPONSE
-
-        return input_information
-    
-    def _setup_task_info(self):
-        if self.prompt_type == "likert_scale":
-            task_description = PROGRESS_LIKERT_SCALE["task_description"]
-            output_format = PROGRESS_LIKERT_SCALE["output_format"]
-        elif self.prompt_type == "three_class":
-            task_description = PROGRESS_THREE_CLASS["task_description"]
-            output_format = PROGRESS_THREE_CLASS["output_format"]
-        elif self.prompt_type == "with_checklist":
-            if self.use_in_progress:
-                task_description = PROGRESS_WITH_CHECKLIST_IN_PROGRESS["task_description"]
-                output_format = PROGRESS_WITH_CHECKLIST_IN_PROGRESS["output_format"]
-            else:
-                task_description = PROGRESS_WITH_CHECKLIST["task_description"]
-                output_format = PROGRESS_WITH_CHECKLIST["output_format"]
-        else:
-            raise ValueError(f"Invalid prompt type: {self.prompt_type}")
-        return task_description, output_format
-    
-    def _get_user_prompt_template(self):
-        if self.prompt_type == "likert_scale":
-            user_prompt = JUDGE_LIKERT_SCALE_PROMPT_TEMPLATE["user"]
-        elif self.prompt_type == "three_class":
-            user_prompt = JUDGE_THREE_CLASS_PROMPT_TEMPLATE["user"]
-        elif self.prompt_type == "with_checklist":
-            user_prompt = JUDGE_WITH_CHECKLIST_PROMPT_TEMPLATE["user"]
-        else:
-            raise ValueError(f"Invalid prompt type: {self.prompt_type}")
-        return user_prompt
-    
-    def _get_user_message(self):
-        # setup input information (user_instruction, trajectory, current_state, agent_response, checklist)
-        input_information_template = self._setup_input_information()
-        input_information = input_information_template.format(**self.input_info)
-
-        if self.prompt_type == "ours":
-            if self.use_checklist:
-                user_prompt = JUDGE_OURS_PROMPT_TEMPLATE["user"].format(
-                    input_information=input_information,
-                )
-            else:
-                user_prompt = JUDGE_OURS_WO_CHECKLIST_PROMPT_TEMPLATE["user"].format(
-                    input_information=input_information,
-                )
-        else:
-            task_description, output_format = self._setup_task_info()
-            # get user prompt template by prompt type
-            user_prompt_template = self._get_user_prompt_template()
-            user_prompt = user_prompt_template.format(
-                action_space=ACTION_SPACE_PROMPT,
-                task_description=task_description,
-                input_information=input_information,
-                output_format=output_format
-            )
-
-        # process multimodal message
-        if self.use_multimodal:
-            image_list = self.input_info.get("image_list", [])
-            user_message = self._process_multimodal_message(user_prompt, image_list)
-        else:
-            user_message = {"role": "user", "content": user_prompt}
-
-        return user_message
-
-
-class GroundingMessage(BaseMessage):
-    '''
-    Grounding Judge Message
-    '''
-    def __init__(self, input_info:dict, use_multimodal:bool, prompt_type:str, text_obs: str, image_obs: str):
-        super().__init__(input_info, use_multimodal)
-        self.prompt_type = prompt_type
-        self.text_obs = text_obs
-        self.image_obs = image_obs
-
-    def _get_system_message(self):
-        if self.prompt_type == "ours":
-            # TODO: implement ours
-            system_message = {"role": "system", "content": "You are a helpful assistant."}
-        elif self.prompt_type == "default":
-            system_message = {"role": "system", "content": JUDGE_GROUNDING_PROMPT_TEMPLATE["system"]}
-        else:
-            raise ValueError(f"Invalid prompt type: {self.prompt_type}")
-        return system_message
-
-    def _setup_input_information(self):
-        observation = "## Current State\n"
-
-        observation += CURRENT_URL
-
-        # text observation
-        if self.text_obs:
-            observation += TEXT_OBSERVATION
-        
-        # image observation (som, coord, none)
-        if self.image_obs == "som":
-            observation += SOM_IMAGE_OBSERVATION
-        elif self.image_obs == "coord":
-            observation += COORD_IMAGE_OBSERVATION
-
-        # input_information = USER_INSTRUCTION + TRAJECTORY + observation + AGENT_RESPONSE # with trajectory
-        input_information = USER_INSTRUCTION + observation + AGENT_RESPONSE # without trajectory
-
-        return input_information
-    
-    def _get_user_message(self):
-        if self.prompt_type == "ours":
-            # TODO: implement ours
-            user_message = {"role": "user", "content": "TODO"}
-        elif self.prompt_type == "default":
-            action_space = ACTION_SPACE_PROMPT
-            task_description = GROUNDING["task_description"]
-            output_format = GROUNDING["output_format"]
-            input_information_template = self._setup_input_information() 
-            input_information = input_information_template.format(**self.input_info)
-
-            user_prompt = JUDGE_GROUNDING_PROMPT_TEMPLATE["user"].format(
-                action_space=action_space,
-                task_description=task_description,
-                input_information=input_information,
-                output_format=output_format
-            )
-
-            # process multimodal message
-            if self.use_multimodal:
-                image_list = self.input_info.get("image_list", [])
-                user_message = self._process_multimodal_message(user_prompt, image_list)
-            else:
-                user_message = {"role": "user", "content": user_prompt} 
-        else:
-            raise ValueError(f"Invalid prompt type: {self.prompt_type}")
-        return user_message
-
-
-class ChecklistMessage(BaseMessage):
-    '''
-    Checklist Message
-    '''
-    def __init__(self, input_info:dict, use_multimodal:bool, prompt_type:str):
-        super().__init__(input_info, use_multimodal)
-        self.prompt_type = prompt_type
-    
-    def _get_system_message(self):
-        if self.prompt_type == "ours":
-            # TODO: implement ours
-            system_message = {"role": "system", "content": CHECKLIST_OURS_SYSTEM_PROMPT}
-        elif self.prompt_type == "default":
-            system_message = {"role": "system", "content": CHECKLIST_SYSTEM_PROMPT}
-        else:
-            raise ValueError(f"Invalid prompt type: {self.prompt_type}")
-        return system_message
-    
-    def _get_user_message(self):
-        if self.prompt_type == "ours":
-            user_message = {"role": "user", "content": CHECKLIST_OURS_USER_PROMPT.format(**self.input_info)}
-        elif self.prompt_type == "default":
-            user_message = {"role": "user", "content": CHECKLIST_USER_PROMPT.format(**self.input_info)}
-        else:
-            raise ValueError(f"Invalid prompt type: {self.prompt_type}")
-        return user_message
-    
-
-def get_messages(input_info:dict, inference_mode:str, prompt_type:str, text_obs:str=None, image_obs:str=None, use_multimodal:bool=False, use_checklist:bool=False, use_in_progress:bool=False):
-    message_list = []
-    if inference_mode == "judge_grounding":
-        message = GroundingMessage(input_info, use_multimodal=use_multimodal, prompt_type=prompt_type, text_obs=text_obs, image_obs=image_obs)
-    elif inference_mode == "judge_progress":
-        message = ProgressMessage(input_info, use_multimodal=use_multimodal, prompt_type=prompt_type, text_obs=text_obs, image_obs=image_obs, use_checklist=use_checklist, use_in_progress=use_in_progress)
-    elif inference_mode == "checklist_generation":
-        message = ChecklistMessage(input_info, use_multimodal=False, prompt_type=prompt_type)
-    else:
-        raise ValueError(f"Invalid inference mode: {inference_mode}")
-    
-    system_message, user_message = message.get_messages()
-    
-    message_list.append(system_message)
-    message_list.append(user_message)
-    return message_list

agent/mini_bench/prompts/eval_type.py

@@ -1,107 +0,0 @@
-# Task Description & Output Format
-GROUNDING_TASK = """Your task is to evaluate whether the agent's ACTION is properly grounded in its THOUGHT, considering the current state of the webpage.
-Use the user instruction, the current webpage state, and the agent's thought and action as evidence for your judgment. Your evaluation should assess whether the ACTION logically follows from the THOUGHT and is feasible and appropriate in the given environment.
-Mark the action as 'Yes' only if it is clearly and fully grounded in the thought and current webpage state. If there is any inconsistency, ambiguity, irrelevance, or if the action is not supported by the current page state, mark it as 'No'."""
-
-GROUNDING_FORMAT = """Please return your response in the following format:
-REASON: [Your explanation for whether the action is properly grounded]
-JUDGE: [Yes / No]"""
-
-
-PROGRESS_LIKERT_SCALE_TASK = """Evaluate how helpful the given thought and action is for achieving the goal. Use the following scale:
-**Scoring Criteria (1 to 5):**
-- **5 (Very Helpful)**: The action directly and effectively moves toward fulfilling a key part of the goal.
-- **4 (Helpful)**: The action contributes meaningfully to progress, though it may require follow-up actions.
-- **3 (Somewhat Helpful)**: The action is partially relevant or a preparatory step, but doesn’t make immediate progress.
-- **2 (Slightly Helpful)**: The action is weakly related to the goal or might only indirectly help.
-- **1 (Not Helpful)**: The action is unrelated, redundant, or distracts from the goal."""
-
-PROGRESS_LIKERT_SCALE_FORMAT = """Please return your response in the following format:
-REASON: [Your explanation for the score]
-SCORE: [1-5]"""
-
-
-PROGRESS_THREE_CLASS_TASK = """Evaluate how helpful the given thought and action is for achieving the goal. Use the following scale:
-**Scoring Criteria:**
-- **1 (Helpful)**: The action clearly contributes to achieving the goal. It takes a necessary or productive step toward completing the task.
-- **0 (Neutral)**: The action is neither helpful nor harmful. It may be a placeholder, irrelevant at the current step, or too ambiguous to evaluate.
-- **-1 (Not Helpful)**: The action works against the goal, causes confusion, repeats a previous step unnecessarily, or leads the agent off track."""
-
-PROGRESS_THREE_CLASS_FORMAT = """Please return your response in the following format:
-REASON: [Your explanation for the score]
-SCORE: [-1 / 0 / 1]"""
-
-
-PROGRESS_WITH_CHECKLIST_TASK = """Your task is to evaluate how well the agent's THOUGHT and ACTION satisfy each item in the checklist.
-Use the task instruction, trajectory (including previously completed steps from history), current webpage state, and the agent's current response as evidence for your evaluation.
-For each checklist item:
-- Mark it as 'Yes' if it is clearly and fully satisfied either in the current response or already completed in the history.
-- Mark it as 'No' if there is ambiguity, insufficient evidence, or the step is incomplete or not yet started."""
-
-PROGRESS_WITH_CHECKLIST_FORMAT = """Please return your response in the following format:
-REASON: [Write a single, coherent paragraph explaining how well the agent's response satisfies the checklist overall. Use both the history and the agent's current thought/action as evidence. Mention specific strengths or missing elements that influence your decision.]
-CHECKLIST EVALUATION:  
-Checklist X: [Yes / No]  
-"""
-
-PROGRESS_WITH_CHECKLIST_IN_PROGRESS_TASK = """Your task is to evaluate how well the agent's THOUGHT and ACTION satisfy each item in the checklist.
-Use the task instruction, trajectory (including previously completed steps from history), current webpage state, and the agent's current response as evidence for your evaluation. Clearly consider any items already successfully completed or currently in progress according to the provided trajectory.
-For each checklist item:
-- Mark it as 'Yes' if it is clearly and fully satisfied either in the current response or already completed in the history.
-- Mark it as 'In Progress' if the agent has made partial but meaningful progress toward completing the item.
-- Mark it as 'No' if there is ambiguity, insufficient evidence, or the step is incomplete or not yet started."""
-
-PROGRESS_WITH_CHECKLIST_IN_PROGRESS_FORMAT = """Please return your response in the following format:
-REASON: [Write a single, coherent paragraph explaining how well the agent's response satisfies the checklist overall. Use both the history and the agent's current thought/action as evidence. Mention specific strengths or missing elements that influence your decision.]
-CHECKLIST EVALUATION:  
-Checklist X: [Yes / In Progress / No]  
-"""
-
-
-GROUNDING_OURS_TASK = """
-"""
-
-GROUNDING_OURS_FORMAT = """
-"""
-
-PROGRESS_OURS_TASK = """
-"""
-
-PROGRESS_OURS_FORMAT = """
-"""
-
-## EVALUATION TYPE
-GROUNDING = {
-    "task_description": GROUNDING_TASK,
-    "output_format": GROUNDING_FORMAT,
-}
-
-GROUNDING_OURS = {
-    "task_description": GROUNDING_OURS_TASK,
-    "output_format": GROUNDING_OURS_FORMAT,
-}
-
-PROGRESS_LIKERT_SCALE = {
-    "task_description": PROGRESS_LIKERT_SCALE_TASK,
-    "output_format": PROGRESS_LIKERT_SCALE_FORMAT,
-}
-
-PROGRESS_THREE_CLASS = {
-    "task_description": PROGRESS_THREE_CLASS_TASK,
-    "output_format": PROGRESS_THREE_CLASS_FORMAT,
-}
-
-PROGRESS_WITH_CHECKLIST = {
-    "task_description": PROGRESS_WITH_CHECKLIST_TASK,
-    "output_format": PROGRESS_WITH_CHECKLIST_FORMAT,
-}
-
-PROGRESS_WITH_CHECKLIST_IN_PROGRESS = {
-    "task_description": PROGRESS_WITH_CHECKLIST_IN_PROGRESS_TASK,
-    "output_format": PROGRESS_WITH_CHECKLIST_IN_PROGRESS_FORMAT,
-}
-
-PROGRESS_OURS = {
-    "task_description": PROGRESS_OURS_TASK,
-    "output_format": PROGRESS_OURS_FORMAT,
-}

agent/mini_bench/prompts/image_utils.py

@@ -1,19 +0,0 @@
-import base64
-import io
-from PIL import Image
-
-
-def image_to_base64_url(image: str | Image.Image):
-    if isinstance(image, str):
-        with open(image, "rb") as f:
-            image = f.read()
-    elif isinstance(image, Image.Image):
-        if image.mode in ("RGBA", "LA"):
-            image = image.convert("RGB")
-        with io.BytesIO() as buffer:
-            image.save(buffer, format="PNG")
-            image = buffer.getvalue()
-    else:
-        raise ValueError(f"Invalid image type: {type(image)}")
-    
-    return "data:image/png;base64," + base64.b64encode(image).decode("utf-8")

agent/mini_bench/prompts/input_information.py

@@ -1,36 +0,0 @@
-USER_INSTRUCTION = """## User Instruction
-{intent}
-"""
-
-TRAJECTORY = """## Trajectory
-{trajectory}"""
-
-AGENT_RESPONSE = """## Agent's Response
-THOUGHT: {thought}
-ACTION: {action}
-"""
-
-CHECKLIST = """## Checklist
-{checklist}
-"""
-
-
-# Observation
-CURRENT_URL = """### Current URL
-{current_url}
-"""
-
-TEXT_OBSERVATION = """### AXTREE
-Note: [bid] is the unique alpha-numeric identifier at the beginning of lines for each element in the AXTree. Always use bid to refer to elements in your actions.
-{text_observation}
-"""
-
-SOM_IMAGE_OBSERVATION = """### SOM Image Screenshot
-Here is a current image screenshot of the page, it is annotated with bounding boxes and corresponding bids:
-<IMAGE_PLACEHOLDER>
-"""
-
-COORD_IMAGE_OBSERVATION = """### Raw Image Screenshot
-Here is a screenshot of the page:
-<IMAGE_PLACEHOLDER>
-"""

agent/mini_bench/prompts/judge_prompt.py

@@ -1,159 +0,0 @@
-# SYSTEM PROMPT
-DEFAULT_SYSTEM_PROMPT_FORMAT = "You are an expert evaluator of web agent. {role_description}"
-
-PROGRESS_WITHOUT_CHECKLIST_ROLE = "Your task is to assess how helpful a given agent's THOUGHT and ACTION is in making progress toward the user's goal, based on the current state of the webpage."
-PROGRESS_WITH_CHECKLIST_ROLE = "Your task is to assess how helpful a given agent's THOUGHT and ACTION is in making progress toward the user's goal, based on the current state of the webpage."
-
-GROUNDING_ROLE = "Your task is to assess whether the ACTION taken by the agent is properly grounded, based on agent's THOUGHT and the current state of the webpage."
-
-# USER PROMPT
-DEFAULT_USER_PROMPT_FORMAT = """# Action space:
-{action_space}
-
-# Task Description
-{task_description}
-
-# Given Information
-{input_information}
-
-# Output Format
-{output_format}
-"""
-
-
-JUDGE_OURS_WO_CHECKLIST_USER_PROMPT_FORMAT = """You are an expert evaluator of web agent. Your task is to assess how helpful a given agent's THOUGHT and ACTION is in making progress toward the user's goal, based on the current state of the webpage.
-
-# Task Description
-Evaluate how well the agent’s THOUGHT and ACTION satisfy each item in the checklist using the task instruction, trajectory (including previously completed steps), current webpage state, and the agent’s latest response. Start by writing a concise paragraph summarizing the agent’s overall performance. Refer to the reasoning provided in the trajectory, and discuss whether the THOUGHT is appropriate and the ACTION moves the task forward.
-
-# Given Information
-{input_information}
-"""
-
-
-JUDGE_OURS_USER_PROMPT_FORMAT = """You are an expert evaluator of web agent. Your task is to assess how helpful a given agent's THOUGHT and ACTION is in making progress toward the user's goal, based on the current state of the webpage.
-
-# Task Description
-Evaluate how well the agent’s THOUGHT and ACTION satisfy each item in the checklist using the task instruction, trajectory (including previously completed steps), current webpage state, and the agent’s latest response. Start by writing a concise paragraph summarizing the agent’s overall performance. Refer to the reasoning provided in the trajectory, and discuss whether the THOUGHT is appropriate and the ACTION moves the task forward.
-Then, assess each checklist item individually using the following labels:
-- Yes: The item is fully and clearly satisfied, either in the current response or previously completed.
-- In Progress: There is meaningful partial progress toward completing the item.
-- No: The item is not satisfied due to ambiguity, insufficient evidence, or lack of progress.
-
-# Given Information
-{input_information}
-"""
-
-
-JUDGE_OURS_BT_MODELING_USER_PROMPT_FORMAT = """You are an expert web agent that browses internet via GUI actions. Your task is to achieve the user's goal described in the user instruction.
-
-# Task Description
-Generate the most appropriate GUI action to achieve the user's goal. When choosing your action, consider the current webpage state and the checklist which can be interpreted as subtasks.
-
-# Given Information
-## User Instruction
-{intent}
-
-## Trajectory
-{trajectory}
-
-## Current State
-### Current URL
-{current_url}
-
-### AXTREE
-Note: [bid] is the unique alpha-numeric identifier at the beginning of lines for each element in the AXTree. Always use bid to refer to elements in your actions.
-{text_observation}
-
-## Checklist
-{checklist}
-
-## Agent's Response
-"""
-
-JUDGE_OURS_BT_MODELING_BASE_PROMPT = """You are an expert web agent that browses internet via GUI actions. Your task is to achieve the user's goal described in the user instruction.
-
-# Task Description
-Generate the most appropriate GUI action to achieve the user's goal. When choosing your action, consider the current webpage state and the checklist which can be interpreted as subtasks.
-
-# Given Information
-## User Instruction
-{intent}
-
-## Trajectory
-{trajectory}
-
-## Current State
-### Current URL
-{current_url}
-
-### AXTREE
-Note: [bid] is the unique alpha-numeric identifier at the beginning of lines for each element in the AXTree. Always use bid to refer to elements in your actions.
-{text_observation}
-"""
-
-JUDGE_OURS_IMAGE_INPUT = """
-### Image Screenshot
-<IMAGE_PLACEHOLDER>
-"""
-
-JUDGE_OURS_WITH_CHECKLIST = """
-## Checklist
-{checklist}
-"""
-
-BT_MODELING_RESPONSE_FORMAT = """
-THOUGHT: {thought}
-ACTION: {action}
-"""
-
-## PROMPT TEMPLATE
-JUDGE_GROUNDING_PROMPT_TEMPLATE = {
-    "system": DEFAULT_SYSTEM_PROMPT_FORMAT.format(role_description=GROUNDING_ROLE),
-    "user": DEFAULT_USER_PROMPT_FORMAT,
-}
-
-JUDGE_LIKERT_SCALE_PROMPT_TEMPLATE = {
-    "system": DEFAULT_SYSTEM_PROMPT_FORMAT.format(role_description=PROGRESS_WITHOUT_CHECKLIST_ROLE),
-    "user": DEFAULT_USER_PROMPT_FORMAT
-}
-
-JUDGE_THREE_CLASS_PROMPT_TEMPLATE = {
-    "system": DEFAULT_SYSTEM_PROMPT_FORMAT.format(role_description=PROGRESS_WITHOUT_CHECKLIST_ROLE),
-    "user": DEFAULT_USER_PROMPT_FORMAT
-}
-
-JUDGE_WITH_CHECKLIST_PROMPT_TEMPLATE = {
-    "system": DEFAULT_SYSTEM_PROMPT_FORMAT.format(role_description=PROGRESS_WITH_CHECKLIST_ROLE),
-    "user": DEFAULT_USER_PROMPT_FORMAT
-}
-
-JUDGE_OURS_PROMPT_TEMPLATE = {
-    "system": "",
-    "user": JUDGE_OURS_USER_PROMPT_FORMAT,
-}
-
-JUDGE_OURS_WO_CHECKLIST_PROMPT_TEMPLATE = {
-    "system": "",
-    "user": JUDGE_OURS_WO_CHECKLIST_USER_PROMPT_FORMAT,
-}
-
-JUDGE_OURS_BT_MODELING_PROMPT_TEMPLATE = {
-    "user": JUDGE_OURS_BT_MODELING_BASE_PROMPT+JUDGE_OURS_WITH_CHECKLIST+"\n## Agent's Response\n",
-    "assistant": BT_MODELING_RESPONSE_FORMAT,
-}
-
-JUDGE_OURS_BT_MODELING_MULTIMODAL_PROMPT_TEMPLATE = {
-    "user": JUDGE_OURS_BT_MODELING_BASE_PROMPT+JUDGE_OURS_IMAGE_INPUT+JUDGE_OURS_WITH_CHECKLIST+"\n## Agent's Response\n",
-    "assistant": BT_MODELING_RESPONSE_FORMAT,
-}
-
-JUDGE_OURS_BT_MODELING_WO_CHECKLIST_PROMPT_TEMPLATE = {
-    "user": JUDGE_OURS_BT_MODELING_BASE_PROMPT+"\n## Agent's Response\n",
-    "assistant": BT_MODELING_RESPONSE_FORMAT,
-}
-
-JUDGE_OURS_BT_MODELING_MULTIMODAL_WO_CHECKLIST_PROMPT_TEMPLATE = {
-    "user": JUDGE_OURS_BT_MODELING_BASE_PROMPT+JUDGE_OURS_IMAGE_INPUT+"\n## Agent's Response\n",
-    "assistant": BT_MODELING_RESPONSE_FORMAT,
-}

agent/mini_bench/prompts/utils.py

@@ -1,18 +0,0 @@
-from langchain.schema import HumanMessage, AIMessage, SystemMessage
-
-def convert_dict_messages(dict_messages):
-    message_objs = []
-    for msg in dict_messages:
-        role = msg.get("role")
-        content = msg.get("content", "")
-        
-        if role == "user":
-            message_objs.append(HumanMessage(content=content))
-        elif role == "assistant":
-            message_objs.append(AIMessage(content=content))
-        elif role == "system":
-            message_objs.append(SystemMessage(content=content))
-        else:
-            raise ValueError(f"Unknown role: {role}")
-    
-    return message_objs

agent/mini_bench/reward_agent.py

@@ -1,465 +0,0 @@
-from abc import ABC, abstractmethod
-import time
-import requests
-import json
-import math
-from langsmith import Client
-import numpy as np
-from langchain_openai import ChatOpenAI
-
-from .prompts import get_messages
-from .prompts.judge_prompt import (
-    JUDGE_OURS_BT_MODELING_PROMPT_TEMPLATE,
-    JUDGE_OURS_BT_MODELING_WO_CHECKLIST_PROMPT_TEMPLATE,
-    JUDGE_OURS_BT_MODELING_MULTIMODAL_PROMPT_TEMPLATE,
-    JUDGE_OURS_BT_MODELING_MULTIMODAL_WO_CHECKLIST_PROMPT_TEMPLATE
-)
-from .prompts.image_utils import image_to_base64_url
-from .prompts.utils import convert_dict_messages
-
-MAX_RETRY = 3
-RETRY_SLEEP = 5
-MODEL_COST_MAPPING = {
-    "gpt-4o-mini": {
-        "input_token_cost": 0.15,
-        "output_token_cost": 0.6
-    },
-    "gpt-4o": {
-        "input_token_cost": 2.5,
-        "output_token_cost": 10
-    },
-}
-
-
-class Agent(ABC):
-    @abstractmethod
-    def generate_response(self, inputs: dict) -> str:
-        pass
-
-class BaseAgent(Agent):
-    def __init__(self, agent_config: dict):
-        self.agent_config = agent_config
-        self._setup()
-
-    def _init_llm_object(self, **extra_kwargs):
-        config = self.agent_config
-        config.update(extra_kwargs)
-        
-        use_log_probs = config.get("use_log_probs", False)
-        if use_log_probs:
-            self.llm = ChatOpenAI(
-                model=config["model_name"], 
-                base_url=config["base_url"], 
-                api_key=config["api_key"], 
-                temperature=config["temperature"],
-                timeout=300,
-                logprobs=True,
-                top_logprobs=10,
-                n=config.get('n', None)
-            )
-        else:
-            self.llm = ChatOpenAI(
-                model=config["model_name"], 
-                base_url=config["base_url"], 
-                api_key=config["api_key"], 
-                temperature=config["temperature"],
-                timeout=300,
-                n=config.get('n', None)
-            )
-    
-    def _setup(self):
-        self._init_llm_object()
-        
-        self.temperature = self.agent_config["temperature"]
-        self.num_generate = self.agent_config["num_generate"]
-        self.use_checklist = self.agent_config.get("use_checklist", False)
-        self.use_multimodal = self.agent_config.get("use_multimodal", False)
-
-        # setup cost
-        model_cost = MODEL_COST_MAPPING.get(self.agent_config["model_name"], None)
-        if model_cost and "api" in self.agent_config["base_url"]:
-            self.input_token_cost = model_cost["input_token_cost"]
-            self.output_token_cost = model_cost["output_token_cost"]
-        else:
-            self.input_token_cost = 0.0
-            self.output_token_cost = 0.0
-
-    def generate_with_retry(self, model_input, constraint_str_list: list = None):
-        total_input_tokens = 0
-        total_output_tokens = 0
-        if self.temperature == 0:
-            response = self.llm.invoke(model_input)
-            total_input_tokens += response.response_metadata["token_usage"]["prompt_tokens"]
-            total_output_tokens += response.response_metadata["token_usage"]["completion_tokens"]
-        else:
-            for i in range(MAX_RETRY):
-                try:
-                    response = self.llm.invoke(model_input)
-                    total_input_tokens += response.response_metadata["token_usage"]["prompt_tokens"]
-                    total_output_tokens += response.response_metadata["token_usage"]["completion_tokens"]
-                    if constraint_str_list:
-                        pass_constraint_num = 0
-                        for constraint_str in constraint_str_list:
-                            if constraint_str in response.content:
-                                pass_constraint_num += 1
-                        if pass_constraint_num == len(constraint_str_list):
-                            break
-                        else:
-                            print(f"Agent has fomat issue, retry... {i+1}/{MAX_RETRY}")
-                    else:
-                        break
-                except Exception as e:
-                    print(f"Agent returned an Error: {e}")
-                    response = None
-                    time.sleep(RETRY_SLEEP)
-        
-        cost = self.input_token_cost * total_input_tokens / 1000000 + self.output_token_cost * total_output_tokens / 1000000
-        
-        if response is None:
-            return "", cost
-        else:
-            return response.content, cost
-    
-    def prepare_message(self, model_input: dict, prompt_type: str):
-        message = []
-        return message
-
-    def generate_response(self, model_input: dict, prompt_type: str, constraint_str_list: list = None,):
-        total_cost = 0
-        response_list = []
-        # prepare message
-        message = self.prepare_message(model_input, prompt_type)
-
-        # n sampling
-        for i in range(self.num_generate):
-            response, cost = self.generate_with_retry(message, constraint_str_list)
-            response_list.append(response)
-            total_cost += cost
-        
-        return response_list, total_cost
-
-
-class GroundingJudgeAgent(BaseAgent):
-    def __init__(self, agent_config: dict):
-        super().__init__(agent_config)
-        self._setup()
-    
-    def prepare_message(self, model_input: dict, prompt_type):
-        message = get_messages(
-            input_info=model_input,
-            inference_mode="judge_grounding",
-            prompt_type=prompt_type,
-            use_multimodal=self.use_multimodal,
-            text_obs=self.agent_config["text_obs_type"],
-            image_obs=self.agent_config["image_obs_type"]
-        )
-        return message
-
-
-class ProgressJudgeAgent(BaseAgent):
-    def __init__(self, agent_config: dict):
-        super().__init__(agent_config)
-        self._setup()
-    
-    def prepare_message(self, model_input: dict, prompt_type):
-        if self.agent_config["input_type"]=="text_only":
-            use_multimodal = False
-            text_obs = self.agent_config["text_obs_type"]
-            image_obs = None
-        elif self.agent_config["input_type"]=="image_only":
-            use_multimodal = True
-            text_obs = None
-            image_obs = self.agent_config["image_obs_type"]
-        elif self.agent_config["input_type"]=="text_image":
-            use_multimodal = True
-            text_obs = self.agent_config["text_obs_type"]
-            image_obs = self.agent_config["image_obs_type"]
-        else:
-            raise ValueError(f"Invalid input type: {self.agent_config['input_type']}")
-        
-        if self.agent_config["use_in_progress"]:
-            use_in_progress = True
-        else:
-            use_in_progress = False
-
-        message = get_messages(
-            input_info=model_input,
-            inference_mode="judge_progress",
-            prompt_type=prompt_type,
-            use_checklist=self.use_checklist,
-            use_multimodal=use_multimodal,
-            text_obs=text_obs,
-            image_obs=image_obs,
-            use_in_progress=use_in_progress
-        )
-        return message
-    
-    def get_judge_probs(self, logprobs: list):
-        # target_judge = {
-        #     "yes": [" Yes", "Yes", "ĠYes", "ĊYes"],
-        #     "no": [" No", "No", "ĠNo", "ĊNo"],
-        #     "in": [" In", "In", "ĠIn", "ĊIn"]
-        # }
-        target_judge = {
-            "yes": [
-                "ĠYes", "Yes", "ĊYes",
-                "Ġyes", "yes", "Ċyes",
-                "ĠYES", "YES", "ĊYES",
-                "ĠDone", "Done", "ĊDone",
-                "ĠCompleted", "Completed", "ĊCompleted",
-                "ĠCorrect", "Correct", "ĊCorrect"
-            ],
-            "no": [
-                "ĠNo", "No", "ĊNo",
-                "ĠNO", "NO", "ĊNO",
-                "ĠNot", "Not", "ĊNot",
-                "ĠNone", "None", "ĊNone",
-                "ĠNope", "Nope", "ĊNope",
-                "ĠUn", "Un", "ĊUn",
-                "ĠWrong", "Wrong", "ĊWrong"
-            ],
-            "in": [
-                "ĠIn", "In", "ĊIn",
-                "ĠPending", "Pending", "ĊPending",
-                "ĠPart", "Part", "ĊPart",
-                "ĠPartial", "Partial", "ĊPartial",
-                "ĠInProgress", "InProgress", "ĊInProgress"
-            ]
-        }
-        response_str = ""
-        judge_probs_list = []
-        for i, log_prob in enumerate(logprobs):
-            # Start to find judge string
-            if "<answer>" in response_str:
-                find_judge_str = False
-                for judge_type in target_judge:
-                    if log_prob["token"] in target_judge[judge_type]:
-                        # print(log_prob)
-                        find_judge_str = True
-                        break
-                if find_judge_str:
-                    token_judge_dict = {
-                        "yes": None,
-                        "no": None,
-                        "in": None
-                    }
-                    for token_info in log_prob["top_logprobs"]:
-                        for judge_type in target_judge:
-                            for judge_str in target_judge[judge_type]:
-                                if judge_str in token_info["token"] :
-                                    if token_judge_dict[judge_type] is None:
-                                        token_judge_dict[judge_type] = math.exp(token_info["logprob"])
-                                    else:
-                                        token_judge_dict[judge_type] += math.exp(token_info["logprob"])
-
-                    token_judge_dict = {
-                        "yes": math.log(token_judge_dict["yes"]) if token_judge_dict["yes"] is not None else -float('inf'),
-                        "no": math.log(token_judge_dict["no"]) if token_judge_dict["no"] is not None else -float('inf'),
-                        "in": math.log(token_judge_dict["in"]) if token_judge_dict["in"] is not None else -float('inf')
-                    }
-                    judge_probs_list.append(token_judge_dict)
-            
-            if "</answer>" in response_str:
-                break
-            
-            response_str += log_prob["token"]
-
-        if len(judge_probs_list) == 0:
-            return [{
-                "yes": 0.0,
-                "no": 0.0,
-                "in": 0.0
-            }]
-        else:
-            # convert with softmax
-            final_judge_probs_list = []
-            max_in_prob = -float('inf')
-            for idx, judge_probs in enumerate(judge_probs_list):
-                exp_logprobs = [math.exp(x) for x in [judge_probs["yes"], judge_probs["no"], judge_probs["in"]]]
-                sum_exp_logprobs = sum(exp_logprobs)
-                softmax_probs = [x / sum_exp_logprobs for x in exp_logprobs]
-                if softmax_probs[2] > max_in_prob:
-                    max_in_prob = softmax_probs[2]
-                final_judge_probs_list.append({
-                    "yes": softmax_probs[0], 
-                    "no": softmax_probs[1],
-                    "in": softmax_probs[2]
-                })
-            return final_judge_probs_list
-    
-    def generate_probs(self, model_input: dict, prompt_type: str, n=1, temperature=None):
-        total_cost = 0
-        # prepare message
-        message = self.prepare_message(model_input, prompt_type)
-        messages = convert_dict_messages(message)
-
-        kwargs = {'n': n}
-        if temperature is not None:
-            kwargs['temperature'] = temperature
-        self._init_llm_object(**kwargs)
-        
-        try:
-            response = self.llm.generate([messages])  # assume single batch
-        finally:
-            print('request url: ', self.agent_config['base_url'])
-
-        
-        # parse responses
-        response_list = []
-        for generation in response.generations[0]:  # assume singel batch
-            # parse logprobs
-            logprobs = generation.message.response_metadata["logprobs"]["content"]
-            response_list.append(
-                {
-                    "response": generation.message.content,
-                    "judge_probs": self.get_judge_probs(logprobs)
-                }
-            )
-
-        # calculate cost
-        total_input_tokens = response.llm_output["token_usage"]["prompt_tokens"]
-        total_output_tokens = response.llm_output["token_usage"]["completion_tokens"]
-        total_cost = self.input_token_cost * total_input_tokens / 1000000 + self.output_token_cost * total_output_tokens / 1000000  
-        
-        return response_list, total_cost
-
-
-class ChecklistGenerationAgent(BaseAgent):
-    def __init__(self, agent_config: dict):
-        super().__init__(agent_config)
-        self._setup()
-    
-    def prepare_message(self, model_input: dict, prompt_type):
-        message = get_messages(
-            input_info=model_input,
-            inference_mode="checklist_generation",
-            prompt_type=prompt_type
-        )
-        return message
-    
-
-class ClassifierRewardAgent(Agent):
-    def __init__(self, url: str, use_checklist: bool = False, use_multimodal: bool = False):
-        self.url = url
-        self.use_checklist = use_checklist
-        self.use_multimodal = use_multimodal
-
-    def _process_multimodal_message(self, prompt: str, image_list: list[str]):
-        multimodal_message = []
-        text_prompt_prefix = prompt.split("<IMAGE_PLACEHOLDER>")[0]
-        text_prompt_suffix = prompt.split("<IMAGE_PLACEHOLDER>")[1]
-        multimodal_message = [
-            {"type": "text", "text": text_prompt_prefix},
-            # {"type": "image_url", "image_url": {"url": image_to_base64_url(image_list[0])}},
-            {"type": "image", "image": image_to_base64_url(image_list[0])},
-            {"type": "text", "text": text_prompt_suffix}
-        ]
-        return multimodal_message
-
-    def _make_query(self, user_prompt_template: dict, model_input: dict | list[dict]):
-        if self.use_multimodal:
-            tmp_user_prompt = user_prompt_template["user"].format(
-                **model_input
-            )
-            user_prompt =  self._process_multimodal_message(tmp_user_prompt, model_input["image_list"])
-        else:
-            user_prompt = user_prompt_template["user"].format(
-                **model_input
-            )
-        assistant_prompt = user_prompt_template["assistant"].format(
-            **model_input
-        )
-        query = [
-            {"role": "user", "content": user_prompt},
-            {"role": "assistant", "content": assistant_prompt}
-        ]
-        return query
-
-    def prepare_message(self, model_input: dict | list[dict], batch: bool = False):
-        if self.use_checklist:
-            if self.use_multimodal:
-                user_prompt_template = JUDGE_OURS_BT_MODELING_MULTIMODAL_PROMPT_TEMPLATE
-            else:
-                user_prompt_template = JUDGE_OURS_BT_MODELING_PROMPT_TEMPLATE
-        else:
-            if self.use_multimodal:
-                user_prompt_template = JUDGE_OURS_BT_MODELING_MULTIMODAL_WO_CHECKLIST_PROMPT_TEMPLATE
-            else:
-                user_prompt_template = JUDGE_OURS_BT_MODELING_WO_CHECKLIST_PROMPT_TEMPLATE
-
-        if self.use_multimodal:
-            if batch:
-                message = [self._make_query(user_prompt_template, input) for input in model_input]
-            else:
-                message = [self._make_query(user_prompt_template, model_input)]
-        else:
-            if batch:
-                message = {
-                    "query": [self._make_query(user_prompt_template, input) for input in model_input],
-                    "promptts": []
-                }
-            else:
-                message = {
-                    "query": self._make_query(user_prompt_template, model_input),
-                    "prompts": []
-                }
-
-        return message
-    
-    def get_rm_scroe(self, message: dict | list):
-        headers = {"Content-Type": "application/json"}
-        
-        try:
-            if self.use_multimodal:
-                response = requests.post(
-                    self.url,
-                    json={"messages": message},
-                    timeout=600
-                )
-            else:
-                response = requests.post(
-                    self.url,
-                    headers=headers,
-                    data=json.dumps(message),
-                    timeout=300
-                )
-            response.raise_for_status()
-            
-            response_json = response.json()
-
-            if "rewards" not in response_json:
-                print(f"Error: 'rewards' key not found in API response: {response_json}")
-                return []
-            
-            if "get_reward" in self.url:
-                # use openrlhf
-                return response_json["rewards"]
-            elif "pooling" in self.url:
-                # use vllm server
-                return response_json["reward"]
-            else:
-                # error
-                raise ValueError(f"Invalid URL: {self.url}")
-
-        except requests.exceptions.Timeout:
-            print(f"Error: Request timed out to {self.url}")
-            return []
-        except requests.exceptions.RequestException as e:
-            print(f"Error during request to {self.url}: {e}")
-            return []
-        except json.JSONDecodeError:
-            print(f"Error: Failed to decode JSON response from {self.url}")
-            return []
-        except KeyError as e:
-             print(f"Error: Missing key {e} in response from {self.url}")
-             return []
-
-
-    def generate_response(self, model_input: dict | list[dict], batch: bool = False):
-        if batch:
-            message = self.prepare_message(model_input, batch=True)
-        else:
-            message = self.prepare_message(model_input)
-        rewards = self.get_rm_scroe(message)
-        
-        return rewards, 0

agent/mini_bench/utils.py

@@ -1,269 +0,0 @@
-import json
-import base64
-import io
-import html
-from PIL import Image
-
-
-def image_to_base64_url(image: str | Image.Image):
-    if isinstance(image, str):
-        with open(image, "rb") as f:
-            image = f.read()
-    elif isinstance(image, Image.Image):
-        if image.mode in ("RGBA", "LA"):
-            image = image.convert("RGB")
-        with io.BytesIO() as buffer:
-            image.save(buffer, format="PNG")
-            image = buffer.getvalue()
-    else:
-        raise ValueError(f"Invalid image type: {type(image)}")
-    
-    return "data:image/png;base64," + base64.b64encode(image).decode("utf-8")
-
-
-def load_json(file_path: str) -> dict:
-    with open(file_path, "r") as f:
-        return json.load(f)
-    
-def save_json(data: dict, file_path: str):
-    with open(file_path, "w") as f:
-        json.dump(data, f, indent=4)
-
-def str_to_bool(s: str) -> bool:
-    if s.lower() in ["true", "1", "yes", "y"]:
-        return True
-    elif s.lower() in ["false", "0", "no", "n"]:
-        return False
-    else:
-        raise ValueError(f"Invalid boolean string: {s}")
-    
-
-def create_html_report(json_path, html_path, checklist_generation=False):
-    """
-    Reads the given JSON result file and generates a filterable HTML report.
-
-    Args:
-        json_path (str): Path to the input JSON file.
-        html_path (str): Path to the output HTML file.
-    """
-    try:
-        with open(json_path, 'r', encoding='utf-8') as f:
-            data = json.load(f)
-    except FileNotFoundError:
-        print(f"Error: JSON file not found - {json_path}") # Error message in English
-        return
-    except json.JSONDecodeError:
-        print(f"Error: JSON file parsing error - {json_path}") # Error message in English
-        return
-    except Exception as e:
-        print(f"Unexpected error during data loading: {e}") # Error message in English
-        return
-
-    # Extract unique Task IDs and sort them
-    task_ids = sorted(list(set(item.get("task_id") for item in data if item.get("task_id") is not None)))
-
-    html_content = """
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Benchmark Results Report</title>
-    <style>
-        body { font-family: sans-serif; line-height: 1.6; padding: 20px; }
-        .task-step { border: 1px solid #ccc; margin-bottom: 20px; padding: 15px; border-radius: 5px; background-color: #f9f9f9; }
-        .task-step h2 { margin-top: 0; color: #333; border-bottom: 1px solid #eee; padding-bottom: 5px;}
-        .task-step h3 { color: #555; margin-top: 15px; margin-bottom: 5px; }
-        .task-step h4 { color: #777; margin-top: 10px; margin-bottom: 5px; font-style: italic;}
-        pre { background-color: #eee; padding: 10px; border-radius: 3px; white-space: pre-wrap; word-wrap: break-word; font-size: 0.9em; margin-top: 5px; }
-        details { margin-top: 10px; border: 1px solid #ddd; border-radius: 3px; background-color: #fff; }
-        summary { cursor: pointer; padding: 8px; background-color: #f8f9fa; font-weight: bold; border-bottom: 1px solid #ddd; }
-        details[open] summary { border-bottom: 1px solid #ddd; }
-        details > pre { border: none; background-color: #fff; padding: 10px 8px; }
-        .response-item-toggle { margin-top: 10px; }
-        .chosen-section { border-left: 5px solid #4CAF50; padding-left: 10px; margin-top: 15px; }
-        .rejected-section { border-left: 5px solid #f44336; padding-left: 10px; margin-top: 15px; }
-        hr { border: 0; border-top: 1px solid #eee; margin: 15px 0; }
-        .thought-action { background-color: #f0f0f0; padding: 10px; border-radius: 3px; margin-bottom: 10px; border: 1px solid #e0e0e0;}
-        .thought-action h4 { margin-top: 0; color: #666; }
-        .task-container { display: none; }
-        .filter-controls { margin-bottom: 20px; padding: 10px; background-color: #e9ecef; border-radius: 5px; }
-        .filter-controls label { margin-right: 10px; font-weight: bold; }
-        .filter-controls select { padding: 5px; border-radius: 3px; border: 1px solid #ced4da; }
-    </style>
-</head>
-<body>
-    <h1>Benchmark Results Report</h1>
-
-    <!-- Task ID Filter Dropdown -->
-    <div class="filter-controls">
-        <label for="taskSelector">Select Task ID:</label>
-        <select id="taskSelector">
-            <option value="">-- Show All --</option>
-"""
-    # Add dropdown options
-    for tid in task_ids:
-        html_content += f'            <option value="{html.escape(str(tid))}">{html.escape(str(tid))}</option>\n'
-
-    html_content += """
-        </select>
-    </div>
-
-    <!-- Results Display Area -->
-    <div id="resultsArea">
-"""
-
-    # Process each Task/Step data
-    for i, step_data in enumerate(data):
-        task_id = step_data.get("task_id", "N/A")
-        step_id = step_data.get("step_id", "N/A")
-        intent = step_data.get("intent", "N/A")
-        start_url = step_data.get("start_url", "N/A")
-        gt_checklist = step_data.get("gt_checklist", "N/A")
-        generated_checklist = step_data.get("generated_checklist", None)
-        trajectory = step_data.get("trajectory", "N/A")
-        text_observation = step_data.get("text_observation", "N/A")
-        source_name = step_data.get("source_name", "")
-
-        # Wrap each Task/Step in a container with a unique ID (hidden initially)
-        html_content += f"""
-    <div class="task-container" data-task-id="{html.escape(str(task_id))}">
-        <div class="task-step">
-            <h2>Task ID: {html.escape(str(task_id))} | Step ID: {html.escape(str(step_id))} {f'({html.escape(source_name)})' if source_name else ''}</h2>
-            <h3>Intent:</h3>
-            <p>{html.escape(intent)}</p>
-            <p><strong>Start URL:</strong> <a href="{html.escape(start_url)}" target="_blank">{html.escape(start_url)}</a></p>
-
-            <h3>Ground Truth Checklist:</h3>
-            <pre>{html.escape(gt_checklist)}</pre>
-"""
-        if checklist_generation and generated_checklist is not None:
-            html_content += f"""
-            <details>
-                <summary>Generated Checklist (Click to expand/collapse)</summary>
-                <pre>{html.escape(str(generated_checklist))}</pre>
-            </details>
-"""
-
-        html_content += f"""
-            <details>
-                <summary>Trajectory (Click to expand/collapse)</summary>
-                <pre>{html.escape(trajectory)}</pre>
-            </details>
-
-            <details>
-                <summary>Text Observation (Click to expand/collapse)</summary>
-                <pre>{html.escape(text_observation)}</pre>
-            </details>
-            <hr>
-"""
-
-        # Chosen Responses
-        if 'chosen' in step_data and step_data['chosen']:
-            html_content += '<div class="chosen-section"><h3>Chosen Responses:</h3>'
-            for choice_block in step_data['chosen']:
-                thought = choice_block.get('thought', 'N/A')
-                action = choice_block.get('action', 'N/A')
-                responses = choice_block.get('response', [])
-                scores = choice_block.get('score', [])
-
-                # Add Thought and Action information
-                html_content += f"""
-            <div class="thought-action">
-                <h4>Thought:</h4>
-                <pre>{html.escape(thought)}</pre>
-                <h4>Action:</h4>
-                <pre>{html.escape(action)}</pre>
-            </div>"""
-
-                # Loop through responses and create toggles
-                for idx, (response, score) in enumerate(zip(responses, scores)):
-                     html_content += f"""
-            <details class="response-item-toggle">
-                <summary>Judge Response {idx + 1}: {html.escape(str(score))}</summary>
-                <pre>{html.escape(str(response))}</pre>
-            </details>"""
-            html_content += '</div>' # End chosen-section
-
-        # Rejected Responses
-        if 'rejected' in step_data and step_data['rejected']:
-            html_content += '<div class="rejected-section"><h3>Rejected Responses:</h3>'
-            for rejection_block in step_data['rejected']:
-                thought = rejection_block.get('thought', 'N/A')
-                action = rejection_block.get('action', 'N/A')
-                responses = rejection_block.get('response', [])
-                scores = rejection_block.get('score', [])
-
-                # Add Thought and Action information
-                html_content += f"""
-            <div class="thought-action">
-                <h4>Thought:</h4>
-                <pre>{html.escape(thought)}</pre>
-                <h4>Action:</h4>
-                <pre>{html.escape(action)}</pre>
-            </div>"""
-
-                # Loop through responses and create toggles
-                for idx, (response, score) in enumerate(zip(responses, scores)):
-                     html_content += f"""
-            <details class="response-item-toggle">
-                <summary>Judge Response {idx + 1}: {html.escape(str(score))}</summary>
-                <pre>{html.escape(str(response))}</pre>
-            </details>"""
-            html_content += '</div>' # End rejected-section
-
-        html_content += """
-        </div> <!-- End task-step -->
-    </div> <!-- End task-container -->
-"""
-
-    # Finalize HTML and add JavaScript
-    html_content += """
-    </div> <!-- End resultsArea -->
-
-    <script>
-        document.addEventListener('DOMContentLoaded', function() {
-            const taskSelector = document.getElementById('taskSelector');
-            const taskContainers = document.querySelectorAll('.task-container');
-
-            function filterTasks() {
-                const selectedTaskId = taskSelector.value;
-
-                taskContainers.forEach(container => {
-                    const containerTaskId = container.getAttribute('data-task-id');
-                    // Show if no Task ID is selected (Show All) or if the container's Task ID matches
-                    if (selectedTaskId === "" || containerTaskId === selectedTaskId) {
-                        container.style.display = 'block';
-                    } else {
-                        // Otherwise, hide it
-                        container.style.display = 'none';
-                    }
-                });
-            }
-
-            // Run filter function on dropdown change
-            taskSelector.addEventListener('change', filterTasks);
-
-            // Run initial filtering on page load (default: Show All)
-            filterTasks();
-        });
-    </script>
-
-</body>
-</html>
-"""
-
-    # Save the HTML file
-    try:
-        with open(html_path, 'w', encoding='utf-8') as f:
-            f.write(html_content)
-        print(f"Completed: HTML report created at {html_path}")
-    except IOError:
-        print(f"Error: Failed to write HTML file - {html_path}")
-    except Exception as e:
-        print(f"Unexpected error during HTML file saving: {e}")
-
-# --- Example Usage ---
-# input_json_file = 'path/to/your/results.json'
-# output_html_file = 'trajectory_report.html'
-# create_html_report(input_json_file, output_html_file)

agent/reward.py

@@ -1,96 +0,0 @@
-import time
-from typing import List, Dict, Any, Optional, Union
-import numpy as np
-from .mini_bench.reward_agent import ProgressJudgeAgent
-from .reward_postprocessor import REWARD_PROCESSORS, REWARD_PROCESSOR_N_SAMPLES, extract_judge_hash
-import json
-import os
-from concurrent.futures import ThreadPoolExecutor, as_completed
-
-def _process_unit(idx, unit, configs, n_samples, reward_processor, max_retries=5):
-    """하나의 unit을 처리해 (idx, reward, thought)를 돌려준다."""
-    agent = ProgressJudgeAgent(configs)
-    current_temperature = configs["temperature"]
-
-    rewards = []
-    n_err = 0
-    retry_count = 0
-    judge_hash_count_thought = {}
-
-    while len(rewards) < n_samples and retry_count < max_retries:
-        # 외부 API 호출
-        responses, _ = agent.generate_probs(
-            unit, "ours", n=n_samples - len(rewards), temperature=current_temperature
-        )
-
-        for response in responses:
-            content = response["response"]
-            thought = content                          # 전체를 로그로 저장
-            reward = REWARD_PROCESSORS[reward_processor](response)
-            rewards.append(reward)
-
-            if np.isnan(reward) or reward is None:
-                n_err += 1
-            else:
-                judge_hash = extract_judge_hash(response)
-                judge_hash_count_thought[judge_hash] = (judge_hash_count_thought.get(judge_hash, (0, None))[0] + 1, thought)
-
-        if n_err > 0:
-            # 실패 시 온도를 높여 재시도
-            if n_samples == 1:
-                current_temperature = 0.5
-        retry_count += 1
-
-    reward = np.nanmean(rewards)
-    if np.isnan(reward):
-        print(f"[idx={idx}] Warning: reward is NaN after retries -> set 0")
-        reward = 0.0
-    print(judge_hash_count_thought)
-    thought = max(judge_hash_count_thought.values(), key=lambda x: x[0])[1]
-
-    return idx, reward, thought
-
-
-def get_ar_reward(dataset, base_url, model_name, reward_processor='avg_logits', max_workers=8):
-    """원본 get_ar_reward를 스레드 버전으로 교체."""
-    n_samples = REWARD_PROCESSOR_N_SAMPLES[reward_processor]
-
-    temperature = 0.5 if n_samples > 1 else 0.0
-
-    configs = {
-        "model_name": model_name,
-        "base_url": base_url,
-        "api_key": "empty",
-        "temperature": temperature,
-        "num_generate": 1,
-        "use_checklist": True,
-        "input_type": "text_only",
-        "text_obs_type": "axtree",
-        "image_obs_type": "som",
-        "use_in_progress": True,
-        "use_multimodal": False,
-        "use_log_probs": True,
-    }
-
-    t_start = time.time()
-    results = [None] * len(dataset)
-
-    with ThreadPoolExecutor(max_workers=max_workers) as executor:
-        futures = [
-            executor.submit(
-                _process_unit, idx, unit, configs, n_samples, reward_processor
-            )
-            for idx, unit in enumerate(dataset)
-        ]
-
-        for fut in as_completed(futures):
-            idx, reward, thought = fut.result()
-            results[idx] = (reward, thought)
-
-    # 순서 보존된 리스트로 분리
-    final_rewards = [float(r) for r, _ in results]
-    thoughts = [t for _, t in results]
-
-    print(f"Time taken (threaded): {time.time() - t_start:.2f} s")
-    return final_rewards, thoughts
-

agent/reward_postprocessor.py

@@ -1,41 +0,0 @@
-import numpy as np
-import re
-
-
-def extract_judge_hash(response):
-    """
-    checklist 별로 yes, in, no를 판단한 정보를 hash 형태로 변환하여 반환
-    """
-    content = response['response']
-    
-    try:
-        judge_content = content.lower().replace(' ', '').split('<answer>')[1].split('</answer>')[0]
-    except:
-        import traceback
-        traceback.print_exc()
-        return None
-    pattern = r":yes|:inprogress|:no"
-    matches = re.findall(pattern, judge_content)
-    matches = [{':yes': 'y', ':inprogress': 'i', ':no': 'n'}[match] for match in matches]
-    return ''.join(matches)
-
-def average_logits(response):
-    """
-    yes, in, no를 logits 레벨에서 계산.
-    """
-    judge_probs = response['judge_probs']
-    
-    yes_ = np.mean([r['yes'] for r in judge_probs])
-    in_ = np.mean([r['in'] for r in judge_probs])
-    
-    reward = yes_ + 0.5 * in_
-    return reward
-
-
-REWARD_PROCESSORS = {
-    'avg_logits': average_logits
-}
-
-REWARD_PROCESSOR_N_SAMPLES = {
-    'avg_logits': 5
-}