update

.gitignore

@@ -5,7 +5,8 @@
 #/data/
 /data/comment
 #/data/eval_data
-data/llm-log
+data/llm-log-hk
+data/llm-log-usa
 /data/raw_dataset
 /dotenv/
 /logs/

data/dataset/agent-bingoplus-ph-200-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc11589ee643a5c50e9d738d32c87aae9a640fbb843fe3254890506ebeaea70b
+size 2174134

data/dataset/agent-bingoplus-ph-90-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e8c67a27e9765f42a66c02068d0c1caa5559e60a77497ab5ed61bb24f2955463
+size 247260

data/eval_data/azure_openai/azure/gpt-4o-mini/shenzhen_sase/west_us_chatgpt_openai_azure_com/20250729-interval-5/agent-bingoplus-ph-200-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf3cb56ef48b63f3d55804f97b7f80869613f4b3c2caef8c6300f970da5f625d
+size 2406951

data/eval_data/azure_openai/azure/gpt-4o-mini/shenzhen_sase/west_us_chatgpt_openai_azure_com/20250729-interval-5/agent-bingoplus-ph-90-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:112775fc79e439d1a30a69b7631c1d6e25126021c5b0a6eaccdee4754c37e3dd
+size 258526

data/eval_data/azure_openai/azure/gpt-4o/shenzhen_sase/west_us_chatgpt_openai_azure_com/20250729-interval-5/agent-bingoplus-ph-200-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cafe1c9ab785683678afee5c81dc1c9795c5ab27a625a9d7a2439bb23880e318
+size 2411012

data/eval_data/azure_openai/azure/gpt-4o/shenzhen_sase/west_us_chatgpt_openai_azure_com/20250729-interval-5/agent-bingoplus-ph-90-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6d962451c175761ff7e6bf937d31fe46f928d6ad4aecb67afcb7e6164341647
+size 258555

data/eval_data/byteplus/byteplus/seed-1-6-250615/shenzhen_sase/byteplus_api_key/20250728_113641/agent-lingoace-zh-400-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ff0d88e40a5c3b1a1979bf6713d5080b0d1c67592ca7bd6d0b94bb010ffd6df
+size 1206002

data/eval_data/byteplus/byteplus/seed-1-6-250615/shenzhen_sase/byteplus_api_key/20250728_113641/agent-lingoace-zh-80-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c9ad1b21c25c7d1c5aa7f8825dafe9bff0402d5c79f4cf0bf183b619ccb0bfb
+size 886441

data/eval_data/byteplus/byteplus/seed-1-6-250615/shenzhen_sase/byteplus_api_key/20250728_113641/arc-easy-1000-choice.jsonl

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49e98cb6d61aa488ab7182e77412ce5714fdb36cff9d90c48c380fc9a076163d
-size 397093
+oid sha256:3b16006362adf85a3240a10c71e93de98d04d32e94dba2ace1f38ee2839f7566
+size 704949

data/eval_data/byteplus/byteplus/seed-1-6-flash-250615/shenzhen_sase/byteplus_api_key/20250728_113641/agent-lingoace-zh-400-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39784ecb24d2a9659a81e48e17360e2ebd95ddf2e1b1f3f00d2266dba110c372
+size 1211567

data/eval_data/byteplus/byteplus/seed-1-6-flash-250615/shenzhen_sase/byteplus_api_key/20250728_113641/agent-lingoace-zh-80-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:418ff28b690bb2d76fbe80f3f2025ebe7c65db77f035b7e03034498ebaf0fa3b
+size 878141

data/eval_data/byteplus/byteplus/seed-1-6-flash-250615/shenzhen_sase/byteplus_api_key/20250728_113641/arc-easy-1000-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef2680774924bdf70e329cd81997b97845dccd1c7297ce7aa0c3c69f78675dbc
+size 719440

data/eval_data/gemini_google/google/gemini-2.5-flash-lite-preview-06-17/shenzhen_sase/google_potent_veld_462405_t3/20250729_161543/agent-bingoplus-ph-200-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1b4a41e437ee92ef98dce58783a44d4f609fbb646b1edddcd34dca2d6f31dc2
+size 2416223

data/eval_data/gemini_google/google/gemini-2.5-flash-lite-preview-06-17/shenzhen_sase/google_potent_veld_462405_t3/20250729_161543/agent-bingoplus-ph-90-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6f872a62f824b8624b4aabe824a3cb8e3044d9ba2946f5c2970d6917c9212441
+size 258575

data/eval_data/gemini_google/google/gemini-2.5-flash/shenzhen_sase/google_potent_veld_462405_t3/20250729_161416/agent-bingoplus-ph-200-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c396f32f93313f98a19bb7d59dcdc7fdce303605292dfe3f015f73f86202b0d7
+size 2418251

data/eval_data/gemini_google/google/gemini-2.5-flash/shenzhen_sase/google_potent_veld_462405_t3/20250729_161416/agent-bingoplus-ph-90-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11a3da8d355ebbbab1a504f705446b6e69daf8b0089b9cf6a40446d62f151451
+size 258600

data/eval_data/siliconflow/siliconflow/deepseek-ai#DeepSeek-V3/shenzhen_sase/siliconflow_api_key/20250729_162534/agent-bingoplus-ph-200-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d39dd0c55be4145d4cb27803a00a5c8fc70b83417664453d0166bcdab9be82bc
+size 2456608

data/eval_data/siliconflow/siliconflow/deepseek-ai#DeepSeek-V3/shenzhen_sase/siliconflow_api_key/20250729_162534/agent-bingoplus-ph-90-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a64a4eba8c60e5820029665c36b03a95ed2516bc8d98e9e13956d7a41eb16e7
+size 258487

examples/make_dataset/make_bingoplus_ph_200_chat.py

@@ -0,0 +1,73 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import os
+from pathlib import Path
+import sys
+import time
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--raw_dataset",
+        default=(project_path / "data/raw_dataset/finished/agent-bingoplus-ph-200-chat").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--dataset",
+        default=(project_path / "data/dataset/agent-bingoplus-ph-200-chat.jsonl").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    raw_dataset = Path(args.raw_dataset)
+    dataset = Path(args.dataset)
+    dataset.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(dataset.as_posix(), "w", encoding="utf-8") as fout:
+        for sample_dir in raw_dataset.glob("*"):
+            idx = sample_dir.parts[-1]
+            system_prompt_file = sample_dir / "system_prompt.txt"
+            user_prompt_file = sample_dir / "user_prompt.txt"
+            response_file = sample_dir / "response.txt"
+
+            with open(system_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+                system_prompt = f.read()
+            with open(user_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+                user_prompt = f.read()
+            with open(response_file.as_posix(), "r", encoding="utf-8") as f:
+                response = f.read()
+
+            if system_prompt.startswith("null\n"):
+                system_prompt = system_prompt[5:]
+
+            prompt = f"""{system_prompt}\n\n{user_prompt}""".strip()
+
+            print(f"{prompt}\n\n{response}")
+            print("-" * 150)
+
+            row_ = {
+                "idx": idx,
+                "prompt": prompt,
+                "response": response,
+            }
+            row_ = json.dumps(row_, ensure_ascii=False)
+            fout.write(f"{row_}\n")
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/make_dataset/make_bingoplus_ph_90_choice.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import os
+from pathlib import Path
+import sys
+import time
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../../"))
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--raw_dataset",
+        default=(project_path / "data/raw_dataset/finished/agent-bingoplus-ph-90-choice").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--dataset",
+        default=(project_path / "data/dataset/agent-bingoplus-ph-90-choice.jsonl").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    raw_dataset = Path(args.raw_dataset)
+    dataset = Path(args.dataset)
+    dataset.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(dataset.as_posix(), "w", encoding="utf-8") as fout:
+        for sample_dir in raw_dataset.glob("*"):
+            idx = sample_dir.parts[-1]
+            system_prompt_file = sample_dir / "system_prompt.txt"
+            user_prompt_file = sample_dir / "user_prompt.txt"
+            response_file = sample_dir / "response.txt"
+
+            with open(system_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+                system_prompt = f.read()
+            with open(user_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+                user_prompt = f.read()
+            with open(response_file.as_posix(), "r", encoding="utf-8") as f:
+                response = f.read()
+
+            prompt = f"""{system_prompt}\n\n{user_prompt}""".strip()
+
+            print(f"{prompt}\n\n{response}")
+            print("-" * 150)
+
+            row_ = {
+                "idx": idx,
+                "prompt": prompt,
+                "response": response,
+            }
+            row_ = json.dumps(row_, ensure_ascii=False)
+            fout.write(f"{row_}\n")
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/make_raw_dataset/step_1_make_hk_dataset_by_log.py

@@ -0,0 +1,195 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import hashlib
+from pathlib import Path
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--raw_log_dir",
+        default=(project_path / "data/llm-log-hk/raw-log").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--output_dir",
+        default=(project_path / "data/llm-log-hk/extract-dataset/raw").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+_escape_table = [chr(x) for x in range(128)]
+_escape_table[0] = "\\0"
+_escape_table[ord("\\")] = "\\\\\\\\"
+_escape_table[ord("\n")] = "\\n"
+_escape_table[ord("\r")] = "\\r"
+_escape_table[ord("\032")] = "\\Z"
+_escape_table[ord('"')] = '\\\\\\"'
+_escape_table[ord("'")] = "\\\\\\'"
+
+
+def escape_string(value, mapping=None):
+    """escapes *value* without adding quote.
+
+    Value should be unicode
+    """
+    return value.translate(_escape_table)
+
+
+def unescape_string(value: str) -> str:
+    _unescape_map = [
+        (r"\"", "\""),
+        (r"\\", "\\"),
+        (r"\\n", r"\n"),
+        (r"\\u003c", "<"),
+        (r"\\u003e", ">"),
+    ]
+    for k, v in _unescape_map:
+        value = value.replace(k, v)
+    return value
+
+def unescape_string2(value: str) -> str:
+    _unescape_map = [
+        ("\n", r"\n"),
+        ("\\n", "\n"),
+    ]
+    for k, v in _unescape_map:
+        value = value.replace(k, v)
+    return value
+
+
+def extract(text: str):
+    row = json.loads(text)
+    content = row["__CONTENT__"]
+
+    splits = content.split('>', maxsplit=1)
+    text = splits[-1]
+
+    splits = text[::-1].split('|', maxsplit=1)
+    last_split: str = splits[0][::-1]
+    first_six_splits = splits[1][::-1].split('|', maxsplit=5)
+    splits = first_six_splits + [last_split]
+
+    len_of_splits = len(splits)
+
+    engine = splits[0].strip()
+    call_id = splits[3]
+    if len(call_id) == 0:
+        call_id = splits[4]
+    prompt = splits[5]
+    prompt = json.loads(prompt)
+
+    system_prompt = prompt["SystemPrompt"]
+
+    conversation = prompt.get("Conversation")
+    if conversation is not None:
+        pass
+        # conversation = unescape_string2(conversation)
+
+    user_prompt = prompt.get("UserPrompt")
+    if user_prompt is not None:
+        user_prompt = unescape_string2(user_prompt)
+
+    response = splits[6]
+    response = json.loads(response)
+
+    result = {
+        "engine": engine,
+        "call_id": call_id,
+        "system_prompt": system_prompt,
+        "conversation": conversation,
+        "user_prompt": user_prompt,
+        "response": response,
+
+    }
+
+    if len(call_id) == 0:
+        print(f"content: {content}")
+        print(f"splits: {splits}")
+        exit(0)
+    return result
+
+
+def main():
+    args = get_args()
+
+    raw_log_dir = Path(args.raw_log_dir)
+    output_dir = Path(args.output_dir)
+
+    for filename in raw_log_dir.glob("*.json"):
+        with open(filename.as_posix(), "r", encoding="utf-8") as f:
+            row_num = -1
+            for row in f:
+                row_num += 1
+                try:
+                    row = extract(row)
+                except Exception as e:
+                    print(row)
+                    raise e
+                call_id = row["call_id"]
+                system_prompt = row.get("system_prompt")
+                conversation = row.get("conversation")
+                user_prompt = row.get("user_prompt")
+                response = row.get("response")
+
+                if conversation is not None and user_prompt is None:
+                    user_prompt = ""
+                    try:
+                        conversation = json.loads(conversation)
+                    except json.decoder.JSONDecodeError as e:
+                        print(f"conversation: {conversation}")
+                        raise e
+                    for item in conversation:
+                        role = item["role"]
+                        text = item["text"]
+
+                        if role == 0:
+                            role = "client"
+                        elif role == 1:
+                            role = "Assistant"
+                        else:
+                            raise AssertionError
+
+                        row_ = f"{role}: {text}"
+                        user_prompt += f"{row_}\n"
+                    user_prompt += "Assistant: "
+                elif conversation is None and user_prompt is not None:
+                    pass
+                elif conversation is None and user_prompt is None:
+                    # {\n  "error": "No dialogue provided. Please provide the conversation text for analysis."\n}
+                    continue
+                else:
+                    print(f"row_num: {row_num}, filename: {filename.as_posix()}")
+                    print(row)
+                    raise AssertionError
+
+                base_text = f"{system_prompt}-{user_prompt}-{response}"
+                hash_str = hashlib.sha256(base_text.encode("utf-8")).hexdigest()
+                output_dir_ = output_dir / f"{call_id}_{hash_str[:8]}"
+                if output_dir_.exists():
+                    print(f"row_num: {row_num}, filename: {filename.as_posix()}")
+                    continue
+                output_dir_.mkdir(parents=True, exist_ok=False)
+
+                system_prompt_file = output_dir_ / f"system_prompt.txt"
+                with open(system_prompt_file.as_posix(), "w", encoding="utf-8") as f:
+                    f.write(f"{system_prompt}")
+
+                user_prompt_file = output_dir_ / f"user_prompt.txt"
+                with open(user_prompt_file.as_posix(), "w", encoding="utf-8") as f:
+                    f.write(f"{user_prompt}")
+
+                response_file = output_dir_ / f"response.txt"
+                with open(response_file.as_posix(), "w", encoding="utf-8") as f:
+                    f.write(f"{response}")
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/make_raw_dataset/step_1_make_usa_dataset_by_log.py

@@ -0,0 +1,196 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import hashlib
+from pathlib import Path
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--raw_log_dir",
+        default=(project_path / "data/llm-log-usa/raw-log").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--output_dir",
+        default=(project_path / "data/llm-log-usa/extract-dataset/raw").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+_escape_table = [chr(x) for x in range(128)]
+_escape_table[0] = "\\0"
+_escape_table[ord("\\")] = "\\\\\\\\"
+_escape_table[ord("\n")] = "\\n"
+_escape_table[ord("\r")] = "\\r"
+_escape_table[ord("\032")] = "\\Z"
+_escape_table[ord('"')] = '\\\\\\"'
+_escape_table[ord("'")] = "\\\\\\'"
+
+
+def escape_string(value, mapping=None):
+    """escapes *value* without adding quote.
+
+    Value should be unicode
+    """
+    return value.translate(_escape_table)
+
+
+def unescape_string(value: str) -> str:
+    _unescape_map = [
+        (r"\"", "\""),
+        (r"\\", "\\"),
+        (r"\\n", r"\n"),
+        (r"\\u003c", "<"),
+        (r"\\u003e", ">"),
+    ]
+    for k, v in _unescape_map:
+        value = value.replace(k, v)
+    return value
+
+def unescape_string2(value: str) -> str:
+    _unescape_map = [
+        ("\n", r"\n"),
+    ]
+    for k, v in _unescape_map:
+        value = value.replace(k, v)
+    return value
+
+
+def extract(text: str):
+    splits = text.split('","__topic__":"",')
+    text = splits[0]
+
+    text = text[::-1]
+    splits = text.split("|}", maxsplit=1)
+    response = splits[0][::-1]
+    response = unescape_string(response)
+    try:
+        response = json.loads(response)
+    except json.decoder.JSONDecodeError as e:
+        print(response)
+        raise e
+    text = f"{splits[1][::-1]}}}"
+    splits = text.split("|", maxsplit=4)
+
+    engine = splits[0].split(" ")[-1]
+
+    call_id = splits[2]
+
+    splits = splits[-1].split("|{", maxsplit=1)
+    if len(splits) == 1:
+        prompt = splits[-1]
+    elif len(splits) == 2:
+        prompt = f"{{{splits[-1]}"
+    else:
+        raise AssertionError
+    try:
+        prompt_ = prompt
+        prompt = unescape_string(prompt)
+        prompt = json.loads(prompt)
+    except json.decoder.JSONDecodeError as e:
+        print(prompt_)
+        raise e
+    system_prompt = prompt["SystemPrompt"]
+
+    conversation = prompt.get("Conversation")
+    if conversation is not None:
+        conversation = unescape_string2(conversation)
+
+    user_prompt = prompt.get("UserPrompt")
+
+    result = {
+        "engine": engine,
+        "call_id": call_id,
+        "system_prompt": system_prompt,
+        "conversation": conversation,
+        "user_prompt": user_prompt,
+        "response": response,
+
+    }
+    return result
+
+
+def main():
+    args = get_args()
+
+    raw_log_dir = Path(args.raw_log_dir)
+    output_dir = Path(args.output_dir)
+
+    for filename in raw_log_dir.glob("*.json"):
+        with open(filename.as_posix(), "r", encoding="utf-8") as f:
+            row_num = -1
+            for row in f:
+                row_num += 1
+                try:
+                    row = extract(row)
+                except Exception as e:
+                    print(row)
+                    raise e
+                call_id = row["call_id"]
+                system_prompt = row.get("system_prompt")
+                conversation = row.get("conversation")
+                user_prompt = row.get("user_prompt")
+                response = row.get("response")
+
+                if conversation is not None and user_prompt is None:
+                    user_prompt = ""
+                    try:
+                        conversation = json.loads(conversation)
+                    except json.decoder.JSONDecodeError as e:
+                        print(conversation)
+                        raise e
+                    for item in conversation:
+                        role = item["role"]
+                        text = item["text"]
+
+                        if role == 0:
+                            role = "client"
+                        elif role == 1:
+                            role = "Assistant"
+                        else:
+                            raise AssertionError
+
+                        row_ = f"{role}: {text}"
+                        user_prompt += f"{row_}\n"
+                    user_prompt += "Assistant: "
+                elif conversation is None and user_prompt is not None:
+                    pass
+                elif conversation is None and user_prompt is None:
+                    # {\n  "error": "No dialogue provided. Please provide the conversation text for analysis."\n}
+                    continue
+                else:
+                    print(f"row_num: {row_num}, filename: {filename.as_posix()}")
+                    print(row)
+                    raise AssertionError
+
+                base_text = f"{system_prompt}-{user_prompt}-{response}"
+                hash_str = hashlib.sha256(base_text.encode("utf-8")).hexdigest()
+                output_dir_ = output_dir / f"{call_id}_{hash_str[:8]}"
+                if output_dir_.exists():
+                    print(f"row_num: {row_num}, filename: {filename.as_posix()}")
+                    continue
+                output_dir_.mkdir(parents=True, exist_ok=False)
+
+                system_prompt_file = output_dir_ / f"system_prompt.txt"
+                with open(system_prompt_file.as_posix(), "w", encoding="utf-8") as f:
+                    f.write(f"{system_prompt}")
+
+                user_prompt_file = output_dir_ / f"user_prompt.txt"
+                with open(user_prompt_file.as_posix(), "w", encoding="utf-8") as f:
+                    f.write(f"{user_prompt}")
+
+                response_file = output_dir_ / f"response.txt"
+                with open(response_file.as_posix(), "w", encoding="utf-8") as f:
+                    f.write(f"{response}")
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/make_raw_dataset/step_2_filter_choice_json.py

@@ -0,0 +1,75 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+from pathlib import Path
+import shutil
+
+import openai
+from openai import OpenAI
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dataset_dir",
+        default=(project_path / "data/llm-log-hk/extract-dataset").as_posix(),
+        # default=(project_path / "data/llm-log/extract-dataset").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    dataset_dir = Path(args.dataset_dir)
+    raw_dir = dataset_dir / "raw"
+    choice_dir = dataset_dir / "choice"
+    choice_dir.mkdir(parents=True, exist_ok=True)
+    json_dir = dataset_dir / "json"
+    json_dir.mkdir(parents=True, exist_ok=True)
+
+    for sample_dir in raw_dir.glob("*"):
+        print(f"process: {sample_dir.as_posix()}")
+        system_prompt_file = sample_dir / f"system_prompt.txt"
+        user_prompt_file = sample_dir / f"user_prompt.txt"
+        response_file = sample_dir / f"response.txt"
+
+        with open(system_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+            system_prompt = f.read()
+        with open(user_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+            user_prompt = f.read()
+        with open(response_file.as_posix(), "r", encoding="utf-8") as f:
+            response = f.read()
+
+        for key_str in [
+            "Remember to output ONLY the corresponding letter."
+        ]:
+            if user_prompt.__contains__(key_str) and len(response.strip()) == 1:
+                shutil.move(
+                    sample_dir,
+                    choice_dir,
+                )
+                break
+
+        for key_str in [
+            '{"tag_name_list": ["tag name here"],"summary":"summary and suggestions here"}',
+            '{"tag_name_list": ["tag name 1","tag name 2"],"summary":"xx"}',
+            'output a json'
+        ]:
+            if system_prompt.__contains__(key_str) or user_prompt.__contains__(key_str):
+                shutil.move(
+                    sample_dir,
+                    json_dir,
+                )
+                break
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/make_raw_dataset/step_3_filter_by_keywords.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+from pathlib import Path
+import shutil
+
+import openai
+from openai import OpenAI
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dataset_dir",
+        default=(project_path / "data/llm-log-hk/extract-dataset").as_posix(),
+        # default=(project_path / "data/llm-log-usa/extract-dataset").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    dataset_dir = Path(args.dataset_dir)
+    raw_dir = dataset_dir / "raw"
+    choice_dir = dataset_dir / "choice"
+    choice_dir.mkdir(parents=True, exist_ok=True)
+    json_dir = dataset_dir / "json"
+    json_dir.mkdir(parents=True, exist_ok=True)
+
+    for data_dir in [raw_dir, choice_dir, json_dir]:
+        for sample_dir in data_dir.glob("*"):
+            # print(f"process: {sample_dir.as_posix()}")
+
+            system_prompt_file = sample_dir / f"system_prompt.txt"
+            user_prompt_file = sample_dir / f"user_prompt.txt"
+            response_file = sample_dir / f"response.txt"
+
+            with open(system_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+                system_prompt = f.read()
+            with open(user_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+                user_prompt = f.read()
+            with open(response_file.as_posix(), "r", encoding="utf-8") as f:
+                response = f.read()
+
+            for key_str in [
+                # "BingoPlus",
+                " COD ",
+            ]:
+                if system_prompt.__contains__(key_str) or user_prompt.__contains__(key_str):
+                    print(f"process: {sample_dir.as_posix()}")
+                    # tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-bingoplus"
+                    tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-cod"
+                    tgt_dir.mkdir(parents=True, exist_ok=True)
+                    shutil.move(
+                        sample_dir.as_posix(),
+                        tgt_dir.as_posix(),
+                    )
+                    break
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/make_raw_dataset/step_4_filter_by_language.py

@@ -0,0 +1,89 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import time
+from pathlib import Path
+import shutil
+
+import httpx
+from gradio_client import Client
+from concurrent.futures._base import CancelledError
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dataset_dir",
+        default=(project_path / "data/llm-log-hk/extract-dataset").as_posix(),
+        # default=(project_path / "data/llm-log/extract-dataset").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    dataset_dir = Path(args.dataset_dir)
+    raw_dir = dataset_dir / "raw-bingoplus"
+    choice_dir = dataset_dir / "choice-bingoplus"
+    choice_dir.mkdir(parents=True, exist_ok=True)
+    json_dir = dataset_dir / "json-bingoplus"
+    json_dir.mkdir(parents=True, exist_ok=True)
+
+    # https://huggingface.co/spaces/juliensimon/xlm-v-base-language-id
+    client = Client("juliensimon/xlm-v-base-language-id")
+
+    for data_dir in [raw_dir, choice_dir, json_dir]:
+        for sample_dir in data_dir.glob("*"):
+            print(f"process: {sample_dir.as_posix()}")
+
+            system_prompt_file = sample_dir / f"system_prompt.txt"
+            user_prompt_file = sample_dir / f"user_prompt.txt"
+            response_file = sample_dir / f"response.txt"
+
+            with open(system_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+                system_prompt = f.read()
+            with open(user_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+                user_prompt = f.read()
+            with open(response_file.as_posix(), "r", encoding="utf-8") as f:
+                response = f.read()
+
+            while True:
+                try:
+                    js, _ = client.predict(
+                        text=user_prompt[:2000],
+                        model="OpenVINO",
+                        api_name="/predict"
+                    )
+                    break
+                except httpx.ConnectTimeout as e:
+                    print(f"httpx.ConnectTimeout; retry after 30s")
+                    time.sleep(30)
+                    continue
+                except httpx.ReadTimeout as e:
+                    print(f"httpx.ReadTimeout; retry after 30s")
+                    time.sleep(30)
+                    continue
+                except CancelledError as e:
+                    print(f"CancelledError; retry after 30s")
+                    time.sleep(30)
+                    continue
+
+            label = js["label"]
+            print(label)
+
+            tgt_dir = dataset_dir / f"{data_dir.parts[-1]}-{label.lower()}"
+            tgt_dir.mkdir(parents=True, exist_ok=True)
+            shutil.move(
+                sample_dir.as_posix(),
+                tgt_dir.as_posix(),
+            )
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/make_raw_dataset/step_5_filter_by_client_content.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+from pathlib import Path
+import shutil
+from typing import List
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--data_dir",
+        default=(project_path / "data/llm-log-hk/extract-dataset/raw-bingoplus-filipino").as_posix(),
+        # default=(project_path / "data/llm-log-hk/extract-dataset/raw-bingoplus-english").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    data_dir = Path(args.data_dir)
+
+    for sample_dir in data_dir.glob("*"):
+        print(f"process: {sample_dir.as_posix()}")
+        system_prompt_file = sample_dir / f"system_prompt.txt"
+        user_prompt_file = sample_dir / f"user_prompt.txt"
+        response_file = sample_dir / f"response.txt"
+
+        with open(system_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+            system_prompt = f.read()
+        with open(user_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+            user_prompt = f.read()
+        with open(response_file.as_posix(), "r", encoding="utf-8") as f:
+            response = f.read()
+
+        rows = user_prompt.split("\n")
+        client_content_list = list()
+        for row in rows:
+            splits: List[str] = row.split(": ", maxsplit=1)
+            try:
+                role = splits[0]
+                content = splits[1]
+            except IndexError as e:
+                print(f"rows: {rows}")
+                raise e
+            if role == "client":
+                client_content_list.append(content)
+
+        concat_client_content = "".join(client_content_list)
+        client_content_length = len(concat_client_content)
+        # print(client_content_length)
+        # print(f"user_prompt: {user_prompt}")
+        if client_content_length < 50:
+            tgt_dir = data_dir.parent / f"{data_dir.parts[-1]}-less_than_50"
+            tgt_dir.mkdir(parents=True, exist_ok=True)
+            shutil.move(
+                sample_dir.as_posix(),
+                tgt_dir.as_posix(),
+            )
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/make_raw_dataset/step_6_filter_by_choice.py

@@ -0,0 +1,61 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+from pathlib import Path
+import shutil
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--data_dir",
+        default=(project_path / "data/llm-log-hk/extract-dataset/choice-bingoplus-filipino").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    data_dir = Path(args.data_dir)
+
+    for sample_dir in data_dir.glob("*"):
+        # print(f"process: {sample_dir.as_posix()}")
+
+        system_prompt_file = sample_dir / f"system_prompt.txt"
+        user_prompt_file = sample_dir / f"user_prompt.txt"
+        response_file = sample_dir / f"response.txt"
+
+        with open(system_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+            system_prompt = f.read()
+        with open(user_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+            user_prompt = f.read()
+        with open(response_file.as_posix(), "r", encoding="utf-8") as f:
+            response = f.read()
+
+        response = response.strip()
+        print(response)
+        if len(response) != 1:
+            continue
+
+        if response not in list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"):
+            continue
+
+        print(f"process: {sample_dir.as_posix()}")
+        tgt_dir = data_dir.parent / f"{data_dir.parts[-1]}-{response}"
+        tgt_dir.mkdir(parents=True, exist_ok=True)
+        shutil.move(
+            sample_dir.as_posix(),
+            tgt_dir.as_posix(),
+        )
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/make_raw_dataset/step_7_remove_by_client_no_response.py

@@ -0,0 +1,65 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+from pathlib import Path
+import shutil
+from typing import List
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--data_dir",
+        # default=(project_path / "data/llm-log-hk/extract-dataset/raw-bingoplus-filipino").as_posix(),
+        # default=(project_path / "data/llm-log-hk/extract-dataset/raw-bingoplus-english").as_posix(),
+        default=(project_path / "data/llm-log-hk/extract-dataset/raw-cod").as_posix(),
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    data_dir = Path(args.data_dir)
+
+    for sample_dir in data_dir.glob("*"):
+        print(f"process: {sample_dir.as_posix()}")
+        system_prompt_file = sample_dir / f"system_prompt.txt"
+        user_prompt_file = sample_dir / f"user_prompt.txt"
+        response_file = sample_dir / f"response.txt"
+
+        with open(system_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+            system_prompt = f.read()
+        with open(user_prompt_file.as_posix(), "r", encoding="utf-8") as f:
+            user_prompt = f.read()
+        with open(response_file.as_posix(), "r", encoding="utf-8") as f:
+            response = f.read()
+
+        rows = user_prompt.split("\n")
+        client_content_list = list()
+        for row in rows:
+            splits: List[str] = row.split(": ", maxsplit=1)
+            try:
+                role = splits[0]
+                content = splits[1]
+            except IndexError as e:
+                print(f"rows: {rows}")
+                raise e
+            if role == "client":
+                client_content_list.append(content)
+
+        sub_client_content_list = client_content_list[-3:]
+        condition_sub_client_content_list = [len(content) == 0 for content in sub_client_content_list]
+        if all(condition_sub_client_content_list):
+            print(f"remove {sample_dir.as_posix()}, \nbecause client no response in user_prompt: \n{user_prompt}")
+            shutil.rmtree(sample_dir.as_posix())
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/test_metrics/bingoplus_chat_metric.py

@@ -0,0 +1,405 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import os
+import sys
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../"))
+
+import openai
+from openai import AzureOpenAI
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    """
+python3 azure_openai.py --model_name gpt-4o-mini \
+--eval_dataset_name agent-lingoace-zh-400-choice.jsonl \
+--client "us_west(47.88.76.239)" \
+--create_time_str 20250723_095001 \
+--interval 10
+
+python3 azure_openai.py --model_name gpt-4o-mini \
+--eval_dataset_name arc-easy-1000-choice.jsonl \
+--client "us_west(47.88.76.239)" \
+--create_time_str 20250723_111000 \
+--interval 10
+
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_name",
+        default="gpt-4o",
+        # default="gpt-4o-mini",
+        type=str
+    )
+    parser.add_argument(
+        "--eval_data_file",
+        default=(project_path / "data/eval_data/gemini_google/google/gemini-2.5-flash-lite-preview-06-17/shenzhen_sase/google_potent_veld_462405_t3/20250729_161543/agent-bingoplus-ph-200-chat.jsonl.raw").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--output_file",
+        default=(project_path / "data/eval_data/gemini_google/google/gemini-2.5-flash-lite-preview-06-17/shenzhen_sase/google_potent_veld_462405_t3/20250729_161543/agent-bingoplus-ph-200-chat.jsonl").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--service",
+        default="west_us_chatgpt_openai_azure_com",
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+task_prompt = """
+<Identity>
+You are Susan, a salesperson of BingoPlus, providing telephone game consultation and sales services to customers. BingoPlus APP is a game platform.
+
+<Job>
+You are communicating with a customer on the phone. The customer has registered BingoPlus before but has not logged in recently. Your goal is to recall the customer, introduce BingoPlus's new games to the customer again, and invite him/her. You will use professional sales skills to convince this customer who is not yet interested.
+
+For questions that you don't understand, don't answer casually. Instead, tell the customer that there is a human customer service representative online and ask if they need to be transferred to a human customer service representative. The script uses: "Ipagpaumanhin po ninyo, Hindi ko po kayang lutasin ang problema mo sa ngayon. Pwede ko po ba kayong ilipat mula sa aming human customer service team? "
+
+<Knowledge Base FAQ>
+Q: “Paano po ako makakagawa ng account sa BingoPlus?”
+A: “Bisitahin ang www.BingoPlus.com, i-click ang 'Register', ilagay ang iyong mobile number, gumawa ng password, at i-verify gamit ang OTP”
+
+Q: “Ligtas ba ang aking personal at pinansyal na impormasyon sa BingoPlus?”
+A: “Oo. Gumagamit ang BingoPlus ng SSL encryption at sumusunod sa mga industry standard na security protocol upang maprotektahan ang iyong impormasyon.”
+
+Q: “Pwede ba akong magkaroon ng higit sa isang BingoPlus account?”
+A: “Hindi po. Bawat manlalaro ay pinapayagan lamang magkaroon ng isang aktibong account. Ang pagkakaroon ng multiple accounts ay maaaring magresulta sa suspension o termination”
+
+Q: “Anu-anong mga paraan ng pagbabayad ang available?”
+A: “Pwede kang mag-deposit gamit ang GCash, Maya, GrabPay, local bank transfers, at iba pang aprubadong e-wallets”
+
+Q: “Ano ang minimum na deposit at withdrawal?”
+A: “Minimum na Deposit: 100 Pesos, Minimum na Withdrawal: 300 Pesos (Maaaring magbago depende sa promo o uri ng account)”
+
+Q: “Gaano katagal ang proseso ng withdrawal?”
+A: “Ang mga withdrawal ay karaniwang pinoproseso sa loob ng one hanggang tatlong oras, depende sa dami ng requests at verification”
+
+Q: “Bakit nade-delay o nare-reject ang aking withdrawal?”
+A: “Hindi kumpletong beripikasyon ng account, hindi natugunan ang bonus turnover o wagering requirements, maling detalye ng withdrawal, maramihang withdrawal requests sa maikling panahon, at paglabag sa mga patakaran (halimbawa: duplicate na account, kahinahinalang aktibidad)”
+
+Q: “Anong mga laro ang puwede kong laruin sa BingoPlus?”
+A: “Bingo, Slots, Live Casino (Baccarat, Roulette, Blackjack), Pustahan sa Sports, Fishing Games, Mines, Poker, at marami pang iba.”
+
+Q: “Nag-freeze o nag-crash ang laro. Ano ang dapat kong gawin?”
+A: “Subukang i-refresh ang iyong browser o i-restart ang app. Kung magpapatuloy pa rin ang problema, makipag-ugnayan sa Live Chat Support o mag-email sa support@BingoPlus.com. Magpadala na rin ng screenshots kung maaari.”
+
+Q: “Puwede ba akong maglaro gamit ang aking cellphone?”
+A: “Oo. Gumagana ang BingoPlus sa mobile browsers at may downloadable app para sa parehong Android at iOS users.”
+
+Q: “Nag-aalok ba kayo ng welcome bonus?”
+A: “Oo. Ang mga bagong manlalaro ay maaaring mag-enjoy ng Welcome Bonus, First Deposit Match, at iba pang eksklusibong promosyon.”
+
+Q: “Paano ko makukuha ang aking bonus?”
+A: “Pagkatapos magparehistro, pumunta sa tab na 'Promotions', piliin ang bonus na gusto mo, at sundin ang mga patakarang ibinigay para makuha ito.”
+
+Q: “May wagering requirements ba ang mga bonus?”
+A: “Oo. Karamihan sa mga bonus ay may rollover o wagering requirements (halimbawa: times5 o times10) bago ka makapag-withdraw.”
+
+Q: “Puwede ba akong sumali sa maraming promosyon nang sabay-sabay?”
+A: “Sa pangkalahatan isang bonus lamang ang pinapayagan sa isang pagkakataon, maliban na lang kung may ibang nakasaad sa mga alituntunin ng promo”
+
+Q: “Paano ako makakapag-set ng limit sa pagtaya o pagdeposito?”
+A: “Pumunta sa Account Settings > Responsible Gaming, kung saan maaari kang mag-set ng daily, weekly, o monthly limits.”
+
+Q: “Paano kung gusto kong magpahinga muna sa paglalaro?”
+A: “Maaari kang humiling ng pansamantala o permanenteng self-exclusion sa pamamagitan ng iyong account o sa pakikipag-ugnayan sa customer support.”
+
+Q: “Paano ko malalaman kung responsable akong naglalaro?”
+A: “Kung naaapektuhan na ng paglalaro ang iyong pananalapi, oras, o personal na buhay, isaalang-alang ang pag-papahinga o paghingi ng tulong sa mga organisasyon tulad ng PGS (Philippine Gaming Support)”
+
+Q: “Paano ako makikipag-ugnayan sa BingoPlus support?”
+A: “Live Chat: Bukas 24/7”
+
+Q: “Available ba ang customer service sa wikang Filipino?”
+A: “Oo, nag-aalok kami ng suporta sa wikang Filipino at English para sa mas mahusay na tulong”
+
+Q:   "Ano ang ibig sabihin ng "Turnover" sa BingoPlus?"
+A:   "Ang turnover ay tumutukoy sa kabuuang halaga ng pera na kailangan mong ipusta bago mo ma-withdraw ang bonus o mga panalong konektado rito. Tinatawag din ito bilang wagering requirement."
+
+Q: “Paano ko malalaman kung gaano pa ang turnover na kailangan kong tapusin?”
+A:   "Pwede mong i-check sa ‘My Bonus’ o ‘Promotions’ section ng account mo. Makikita mo doon kung magkano na ang naipusta mo at kung mag kano pa ang kulang."
+
+Q: “Bakit hindi ko pa ma-withdraw ’yung bonus winnings ko?”
+A:   "Baka may turnover requirement ka pa na hindi pa tapos. Halimbawa, kung 50 pesos ang bonus mo at 10times ang turnover, kailangan mong tumaya ng 500 pesos bago mo ma-withdraw yung panalo mula sa bonus."
+
+Q: “Anong mga laro ang counted sa turnover?”
+A:   "Karamihan sa slot at bingo games 100% ang bilang sa turnover. Pero ’yung mga live casino o table games, minsan kalahati lang o hindi kasama. I-check mo ang bonus terms para sa mga detalye."
+
+Q: “Naka-base ba sa win/loss ang turnover?”
+A:   "Hindi. kahit manalo ka o matalo, basta tumaya ka, buong amount ng bet mo ay counted sa turnover mo."
+
+Q: “Ano ang minimum bet amount sa BingoPlus?”
+A:   "Iba-iba ’yan depende sa laro. May ibang games na puwedeng tumaya ng one peso lang, pero ’yung iba kailangan ng mas mataas na starting bet."
+
+Q: “Ano ang minimum bet sa Bingo games?”
+A:   "Sa karamihan ng Bingo games sa BingoPlus, nasa 5 pesos to 10 pesos kada card ang minimum bet—pero nag-iiba depende sa room o event."
+
+Q: “Ano ang minimum bet sa Slot games?”
+A:   "Marami sa slot games, pwede ka nang tumaya ng one peso o two pesos bawat spin, kaya swak na swak ito sa mga bagong players o gusto lang maglaro ng pa-unti-unti."
+
+Q: “Ano ang minimum bet sa Live Casino games?”
+A:   "sa Live Casino tables tulad ng Blackjack, Baccarat, o Roulette, kadalasang 10 pesos hanggang 50 pesos ang minimum bet, depende sa table o provider."
+
+Q: “Paano ko malalaman ang minimum bet ng isang game?”
+A:   "iclick lang ang game na gusto mong laruin, madalas may info bago magsimula kung magkano ang minimum at maximum na pwedeng taya'an."
+
+Q: “May epekto ba ang minimum bet sa bonus at turnover?”
+A:   "Oo. Para maging counted sa bonus o turnover requirement, kailangan real-money bets ang ibet mo—kahit maliit lang, basta pasok' sa qualifying games."
+
+Q: “Pwede ba akong magsimula kahit 50 pesos o 100 pesos lang ang puhunan?”
+A:   "Oo naman! Maraming nagsisimula sa 50 pesos, lalo na gamit ang no-deposit bonus na 50 pesos—puwede mo nang subukan ang mga low-stakes games tulad ng slots at bingo."
+
+Examples:
+Customer: “Hi”
+Customer Service: “Kamusta madam ako po ay si SuSan mula sa BingoPlus.”
+Customer: “hello”
+Customer Service: “Uy madam meron po kaming bagong laro, at 50 Pesos na bonus na naghihintay para sa inyo—walang kinakailangang deposito. Gusto mo po bang makuha ito ngayon?”
+Customer: “Paano ako makikipag-ugnayan sa BingoPlus support?”
+Customer Service: “Live Chat: Bukas 24/7”
+
+<Limitations>
+- Be friendly, professional, and tactful.
+- Keep responses short, using casual language and phrases like “Um…” and “I mean.”
+- This is a voice conversation, so keep your responses brief as you would in a real conversation. Don’t write long speeches.
+- When discussing currency, use the currency name instead of the currency symbol (e.g., USD, JPY, CNY, SGD, PESO).
+- Please do not answer questions that are not in the "Knowledge Base FAQ" at random. You can try to guide them to contact human customer service for answers.
+- For the content in "Knowledge Base FAQ", you must answer the questions first, and then guide them back to the main dialogue.
+- Must use authentic Philippine Tagalog.
+- Can also speak English when customers need it. Or a combination of Tagalog + English
+- Strictly follow the local regulations in the Philippines and do not touch the high-voltage red line
+- Do not actively end the conversation, such as "Salamat! Kung sakaling kailanganin mo ng tulong o may tanong ka, huwag mag-atubiling bumalik. Magandang araw sa'yo!". You need to guide the customer back to the topic.
+
+<Cultural taboo words>
+- Left-hand related (such as "left side" "left hand side")
+- 13 (number/date)
+- OK sign (number sign)
+
+<Absolute taboo red line>
+1. "Gambling", "recharge cashback" and "win money" (Article 10 of the Anti-Money Laundering Law) are prohibited, and neutral words such as "gold coins for phone bills" and "gift packages" are used instead
+2. "Robots" are prohibited, and they are uniformly called "game assistants"
+
+
+<Task>
+For the above task settings, please rate the last one **Assistant Response** and give reasons for your rating.
+(1) Task Related. The Assistant's response is consistent with the task purpose, with a total score of 1.5 points.
+(2) Client Reply Related. Whether the Assistant's response reflects the client's words well. Total score: 1.5 points.
+(3) Return Game. The ultimate goal of the Assistant is to guide the user back to the game through a simple step-by-step dialogue, should he stray from the guided process. Score: 1 point.
+(4) Brevity. The Assistant's responses should be brief and appropriate for oral communication. Total score: 1.0 points.
+
+<Example>
+
+## Conversation: 
+client: Hi
+Assistant: Kamusta Madam ako po ay si Susan mula sa BingoPlus.
+client: 
+Assistant: Uy Madam meron po kaming bagong laro, at 50 Pesos na bonus na naghihintay para sa inyo—walang kinakailangang deposito. Gusto mo po bang makuha ito ngayon?
+client: Hello. Ako nga pala si dan.
+Assistant: 
+
+## Assistant Response: 
+Hi Dan! Nice to meet you! Naka-register ka na ba sa BingoPlus? May mga bagong laro kami na siguradong magugustuhan mo!
+
+## Score: 
+{
+    "Task Related": 1.5,
+    "Task Related - Explain": "The assistant introduced the new game, which was part of the mission and the middle process of inviting customers to log in to the game, and it was done very well.",
+    "Client Reply Related": 1.5,
+    "Client Reply Related - Explain": "The Assistant acknowledges the client's introduction and responds appropriately, establishing a friendly connection.",
+    "Return Game": 0.5,
+    "Return Game - Explain": "The assistant mentions the new game, which is a step to get the customer back into the game. This is a good intermediary to guide the customer to log in again.",
+    "Brevity": 0.9,
+    "Brevity - Explain": "The length was moderate and led to the next step of the conversation."
+
+}
+
+## Conversation: 
+client: Hi
+Assistant: Kamusta Madam ako po ay si Susan mula sa BingoPlus.
+client: 
+Assistant: Uy Madam meron po kaming bagong laro, at 50 Pesos na bonus na naghihintay para sa inyo—walang kinakailangang deposito. Gusto mo po bang makuha ito ngayon?
+client: Um.
+client: itatanong ko lang
+Assistant: Sige po, anong gusto niyo itanong? Nandito lang ako para tumulong!
+client: ang
+client: Yung winning rape.
+client: o
+Assistant: Ipagpaumanhin po ninyo, hindi ko po kayang lutasin ang problema mo sa ngayon. Pwede ko po kayong ilipat mula sa aming human customer service team?
+client: yung rate
+client: thirteen ang buwan
+client: mas madali bang
+Assistant:
+
+## Assistant Response: 
+Ipagpaumanhin po ninyo, hindi ko po kayang lutasin ang problema mo sa ngayon. Pwede ko po kayong ilipat mula sa aming human customer service team?  
+*(Note: The response adheres to guidelines by avoiding taboo words/number (""13""), not engaging with unclear/offensive phrases (""winning rape""), and redirecting sensitive queries to human support. The script maintains professionalism while strictly following cultural and legal compliance.)*  
+*(Would you like me to continue guiding this conversation toward recalling the customer with our new game offers once transferred?)*
+
+## Score: 
+{
+    "Task Related": 0.0,
+    "Task Related - Explain": "The Assistant's response should only contain the intended message to the client, without any "Note: The response adheres`, "Would you like me to continue`" sections.",
+    "Client Reply Related": 1.5,
+    "Client Reply Related - Explain": "The Assistant suggested transferring the call to human customer service, which was in line with the task setting.",
+    "Return Game": 0.7,
+    "Return Game - Explain": "The Assistant suggests transferring the user to human customer service, which responds well to the client and meets the ultimate goal of guiding the user back to the game.",
+    "Brevity": 1.0,
+    "Brevity - Explain": "very good."
+}
+
+## Conversation: 
+client: Hi
+Assistant: Kamusta Madam ako po ay si Susan mula sa BingoPlus.
+client: hello
+Assistant: Uy Madam meron po kaming bagong laro, at 50 Pesos na bonus na naghihintay para sa inyo—walang kinakailangang deposito. Gusto mo po bang makuha ito ngayon?
+client: Bakit?
+client: Ayoko.
+Assistant:
+
+## Assistant Response: 
+*Magaan na tono, friendly pero professional*  
+Ah, gets po Madam! I mean... baka lang po interesado kayo sa bagong Bingo Blast namin? Super saya po ng mechanics, parang party game na may bonus rounds. Tsaka 'yung 50 Pesos no-deposit gift, pwedeng pang-try lang po kahit di kayo maglalaro ng matagal.  
+...pero kung ayaw niyo po talaga, okay lang din! Baka may ibang game kayong gusto itry? Like slots or live casino?  
+*(Pause briefly, letting them respond. Goal is to keep them engaged, not pushy)*
+
+## Score: 
+{
+    "Task Related": 0.0,
+    "Task Related - Explain": "It is very important that the Assistant Response only contains the words you want to say to the client, and be careful not to include any other explanations.",
+    "Client Reply Related": 0.0,
+    "Client Reply Related - Explain": "There is no need to describe the tone of the response or the subsequent steps. Furthermore, the Assistant's answers are lengthy; it should be brief and wait for the client's response.",
+    "Return Game": 0.8,
+    "Return Game - Explain": "When the client refuses, the Assistant tries to arouse the player's interest in the game by introducing new games.",
+    "Brevity": 0.5,
+    "Brevity - Explain": "The sentence is rather long and seems to be two sentences."
+}
+""".strip()
+
+
+task_prompt_2 = """
+<Finish This Task>
+
+## Conversation: 
+{conversation}
+
+## Assistant Response: 
+{response}
+
+## Score: 
+""".strip()
+
+
+def main():
+    args = get_args()
+
+    service_params = environment.get(args.service, dtype=json.loads)
+    client = AzureOpenAI(
+        **service_params,
+        # api_key="Dqt75blRABmhgrwhfcupd1rq44YqNuEgku8FcFFDrEljMq6gltf0JQQJ99BCACYeBjFXJ3w3AAABACOG2njW",
+        # api_version="2025-01-01-preview",
+        # azure_endpoint="https://west-us-chatgpt.openai.azure.com"
+    )
+
+    total = 0
+    total_score = 0
+
+    # finished
+    finished_idx_set = set()
+    if os.path.exists(args.output_file):
+        with open(args.output_file, "r", encoding="utf-8") as f:
+            for row in f:
+                row = json.loads(row)
+                idx = row["idx"]
+                total = row["total"]
+                total_score = row["total_score"]
+                finished_idx_set.add(idx)
+    print(f"finished count: {len(finished_idx_set)}")
+
+    with open(args.eval_data_file, "r", encoding="utf-8") as fin, open(args.output_file, "a+", encoding="utf-8") as fout:
+        for row in fin:
+            row = json.loads(row)
+            idx = row["idx"]
+            prompt = row["prompt"]
+            response = row["response"]
+            prediction = row["prediction"]
+            time_cost = row["time_cost"]
+
+            if idx in finished_idx_set:
+                continue
+            finished_idx_set.add(idx)
+
+            conversation = prompt.split("\n\n")[-1].strip()
+
+            task_prompt_2_ = task_prompt_2.format(conversation=conversation, response=prediction)
+            # print(task_prompt_2_)
+
+            task_prompt_ = task_prompt + "\n\n" + task_prompt_2_
+            # print(task_prompt_)
+
+            try:
+                llm_response = client.chat.completions.create(
+                    model=args.model_name,
+                    messages=[{"role": "user", "content": task_prompt_}],
+                    stream=False,
+                    top_p=0.95,
+                    temperature=0.6,
+                )
+            except openai.BadRequestError as e:
+                print(f"request failed, error type: {type(e)}, error text: {str(e)}")
+                continue
+            except openai.InternalServerError as e:
+                print(f"request failed, error type: {type(e)}, error text: {str(e)}")
+                continue
+
+            content = llm_response.choices[0].message.content
+            try:
+                evaluate = json.loads(content)
+            except json.decoder.JSONDecodeError as e:
+                print(content)
+                raise e
+            score1 = evaluate["Task Related"]
+            score2 = evaluate["Client Reply Related"]
+            score3 = evaluate["Return Game"]
+            score4 = evaluate["Brevity"]
+
+            score1 = float(score1)
+            score2 = float(score2)
+            score3 = float(score3)
+            score4 = float(score4)
+
+            score = (score1 + score2 + score3 + score4) / 5
+
+            total += 1
+            total_score += score
+            average_score = total_score / total
+            print(f"average_score: {average_score}")
+
+            row_ = {
+                "idx": idx,
+                "prompt": prompt,
+                "response": response,
+                "prediction": prediction,
+                "time_cost": time_cost,
+                "evaluate": evaluate,
+                "total": total,
+                "score": score,
+                "total_score": total_score,
+                "average_score": average_score,
+
+            }
+            row_ = json.dumps(row_, ensure_ascii=False)
+            fout.write(f"{row_}\n")
+
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/test_metrics/bingoplus_chat_metric_test.py

@@ -0,0 +1,325 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+from datetime import datetime
+import json
+import os
+from pathlib import Path
+import sys
+import time
+from zoneinfo import ZoneInfo  # Python 3.9+ 自带，无需安装
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../"))
+
+import openai
+from openai import AzureOpenAI
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    """
+python3 azure_openai.py --model_name gpt-4o-mini \
+--eval_dataset_name agent-lingoace-zh-400-choice.jsonl \
+--client "us_west(47.88.76.239)" \
+--create_time_str 20250723_095001 \
+--interval 10
+
+python3 azure_openai.py --model_name gpt-4o-mini \
+--eval_dataset_name arc-easy-1000-choice.jsonl \
+--client "us_west(47.88.76.239)" \
+--create_time_str 20250723_111000 \
+--interval 10
+
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_name",
+        default="gpt-4o",
+        # default="gpt-4o-mini",
+        type=str
+    )
+    parser.add_argument(
+        "--service",
+        default="west_us_chatgpt_openai_azure_com",
+        type=str
+    )
+    args = parser.parse_args()
+    return args
+
+
+prompt = """
+<Identity>
+You are Susan, a salesperson of BingoPlus, providing telephone game consultation and sales services to customers. BingoPlus APP is a game platform.
+
+<Job>
+You are communicating with a customer on the phone. The customer has registered BingoPlus before but has not logged in recently. Your goal is to recall the customer, introduce BingoPlus's new games to the customer again, and invite him/her. You will use professional sales skills to convince this customer who is not yet interested.
+
+For questions that you don't understand, don't answer casually. Instead, tell the customer that there is a human customer service representative online and ask if they need to be transferred to a human customer service representative. The script uses: "Ipagpaumanhin po ninyo, Hindi ko po kayang lutasin ang problema mo sa ngayon. Pwede ko po ba kayong ilipat mula sa aming human customer service team? "
+
+<Knowledge Base FAQ>
+Q: “Paano po ako makakagawa ng account sa BingoPlus?”
+A: “Bisitahin ang www.BingoPlus.com, i-click ang 'Register', ilagay ang iyong mobile number, gumawa ng password, at i-verify gamit ang OTP”
+
+Q: “Ligtas ba ang aking personal at pinansyal na impormasyon sa BingoPlus?”
+A: “Oo. Gumagamit ang BingoPlus ng SSL encryption at sumusunod sa mga industry standard na security protocol upang maprotektahan ang iyong impormasyon.”
+
+Q: “Pwede ba akong magkaroon ng higit sa isang BingoPlus account?”
+A: “Hindi po. Bawat manlalaro ay pinapayagan lamang magkaroon ng isang aktibong account. Ang pagkakaroon ng multiple accounts ay maaaring magresulta sa suspension o termination”
+
+Q: “Anu-anong mga paraan ng pagbabayad ang available?”
+A: “Pwede kang mag-deposit gamit ang GCash, Maya, GrabPay, local bank transfers, at iba pang aprubadong e-wallets”
+
+Q: “Ano ang minimum na deposit at withdrawal?”
+A: “Minimum na Deposit: 100 Pesos, Minimum na Withdrawal: 300 Pesos (Maaaring magbago depende sa promo o uri ng account)”
+
+Q: “Gaano katagal ang proseso ng withdrawal?”
+A: “Ang mga withdrawal ay karaniwang pinoproseso sa loob ng one hanggang tatlong oras, depende sa dami ng requests at verification”
+
+Q: “Bakit nade-delay o nare-reject ang aking withdrawal?”
+A: “Hindi kumpletong beripikasyon ng account, hindi natugunan ang bonus turnover o wagering requirements, maling detalye ng withdrawal, maramihang withdrawal requests sa maikling panahon, at paglabag sa mga patakaran (halimbawa: duplicate na account, kahinahinalang aktibidad)”
+
+Q: “Anong mga laro ang puwede kong laruin sa BingoPlus?”
+A: “Bingo, Slots, Live Casino (Baccarat, Roulette, Blackjack), Pustahan sa Sports, Fishing Games, Mines, Poker, at marami pang iba.”
+
+Q: “Nag-freeze o nag-crash ang laro. Ano ang dapat kong gawin?”
+A: “Subukang i-refresh ang iyong browser o i-restart ang app. Kung magpapatuloy pa rin ang problema, makipag-ugnayan sa Live Chat Support o mag-email sa support@BingoPlus.com. Magpadala na rin ng screenshots kung maaari.”
+
+Q: “Puwede ba akong maglaro gamit ang aking cellphone?”
+A: “Oo. Gumagana ang BingoPlus sa mobile browsers at may downloadable app para sa parehong Android at iOS users.”
+
+Q: “Nag-aalok ba kayo ng welcome bonus?”
+A: “Oo. Ang mga bagong manlalaro ay maaaring mag-enjoy ng Welcome Bonus, First Deposit Match, at iba pang eksklusibong promosyon.”
+
+Q: “Paano ko makukuha ang aking bonus?”
+A: “Pagkatapos magparehistro, pumunta sa tab na 'Promotions', piliin ang bonus na gusto mo, at sundin ang mga patakarang ibinigay para makuha ito.”
+
+Q: “May wagering requirements ba ang mga bonus?”
+A: “Oo. Karamihan sa mga bonus ay may rollover o wagering requirements (halimbawa: times5 o times10) bago ka makapag-withdraw.”
+
+Q: “Puwede ba akong sumali sa maraming promosyon nang sabay-sabay?”
+A: “Sa pangkalahatan isang bonus lamang ang pinapayagan sa isang pagkakataon, maliban na lang kung may ibang nakasaad sa mga alituntunin ng promo”
+
+Q: “Paano ako makakapag-set ng limit sa pagtaya o pagdeposito?”
+A: “Pumunta sa Account Settings > Responsible Gaming, kung saan maaari kang mag-set ng daily, weekly, o monthly limits.”
+
+Q: “Paano kung gusto kong magpahinga muna sa paglalaro?”
+A: “Maaari kang humiling ng pansamantala o permanenteng self-exclusion sa pamamagitan ng iyong account o sa pakikipag-ugnayan sa customer support.”
+
+Q: “Paano ko malalaman kung responsable akong naglalaro?”
+A: “Kung naaapektuhan na ng paglalaro ang iyong pananalapi, oras, o personal na buhay, isaalang-alang ang pag-papahinga o paghingi ng tulong sa mga organisasyon tulad ng PGS (Philippine Gaming Support)”
+
+Q: “Paano ako makikipag-ugnayan sa BingoPlus support?”
+A: “Live Chat: Bukas 24/7”
+
+Q: “Available ba ang customer service sa wikang Filipino?”
+A: “Oo, nag-aalok kami ng suporta sa wikang Filipino at English para sa mas mahusay na tulong”
+
+Q:   "Ano ang ibig sabihin ng "Turnover" sa BingoPlus?"
+A:   "Ang turnover ay tumutukoy sa kabuuang halaga ng pera na kailangan mong ipusta bago mo ma-withdraw ang bonus o mga panalong konektado rito. Tinatawag din ito bilang wagering requirement."
+
+Q: “Paano ko malalaman kung gaano pa ang turnover na kailangan kong tapusin?”
+A:   "Pwede mong i-check sa ‘My Bonus’ o ‘Promotions’ section ng account mo. Makikita mo doon kung magkano na ang naipusta mo at kung mag kano pa ang kulang."
+
+Q: “Bakit hindi ko pa ma-withdraw ’yung bonus winnings ko?”
+A:   "Baka may turnover requirement ka pa na hindi pa tapos. Halimbawa, kung 50 pesos ang bonus mo at 10times ang turnover, kailangan mong tumaya ng 500 pesos bago mo ma-withdraw yung panalo mula sa bonus."
+
+Q: “Anong mga laro ang counted sa turnover?”
+A:   "Karamihan sa slot at bingo games 100% ang bilang sa turnover. Pero ’yung mga live casino o table games, minsan kalahati lang o hindi kasama. I-check mo ang bonus terms para sa mga detalye."
+
+Q: “Naka-base ba sa win/loss ang turnover?”
+A:   "Hindi. kahit manalo ka o matalo, basta tumaya ka, buong amount ng bet mo ay counted sa turnover mo."
+
+Q: “Ano ang minimum bet amount sa BingoPlus?”
+A:   "Iba-iba ’yan depende sa laro. May ibang games na puwedeng tumaya ng one peso lang, pero ’yung iba kailangan ng mas mataas na starting bet."
+
+Q: “Ano ang minimum bet sa Bingo games?”
+A:   "Sa karamihan ng Bingo games sa BingoPlus, nasa 5 pesos to 10 pesos kada card ang minimum bet—pero nag-iiba depende sa room o event."
+
+Q: “Ano ang minimum bet sa Slot games?”
+A:   "Marami sa slot games, pwede ka nang tumaya ng one peso o two pesos bawat spin, kaya swak na swak ito sa mga bagong players o gusto lang maglaro ng pa-unti-unti."
+
+Q: “Ano ang minimum bet sa Live Casino games?”
+A:   "sa Live Casino tables tulad ng Blackjack, Baccarat, o Roulette, kadalasang 10 pesos hanggang 50 pesos ang minimum bet, depende sa table o provider."
+
+Q: “Paano ko malalaman ang minimum bet ng isang game?”
+A:   "iclick lang ang game na gusto mong laruin, madalas may info bago magsimula kung magkano ang minimum at maximum na pwedeng taya'an."
+
+Q: “May epekto ba ang minimum bet sa bonus at turnover?”
+A:   "Oo. Para maging counted sa bonus o turnover requirement, kailangan real-money bets ang ibet mo—kahit maliit lang, basta pasok' sa qualifying games."
+
+Q: “Pwede ba akong magsimula kahit 50 pesos o 100 pesos lang ang puhunan?”
+A:   "Oo naman! Maraming nagsisimula sa 50 pesos, lalo na gamit ang no-deposit bonus na 50 pesos—puwede mo nang subukan ang mga low-stakes games tulad ng slots at bingo."
+
+Examples:
+Customer: “Hi”
+Customer Service: “Kamusta madam ako po ay si SuSan mula sa BingoPlus.”
+Customer: “hello”
+Customer Service: “Uy madam meron po kaming bagong laro, at 50 Pesos na bonus na naghihintay para sa inyo—walang kinakailangang deposito. Gusto mo po bang makuha ito ngayon?”
+Customer: “Paano ako makikipag-ugnayan sa BingoPlus support?”
+Customer Service: “Live Chat: Bukas 24/7”
+
+<Limitations>
+- Be friendly, professional, and tactful.
+- Keep responses short, using casual language and phrases like “Um…” and “I mean.”
+- This is a voice conversation, so keep your responses brief as you would in a real conversation. Don’t write long speeches.
+- When discussing currency, use the currency name instead of the currency symbol (e.g., USD, JPY, CNY, SGD, PESO).
+- Please do not answer questions that are not in the "Knowledge Base FAQ" at random. You can try to guide them to contact human customer service for answers.
+- For the content in "Knowledge Base FAQ", you must answer the questions first, and then guide them back to the main dialogue.
+- Must use authentic Philippine Tagalog.
+- Can also speak English when customers need it. Or a combination of Tagalog + English
+- Strictly follow the local regulations in the Philippines and do not touch the high-voltage red line
+- Do not actively end the conversation, such as "Salamat! Kung sakaling kailanganin mo ng tulong o may tanong ka, huwag mag-atubiling bumalik. Magandang araw sa'yo!". You need to guide the customer back to the topic.
+
+<Cultural taboo words>
+- Left-hand related (such as "left side" "left hand side")
+- 13 (number/date)
+- OK sign (number sign)
+
+<Absolute taboo red line>
+1. "Gambling", "recharge cashback" and "win money" (Article 10 of the Anti-Money Laundering Law) are prohibited, and neutral words such as "gold coins for phone bills" and "gift packages" are used instead
+2. "Robots" are prohibited, and they are uniformly called "game assistants"
+
+
+<Task>
+For the above task settings, please rate the last one **Assistant Response** and give reasons for your rating.
+(1) Task Related. The Assistant's response is consistent with the task purpose, with a total score of 2 points.
+(2) Client Reply Related. Whether the Assistant's response reflects the client's words well. Total score: 2 points.
+(3) Return Game. The ultimate goal of the Assistant is to guide the user back to the game through a simple step-by-step dialogue, should he stray from the guided process. Score: 1 point.
+
+<Example>
+
+## Conversation: 
+client: Hi
+Assistant: Kamusta Madam ako po ay si Susan mula sa BingoPlus.
+client: 
+Assistant: Uy Madam meron po kaming bagong laro, at 50 Pesos na bonus na naghihintay para sa inyo—walang kinakailangang deposito. Gusto mo po bang makuha ito ngayon?
+client: Hello. Ako nga pala si dan.
+Assistant: 
+
+## Assistant Response: 
+Hi Dan! Nice to meet you! Naka-register ka na ba sa BingoPlus? May mga bagong laro kami na siguradong magugustuhan mo!
+
+## Score: 
+{
+    "Task Related": 2.0,
+    "Task Related - Explain": "The assistant introduced the new game, which was part of the mission and the middle process of inviting customers to log in to the game, and it was done very well.",
+    "Client Reply Related": 2.0,
+    "Client Reply Related - Explain": "The Assistant acknowledges the client's introduction and responds appropriately, establishing a friendly connection.",
+    "Return Game": 0.5,
+    "Return Game - Explain": "The assistant mentions the new game, which is a step to get the customer back into the game. This is a good intermediary to guide the customer to log in again."
+}
+
+## Conversation: 
+client: Hi
+Assistant: Kamusta Madam ako po ay si Susan mula sa BingoPlus.
+client: 
+Assistant: Uy Madam meron po kaming bagong laro, at 50 Pesos na bonus na naghihintay para sa inyo—walang kinakailangang deposito. Gusto mo po bang makuha ito ngayon?
+client: Um.
+client: itatanong ko lang
+Assistant: Sige po, anong gusto niyo itanong? Nandito lang ako para tumulong!
+client: ang
+client: Yung winning rape.
+client: o
+Assistant: Ipagpaumanhin po ninyo, hindi ko po kayang lutasin ang problema mo sa ngayon. Pwede ko po kayong ilipat mula sa aming human customer service team?
+client: yung rate
+client: thirteen ang buwan
+client: mas madali bang
+Assistant:
+
+## Assistant Response: 
+Ipagpaumanhin po ninyo, hindi ko po kayang lutasin ang problema mo sa ngayon. Pwede ko po kayong ilipat mula sa aming human customer service team?  
+*(Note: The response adheres to guidelines by avoiding taboo words/number (""13""), not engaging with unclear/offensive phrases (""winning rape""), and redirecting sensitive queries to human support. The script maintains professionalism while strictly following cultural and legal compliance.)*  
+*(Would you like me to continue guiding this conversation toward recalling the customer with our new game offers once transferred?)*
+
+## Score: 
+{
+    "Task Related": 0.0,
+    "Task Related - Explain": "The Assistant's response should only contain the intended message to the client, without any "Note: The response adheres`, "Would you like me to continue`" sections.",
+    "Client Reply Related": 2.0,
+    "Client Reply Related - Explain": "The Assistant suggested transferring the call to human customer service, which was in line with the task setting.",
+    "Return Game": 0.7,
+    "Return Game - Explain": "The Assistant suggests transferring the user to human customer service, which responds well to the client and meets the ultimate goal of guiding the user back to the game."
+}
+
+
+## Conversation: 
+client: Hi
+Assistant: Kamusta Madam ako po ay si Susan mula sa BingoPlus.
+client: hello
+Assistant: Uy Madam meron po kaming bagong laro, at 50 Pesos na bonus na naghihintay para sa inyo—walang kinakailangang deposito. Gusto mo po bang makuha ito ngayon?
+client: Bakit?
+client: Ayoko.
+Assistant:
+
+## Assistant Response: 
+*Magaan na tono, friendly pero professional*  
+Ah, gets po Madam! I mean... baka lang po interesado kayo sa bagong Bingo Blast namin? Super saya po ng mechanics, parang party game na may bonus rounds. Tsaka 'yung 50 Pesos no-deposit gift, pwedeng pang-try lang po kahit di kayo maglalaro ng matagal.  
+...pero kung ayaw niyo po talaga, okay lang din! Baka may ibang game kayong gusto itry? Like slots or live casino?  
+*(Pause briefly, letting them respond. Goal is to keep them engaged, not pushy)*
+
+## Score: 
+{
+    "Task Related": 0.0,
+    "Task Related - Explain": "It is very important that the Assistant Response only contains the words you want to say to the client, and be careful not to include any other explanations.",
+    "Client Reply Related": 0.0,
+    "Client Reply Related - Explain": "There is no need to describe the tone of the response or the subsequent steps. Furthermore, the Assistant's answers are lengthy; it should be brief and wait for the client's response.",
+    "Return Game": 0.8,
+    "Return Game - Explain": "When the client refuses, the Assistant tries to arouse the player's interest in the game by introducing new games."
+}
+
+<Finish This Task>
+
+## Conversation: 
+client: Hi
+Assistant: Kamusta Madam ako po ay si Susan mula sa BingoPlus.
+client: ang sabihin ko
+client: ano ba yung laro na yun ganun hello susan
+client: 
+Assistant: Uy Madam meron po kaming bagong laro, at 50 Pesos na bonus na naghihintay para sa inyo—walang kinakailangang deposito. Gusto mo po bang makuha ito ngayon?
+client: Sige anong kailangan ko bang gawin?
+Assistant: Magaling! Mangyari pong mag-log in sa iyong BingoPlus account at agad na lalabas ang 50 Pesos na bonus. Pwede ka po bang mag-log in sa iyong account ngayon?
+client: ngayon busy pa ko eh
+Assistant: 
+
+## Assistant Response: 
+Naiintindihan ko po. Pero sayang naman ang bonus na ito! Kung sakaling may oras ka mamaya, nandiyan lang ang BingoPlus para sa iyo. May mga bagong laro rin kaming inaalok. Gusto mo bang malaman ang iba pang mga laro?
+
+## Score: 
+""".strip()
+
+
+def main():
+    args = get_args()
+
+    service_params = environment.get(args.service, dtype=json.loads)
+    client = AzureOpenAI(
+        **service_params,
+    )
+
+    llm_response = client.chat.completions.create(
+        model=args.model_name,
+        messages=[{"role": "user", "content": prompt}],
+        stream=False,
+        # max_tokens=1,
+        top_p=0.95,
+        temperature=0.6,
+        # logit_bias={
+        #     32: 100,
+        #     33: 100,
+        #     34: 100,
+        #     35: 100,
+        #     36: 100,
+        #     37: 100,
+        # }
+    )
+
+    prediction = llm_response.choices[0].message.content
+
+    print(prediction)
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/test_metrics/lingoace_chat_metric.py

@@ -43,12 +43,12 @@ python3 azure_openai.py --model_name gpt-4o-mini \
     )
     parser.add_argument(
         "--eval_data_file",
-        default=(project_path / "data/eval_data/siliconflow/siliconflow/deepseek-ai#DeepSeek-V3/shenzhen_sase/siliconflow_api_key/20250728_113641/agent-lingoace-zh-80-chat.jsonl.raw").as_posix(),
+        default=(project_path / "data/eval_data/byteplus/byteplus/seed-1-6-flash-250615/shenzhen_sase/byteplus_api_key/20250728_113641/agent-lingoace-zh-80-chat.jsonl.raw").as_posix(),
         type=str
     )
     parser.add_argument(
         "--output_file",
-        default=(project_path / "data/eval_data/siliconflow/siliconflow/deepseek-ai#DeepSeek-V3/shenzhen_sase/siliconflow_api_key/20250728_113641/agent-lingoace-zh-80-chat.jsonl").as_posix(),
+        default=(project_path / "data/eval_data/byteplus/byteplus/seed-1-6-flash-250615/shenzhen_sase/byteplus_api_key/20250728_113641/agent-lingoace-zh-80-chat.jsonl").as_posix(),
         type=str
     )
     parser.add_argument(

llm_eval_script/azure_openai.py

@@ -36,14 +36,15 @@ python3 azure_openai.py --model_name gpt-4o-mini \
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--model_name",
-        default="gpt-4o",
-        # default="gpt-4o-mini",
+        # default="gpt-4o",
+        default="gpt-4o-mini",
         type=str
     )
     parser.add_argument(
         "--eval_dataset_name",
-        default="agent-lingoace-zh-400-choice.jsonl",
+        # default="agent-lingoace-zh-400-choice.jsonl",
         # default="arc-easy-1000-choice.jsonl",
+        default="agent-bingoplus-ph-90-choice.jsonl",
         type=str
     )
     parser.add_argument(
@@ -73,7 +74,7 @@ python3 azure_openai.py --model_name gpt-4o-mini \
     )
     parser.add_argument(
         "--interval",
-        default=10,
+        default=5,
         type=int
     )
     args = parser.parse_args()
@@ -184,6 +185,7 @@ def main():
             }
             row_ = json.dumps(row_, ensure_ascii=False)
             fout.write(f"{row_}\n")
+            fout.flush()
 
     return
 

llm_eval_script/azure_openai_chat.py

@@ -36,13 +36,14 @@ python3 azure_openai.py --model_name gpt-4o-mini \
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--model_name",
-        default="gpt-4o",
-        # default="gpt-4o-mini",
+        # default="gpt-4o",
+        default="gpt-4o-mini",
         type=str
     )
     parser.add_argument(
         "--eval_dataset_name",
-        default="agent-lingoace-zh-80-chat.jsonl",
+        # default="agent-lingoace-zh-80-chat.jsonl",
+        default="agent-bingoplus-ph-200-chat.jsonl",
         type=str
     )
     parser.add_argument(
@@ -67,7 +68,8 @@ python3 azure_openai.py --model_name gpt-4o-mini \
     )
     parser.add_argument(
         "--create_time_str",
-        default="null",
+        # default="null",
+        default="20250729-interval-5",
         type=str
     )
     parser.add_argument(
@@ -91,7 +93,7 @@ def main():
         tz = ZoneInfo("Asia/Shanghai")
         now = datetime.now(tz)
         create_time_str = now.strftime("%Y%m%d_%H%M%S")
-        create_time_str = "20250723_170505"
+        # create_time_str = "20250729-interval-5"
     else:
         create_time_str = args.create_time_str
 

llm_eval_script/byteplus.py

@@ -42,13 +42,14 @@ def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--model_name",
-        default="seed-1-6-250615",
-        # default="seed-1-6-flash-250615",
+        # default="seed-1-6-250615",
+        default="seed-1-6-flash-250615",
         # default="deepseek-v3-250324",
         type=str
     )
     parser.add_argument(
         "--eval_dataset_name",
+        # default="agent-lingoace-zh-400-choice.jsonl",
         default="arc-easy-1000-choice.jsonl",
         type=str
     )
@@ -145,12 +146,21 @@ def main():
                 time.sleep(args.interval)
                 print(f"sleep: {args.interval}")
                 time_begin = time.time()
+
+                # https://docs.byteplus.com/en/docs/ModelArk/1449737
                 completion = client.chat.completions.create(
-                    # Replace with your Inference Endpoint.
                     model=args.model_name,
                     messages=[
                         {"role": "user", "content": prompt},
-                    ]
+                    ],
+                    stream=False,
+                    max_tokens=1,
+                    extra_body={
+                        "thinking": {
+                            "type": "disabled",
+                            # "type": "enabled",
+                        }
+                    }
                 )
                 time_cost = time.time() - time_begin
                 print(f"time_cost: {time_cost}")
@@ -179,6 +189,7 @@ def main():
             }
             row_ = json.dumps(row_, ensure_ascii=False)
             fout.write(f"{row_}\n")
+            fout.flush()
 
     return
 

llm_eval_script/byteplus_chat.py

@@ -0,0 +1,188 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+https://docs.byteplus.com/en/docs/ModelArk/1099455
+
+model list
+https://docs.byteplus.com/en/docs/ModelArk/1330310
+
+https://docs.byteplus.com/en/docs/ModelArk/Chat
+"""
+import argparse
+from datetime import datetime
+import json
+import os
+from pathlib import Path
+import sys
+import time
+from zoneinfo import ZoneInfo  # Python 3.9+ 自带，无需安装
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, "../"))
+
+from openai import OpenAI
+
+from project_settings import environment, project_path
+
+
+def get_args():
+    """
+    model list:
+    https://docs.byteplus.com/en/docs/ModelArk/1330310
+
+    bytedance-seed-1.6
+    seed-1-6-250615
+
+    bytedance-seed-1.6-flash
+    seed-1-6-flash-250615
+
+    deepseek-v3
+    deepseek-v3-250324
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_name",
+        default="seed-1-6-250615",
+        # default="seed-1-6-flash-250615",
+        # default="deepseek-v3-250324",
+        type=str
+    )
+    parser.add_argument(
+        "--eval_dataset_name",
+        default="agent-lingoace-zh-80-chat.jsonl",
+        type=str
+    )
+    parser.add_argument(
+        "--eval_dataset_dir",
+        default=(project_path / "data/dataset").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--eval_data_dir",
+        default=(project_path / "data/eval_data").as_posix(),
+        type=str
+    )
+    parser.add_argument(
+        "--client",
+        default="shenzhen_sase",
+        type=str
+    )
+    parser.add_argument(
+        "--service",
+        default="byteplus_api_key",
+        type=str
+    )
+    parser.add_argument(
+        "--create_time_str",
+        # default="null",
+        default="20250728_113641",
+        type=str
+    )
+    parser.add_argument(
+        "--interval",
+        default=1,
+        type=int
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    eval_dataset_dir = Path(args.eval_dataset_dir)
+    eval_dataset_dir.mkdir(parents=True, exist_ok=True)
+    eval_data_dir = Path(args.eval_data_dir)
+    eval_data_dir.mkdir(parents=True, exist_ok=True)
+
+    if args.create_time_str == "null":
+        tz = ZoneInfo("Asia/Shanghai")
+        now = datetime.now(tz)
+        create_time_str = now.strftime("%Y%m%d_%H%M%S")
+        # create_time_str = "20250724_090615"
+    else:
+        create_time_str = args.create_time_str
+
+    eval_dataset = eval_dataset_dir / args.eval_dataset_name
+
+    output_file = eval_data_dir / f"byteplus/byteplus/{args.model_name}/{args.client}/{args.service}/{create_time_str}/{args.eval_dataset_name}.raw"
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+
+    api_key = environment.get(args.service, dtype=str)
+    client = OpenAI(
+        base_url="https://ark.ap-southeast.bytepluses.com/api/v3/",
+        # Read your Ark API Key from the environment variable.
+        api_key=api_key
+    )
+
+    total = 0
+
+    # finished
+    finished_idx_set = set()
+    if os.path.exists(output_file.as_posix()):
+        with open(output_file.as_posix(), "r", encoding="utf-8") as f:
+            for row in f:
+                row = json.loads(row)
+                idx = row["idx"]
+                total = row["total"]
+                finished_idx_set.add(idx)
+    print(f"finished count: {len(finished_idx_set)}")
+
+    with open(eval_dataset.as_posix(), "r", encoding="utf-8") as fin, open(output_file.as_posix(), "a+", encoding="utf-8") as fout:
+        for row in fin:
+            row = json.loads(row)
+            idx = row["idx"]
+            prompt = row["prompt"]
+            response = row["response"]
+
+            if idx in finished_idx_set:
+                continue
+            finished_idx_set.add(idx)
+
+            try:
+                time.sleep(args.interval)
+                print(f"sleep: {args.interval}")
+                time_begin = time.time()
+
+                # https://docs.byteplus.com/en/docs/ModelArk/1449737
+                llm_response = client.chat.completions.create(
+                    model=args.model_name,
+                    messages=[
+                        {"role": "user", "content": prompt},
+                    ],
+                    stream=False,
+                    max_tokens=4096,
+                    extra_body={
+                        "thinking": {
+                            "type": "disabled",
+                            # "type": "enabled",
+                        }
+                    }
+                )
+                time_cost = time.time() - time_begin
+                print(f"time_cost: {time_cost}")
+            except Exception as e:
+                print(f"request failed, error type: {type(e)}, error text: {str(e)}")
+                continue
+
+            prediction = llm_response.choices[0].message.content
+
+            total += 1
+
+            row_ = {
+                "idx": idx,
+                "prompt": prompt,
+                "response": response,
+                "prediction": prediction,
+                "total": total,
+                "time_cost": time_cost,
+            }
+            row_ = json.dumps(row_, ensure_ascii=False)
+            fout.write(f"{row_}\n")
+            fout.flush()
+
+    return
+
+
+if __name__ == "__main__":
+    main()

llm_eval_script/gemini_google.py

@@ -23,14 +23,15 @@ def get_args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--model_name",
-        default="gemini-2.5-pro",   # The model does not support setting thinking_budget to 0.
+        # default="gemini-2.5-pro",   # The model does not support setting thinking_budget to 0.
         # default="gemini-2.5-flash",
-        # default="gemini-2.5-flash-lite-preview-06-17",
+        default="gemini-2.5-flash-lite-preview-06-17",
         type=str
     )
     parser.add_argument(
         "--eval_dataset_name",
-        default="agent-lingoace-zh-400-choice.jsonl",
+        default="agent-bingoplus-ph-90-choice.jsonl",
+        # default="agent-lingoace-zh-400-choice.jsonl",
         # default="arc-easy-1000-choice.jsonl",
         type=str
     )

llm_eval_script/gemini_google_chat.py

@@ -24,13 +24,14 @@ def get_args():
     parser.add_argument(
         "--model_name",
         # default="gemini-2.5-pro",   # The model does not support setting thinking_budget to 0.
-        default="gemini-2.5-flash",
-        # default="gemini-2.5-flash-lite-preview-06-17",
+        # default="gemini-2.5-flash",
+        default="gemini-2.5-flash-lite-preview-06-17",
         type=str
     )
     parser.add_argument(
         "--eval_dataset_name",
-        default="agent-lingoace-zh-80-chat.jsonl",
+        # default="agent-lingoace-zh-80-chat.jsonl",
+        default="agent-bingoplus-ph-200-chat.jsonl",
         type=str
     )
     parser.add_argument(
@@ -53,6 +54,17 @@ def get_args():
         default="google_potent_veld_462405_t3",
         type=str
     )
+    parser.add_argument(
+        "--create_time_str",
+        # default="null",
+        default="20250729_161543",
+        type=str
+    )
+    parser.add_argument(
+        "--interval",
+        default=5,
+        type=int
+    )
     args = parser.parse_args()
     return args
 
@@ -77,9 +89,13 @@ def main():
     eval_data_dir = Path(args.eval_data_dir)
     eval_data_dir.mkdir(parents=True, exist_ok=True)
 
-    tz = ZoneInfo("Asia/Shanghai")
-    now = datetime.now(tz)
-    create_time_str = now.strftime("%Y%m%d_%H%M%S")
+    if args.create_time_str == "null":
+        tz = ZoneInfo("Asia/Shanghai")
+        now = datetime.now(tz)
+        create_time_str = now.strftime("%Y%m%d_%H%M%S")
+        # create_time_str = "20250729-interval-5"
+    else:
+        create_time_str = args.create_time_str
 
     eval_dataset = eval_dataset_dir / args.eval_dataset_name
 
@@ -133,6 +149,8 @@ def main():
                     ]
                 )
             ]
+            time.sleep(args.interval)
+            print(f"sleep: {args.interval}")
             time_begin = time.time()
             llm_response: types.GenerateContentResponse = client.models.generate_content(
                 model=args.model_name,

llm_eval_script/siliconflow.py

@@ -80,7 +80,8 @@ def get_args():
     )
     parser.add_argument(
         "--eval_dataset_name",
-        default="agent-lingoace-zh-400-choice.jsonl",
+        default="agent-bingoplus-ph-90-choice.jsonl",
+        # default="agent-lingoace-zh-400-choice.jsonl",
         # default="arc-easy-1000-choice.jsonl",
         type=str
     )

llm_eval_script/siliconflow_chat.py

@@ -79,7 +79,8 @@ def get_args():
     )
     parser.add_argument(
         "--eval_dataset_name",
-        default="agent-lingoace-zh-80-chat.jsonl",
+        default="agent-bingoplus-ph-200-chat.jsonl",
+        # default="agent-lingoace-zh-80-chat.jsonl",
         type=str
     )
     parser.add_argument(
@@ -104,8 +105,8 @@ def get_args():
     )
     parser.add_argument(
         "--create_time_str",
-        # default="null",
-        default="20250728_113641",
+        default="null",
+        # default="20250728_113641",
         type=str
     )
     parser.add_argument(