update

.gitignore

@@ -5,6 +5,7 @@
 #/data/
 /data/comment
 #/data/eval_data
+data/llm-log
 /data/raw_dataset
 /dotenv/
 /logs/

data/eval_data/byteplus/byteplus/seed-1-6-250615/shenzhen_sase/byteplus_api_key/20250728_113641/arc-easy-1000-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49e98cb6d61aa488ab7182e77412ce5714fdb36cff9d90c48c380fc9a076163d
+size 397093

data/eval_data/siliconflow/siliconflow/deepseek-ai#DeepSeek-V3/shenzhen_sase/siliconflow_api_key/20250728_113641/agent-lingoace-zh-400-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb441dc5ebeddd2c0d53d4e8f1919550f870e07cdcc9f4569eaec8a73464b287
+size 1211343

data/eval_data/siliconflow/siliconflow/deepseek-ai#DeepSeek-V3/shenzhen_sase/siliconflow_api_key/20250728_113641/agent-lingoace-zh-80-chat.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5606f956ede82a224bff0430c7496192fc4c8fc3fce35ba703dfea1f7e9b4399
+size 877608

data/eval_data/siliconflow/siliconflow/deepseek-ai#DeepSeek-V3/shenzhen_sase/siliconflow_api_key/20250728_113641/arc-easy-1000-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3c91dce22c3349d86618e64297edb8a5d2671fc7b11fed01f9da52a161e44f6
+size 720488

data/eval_data/siliconflow/siliconflow/deepseek-ai#DeepSeek-V3/shenzhen_sase/siliconflow_api_key/20250728_135005/agent-lingoace-zh-400-choice.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9df3d2954a107a163041528409232c7b578c085929758c4d8f80954548f7a54a
+size 1211301

examples/test_metrics/lingoace_chat_metric.py

@@ -43,12 +43,12 @@ python3 azure_openai.py --model_name gpt-4o-mini \
     )
     parser.add_argument(
         "--eval_data_file",
-        default=(project_path / "data/eval_data/azure_openai/azure/gpt-4o/shenzhen_sase/west_us_chatgpt_openai_azure_com/20250723_170505/agent-lingoace-zh-80-chat.jsonl.raw").as_posix(),
+        default=(project_path / "data/eval_data/siliconflow/siliconflow/deepseek-ai#DeepSeek-V3/shenzhen_sase/siliconflow_api_key/20250728_113641/agent-lingoace-zh-80-chat.jsonl.raw").as_posix(),
         type=str
     )
     parser.add_argument(
         "--output_file",
-        default=(project_path / "data/eval_data/azure_openai/azure/gpt-4o/shenzhen_sase/west_us_chatgpt_openai_azure_com/20250723_170505/agent-lingoace-zh-80-chat.jsonl").as_posix(),
+        default=(project_path / "data/eval_data/siliconflow/siliconflow/deepseek-ai#DeepSeek-V3/shenzhen_sase/siliconflow_api_key/20250728_113641/agent-lingoace-zh-80-chat.jsonl").as_posix(),
         type=str
     )
     parser.add_argument(

llm_eval_script/byteplus.py

@@ -5,6 +5,8 @@ https://docs.byteplus.com/en/docs/ModelArk/1099455
 
 model list
 https://docs.byteplus.com/en/docs/ModelArk/1330310
+
+https://docs.byteplus.com/en/docs/ModelArk/Chat
 """
 import argparse
 from datetime import datetime
@@ -24,10 +26,25 @@ from project_settings import environment, project_path
 
 
 def get_args():
+    """
+    model list:
+    https://docs.byteplus.com/en/docs/ModelArk/1330310
+
+    bytedance-seed-1.6
+    seed-1-6-250615
+
+    bytedance-seed-1.6-flash
+    seed-1-6-flash-250615
+
+    deepseek-v3
+    deepseek-v3-250324
+    """
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--model_name",
-        default="seedance-1-0-lite-t2v-250428",
+        default="seed-1-6-250615",
+        # default="seed-1-6-flash-250615",
+        # default="deepseek-v3-250324",
         type=str
     )
     parser.add_argument(
@@ -55,6 +72,17 @@ def get_args():
         default="byteplus_api_key",
         type=str
     )
+    parser.add_argument(
+        "--create_time_str",
+        # default="null",
+        default="20250728_113641",
+        type=str
+    )
+    parser.add_argument(
+        "--interval",
+        default=1,
+        type=int
+    )
     args = parser.parse_args()
     return args
 
@@ -67,9 +95,13 @@ def main():
     eval_data_dir = Path(args.eval_data_dir)
     eval_data_dir.mkdir(parents=True, exist_ok=True)
 
-    tz = ZoneInfo("Asia/Shanghai")
-    now = datetime.now(tz)
-    create_time_str = now.strftime("%Y%m%d_%H%M%S")
+    if args.create_time_str == "null":
+        tz = ZoneInfo("Asia/Shanghai")
+        now = datetime.now(tz)
+        create_time_str = now.strftime("%Y%m%d_%H%M%S")
+        # create_time_str = "20250724_090615"
+    else:
+        create_time_str = args.create_time_str
 
     eval_dataset = eval_dataset_dir / args.eval_dataset_name
 
@@ -78,7 +110,7 @@ def main():
 
     api_key = environment.get(args.service, dtype=str)
     client = OpenAI(
-        base_url="https://ark.ap-southeast.bytepluses.com/api/v3",
+        base_url="https://ark.ap-southeast.bytepluses.com/api/v3/",
         # Read your Ark API Key from the environment variable.
         api_key=api_key
     )
@@ -110,6 +142,8 @@ def main():
             finished_idx_set.add(idx)
 
             try:
+                time.sleep(args.interval)
+                print(f"sleep: {args.interval}")
                 time_begin = time.time()
                 completion = client.chat.completions.create(
                     # Replace with your Inference Endpoint.

llm_eval_script/siliconflow.py

@@ -35,6 +35,7 @@ Model Name:
 Qwen/Qwen3-8B
 deepseek-ai/DeepSeek-R1-0528-Qwen3-8B
 deepseek-ai/DeepSeek-R1-Distill-Qwen-7B
+deepseek-ai/DeepSeek-V3
 Tips:
 (1)为了让它只输出一个字符,设置 max_tokens=1
 
@@ -69,7 +70,8 @@ def get_args():
         "--model_name",
         # default="Pro/deepseek-ai/DeepSeek-R1",
         # default="tencent/Hunyuan-A13B-Instruct",
-        default="Qwen/Qwen3-8B",
+        default="deepseek-ai/DeepSeek-V3",
+        # default="Qwen/Qwen3-8B",
         # default="deepseek-ai/DeepSeek-R1",
         # default="deepseek-ai/DeepSeek-R1-0528-Qwen3-8B",
         # default="deepseek-ai/DeepSeek-R1-Distill-Qwen-7B",
@@ -105,11 +107,12 @@ def get_args():
     parser.add_argument(
         "--create_time_str",
         default="null",
+        # default="20250728_113641",
         type=str
     )
     parser.add_argument(
         "--interval",
-        default=10,
+        default=1,
         type=int
     )
     args = parser.parse_args()

llm_eval_script/siliconflow_chat.py

@@ -69,10 +69,11 @@ def get_args():
         "--model_name",
         # default="Pro/deepseek-ai/DeepSeek-R1",
         # default="tencent/Hunyuan-A13B-Instruct",
+        default="deepseek-ai/DeepSeek-V3",
         # default="Qwen/Qwen3-8B",
         # default="deepseek-ai/DeepSeek-R1",
         # default="deepseek-ai/DeepSeek-R1-0528-Qwen3-8B",
-        default="deepseek-ai/DeepSeek-R1-Distill-Qwen-7B",
+        # default="deepseek-ai/DeepSeek-R1-Distill-Qwen-7B",
         # default="baidu/ERNIE-4.5-300B-A47B",
         type=str
     )
@@ -103,7 +104,8 @@ def get_args():
     )
     parser.add_argument(
         "--create_time_str",
-        default="null",
+        # default="null",
+        default="20250728_113641",
         type=str
     )
     parser.add_argument(