cat-shark
8963afa771
为评估"更便宜的 LLM 能否替换 llm-translate 默认模型"新增真实数据驱动的 评测工具（不 mock 模型，直接调用生产节点实现，模型仅作为 params 变量）： - scripts/build_translate_eval.py：把 ocr-subtitle 的中文烧录字幕产物与 learn-translate 的日语 whisper ASR 按时间配对，产出候选池； - scripts/build_segment_eval.py：改为**窗口级**配对（一条中文基准字幕 + 其时间窗内 1-3 条日语 cue）。逐条配对不可用——烧录字幕是按屏幕合并的 整行，与 whisper 的 cue 切分不同，直接逐条对照会被基准错位污染； - scripts/bench_translate_models.py：对每个模型跑完整片、记录单次调用耗时 与 token、输出窗口级多模型对照表供人工 review；VL 模型需剔除 enable_thinking（Qwen3-VL 不接受该参数，生产代码固定携带）； - scripts/run_translate_bench_queue.py：批量评测队列； - tests/test_translate_model_bench.py：评测数据契约与对照输出一致性测试 （含先红后绿修复：stdout 与 markdown 两套输出格式漂移）； - testdata/translate_eval/：候选池、窗口池与 124 窗口人工评测集资产。 评测结论见 data/experiments/translate_models/REPORT.md（gitignored）。
62 lines
2.5 KiB
Python
62 lines
2.5 KiB
Python
"""按序跑完全部评测模型（云端优先，本地模型等下载完成后再跑）。
|
|
|
|
每个模型一条命令，串行执行（不并发，符合本次评测目标）；stdout/耗时写入
|
|
各 tag 目录下的 run.log。中断后重跑会跳过已有 summary.json 的 tag。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
OUT_ROOT = PROJECT_ROOT / "data/experiments/translate_models"
|
|
|
|
CLOUD_BASE = "https://api.siliconflow.cn/v1/chat/completions"
|
|
LOCAL_BASE = "http://localhost:11434/v1/chat/completions"
|
|
|
|
# (tag, model, 额外参数)。基线 Qwen3.6-35B-A3B 单列，不重跑（已有 run_d386 产物）。
|
|
TASKS: list[tuple[str, str, list[str]]] = [
|
|
("cloud-qwen3.5-35b-a3b", "Qwen/Qwen3.5-35B-A3B", []),
|
|
("cloud-qwen3-vl-30b-a3b", "Qwen/Qwen3-VL-30B-A3B-Instruct", ["--strip-thinking"]),
|
|
("cloud-qwen3-14b", "Qwen/Qwen3-14B", []),
|
|
("cloud-qwen3-8b", "Qwen/Qwen3-8B", []),
|
|
("cloud-qwen3.5-9b", "Qwen/Qwen3.5-9B", []),
|
|
("local-qwen3-14b", "qwen3:14b", ["--api-base", LOCAL_BASE, "--api-key", "", "--timeout", "900", "--warmup"]),
|
|
("local-qwen3-30b-a3b", "qwen3:30b-a3b", ["--api-base", LOCAL_BASE, "--api-key", "", "--timeout", "900", "--warmup"]),
|
|
]
|
|
|
|
|
|
def main() -> None:
|
|
only = sys.argv[1:] or None
|
|
for tag, model, extra in TASKS:
|
|
if only and tag not in only:
|
|
continue
|
|
out_dir = OUT_ROOT / tag
|
|
if (out_dir / "summary.json").is_file():
|
|
print(f"[skip] {tag} 已有 summary.json", flush=True)
|
|
continue
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
cmd = [
|
|
sys.executable, str(PROJECT_ROOT / "scripts/bench_translate_models.py"), "run",
|
|
"--model", model, "--tag", tag, *extra,
|
|
]
|
|
print(f"[start] {tag} :: {' '.join(cmd[2:])}", flush=True)
|
|
started = time.monotonic()
|
|
with (out_dir / "run.log").open("w", encoding="utf-8") as log:
|
|
proc = subprocess.run(cmd, cwd=PROJECT_ROOT, stdout=log, stderr=subprocess.STDOUT)
|
|
elapsed = time.monotonic() - started
|
|
print(f"[done] {tag} rc={proc.returncode} {elapsed/60:.1f}min", flush=True)
|
|
summary = out_dir / "summary.json"
|
|
if summary.is_file():
|
|
data = json.loads(summary.read_text(encoding="utf-8"))
|
|
print(" ", {k: data.get(k) for k in
|
|
("status", "cues_out", "wall_s", "per_cue_s", "failed_calls", "error")}, flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|