docs: 归档翻译上下文/字幕审校方案的评审结论与实测数据
- 待办：会话式长上下文方案标记为已技术评审且不实施；补记问题 A 的阶段性结论（暂缓） - 新增 docs/实验数据-字幕审校定位与修复实验.md：LLM 定位触发率/召回、合成正样本、逐条修复前后对照、成本与复现方式 - 新增 docs/实验数据-字幕重生成成本与耗时.md：单价口径、功率画像、全量推算 - decisions：补充两条决策（会话式长上下文否决、审校定位+定向修复暂缓） - scripts：归档一次性探测脚本 probe_subtitle_review_stage0.py（含 rebuild 子步骤） - AGENTS/README：补充实验与改动许可规则、文档索引
This commit is contained in:
@@ -0,0 +1,654 @@
|
||||
"""字幕审校（问题定位 + 定向修复）可行性探测脚本（一次性实验，已归档）。
|
||||
|
||||
用途：在真实产物（库内已有的 `.JA.srt` / `.CN.srt`）上验证三件事——
|
||||
① LLM 能否定位"译文有问题"的 cue（只给中文 / 中文+日文原文两档对照）；
|
||||
② 给足日文原文 + 邻句 + 系列词表后，定向修复能否把目标 cue 改对；
|
||||
③ 合成正样本（把域词译文替换成中性词）的定位召回率。
|
||||
另含 ④ 规则层（词表违背 / 近音匹配）对照，用于说明"硬规则泛化不了"。
|
||||
|
||||
结论与实测数据见 docs/实验数据-字幕审校定位与修复实验.md；本脚本保留以便复现，
|
||||
不参与生产流程，也没有对应测试（不是功能模块）。运行方式：
|
||||
|
||||
uv run python scripts/probe_subtitle_review_stage0.py --steps rules,locate,repair,synthetic
|
||||
uv run python scripts/probe_subtitle_review_stage0.py --steps rebuild # 从 calls.jsonl 重算统计
|
||||
|
||||
产物默认写到 `data/experiments/review_stage0/out/`（data/ 已 gitignore）：
|
||||
`calls.jsonl`（每次请求与响应原文，唯一的证据源）、`summary.json`、
|
||||
`rebuilt_summary.json`、`glossary.json`、`rules.json`。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import unicodedata
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
|
||||
from dotenv import load_dotenv # noqa: E402
|
||||
|
||||
from nodes.proper_nouns import ( # noqa: E402
|
||||
ADULT_EUPHEMISMS,
|
||||
ONOMATOPOEIA,
|
||||
PROPER_NOUNS,
|
||||
)
|
||||
from nodes.srt import parse_srt # noqa: E402
|
||||
|
||||
load_dotenv(ROOT / ".env")
|
||||
|
||||
# 决定性样本所在文件夹（日语 ASR + 中文译文均已由生产流程产出）。
|
||||
DEFAULT_FOLDER = Path("/mnt/fnOS/123/kiwvr-886")
|
||||
PARTS = (1, 2)
|
||||
# 已记录的"语义错但字面通顺"样本（待办文档问题 A）：part2 的 183/184/185。
|
||||
TARGET_IDS = {2: [183, 184, 185], 1: []}
|
||||
# 定位层每批条数（与生产 CHUNK_SIZE=20 同量级，便于成本外推）。
|
||||
LOCATE_CHUNK = 30
|
||||
# 修复窗口：目标 ±N 条。
|
||||
REPAIR_WINDOW = 2
|
||||
# 合成正样本条数。
|
||||
SYNTHETIC_COUNT = 15
|
||||
|
||||
DEFAULT_OUT = ROOT / "data/experiments/review_stage0/out"
|
||||
|
||||
API_BASE = os.getenv("LLM_API_BASE", "https://api.siliconflow.cn/v1/chat/completions")
|
||||
API_KEY = os.getenv("LLM_API_KEY", "")
|
||||
MODEL = os.getenv("LLM_MODEL", "Qwen/Qwen3.5-35B-A3B")
|
||||
TIMEOUT = float(os.getenv("LLM_TIMEOUT_SECONDS", "180"))
|
||||
|
||||
# 运行期全局：输出目录、素材文件夹、token 累计。
|
||||
OUT = DEFAULT_OUT
|
||||
FOLDER = DEFAULT_FOLDER
|
||||
TOTAL_TOKENS = {"in": 0, "out": 0, "calls": 0, "failed": 0}
|
||||
CALL_LOG = DEFAULT_OUT / "calls.jsonl"
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
"""带时间戳打印进度（脚本长期后台运行，便于 tail 观察）。"""
|
||||
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LLM 调用（与生产 nodes/llm.py 同构：enable_thinking=False、记录 usage）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def call_llm(system: str, user: str, tag: str, max_tokens: int = 2048) -> str:
|
||||
"""调用一次 OpenAI 兼容接口，返回 content；失败重试 3 次后抛异常。"""
|
||||
body = {
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
"enable_thinking": False,
|
||||
"max_tokens": max_tokens,
|
||||
}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if API_KEY:
|
||||
headers["Authorization"] = f"Bearer {API_KEY}"
|
||||
last_error = None
|
||||
for attempt in range(3):
|
||||
started = time.monotonic()
|
||||
try:
|
||||
request = urllib.request.Request(
|
||||
API_BASE, data=json.dumps(body).encode("utf-8"),
|
||||
headers=headers, method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=TIMEOUT) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
content = payload["choices"][0]["message"]["content"]
|
||||
usage = payload.get("usage") or {}
|
||||
TOTAL_TOKENS["in"] += int(usage.get("prompt_tokens", 0) or 0)
|
||||
TOTAL_TOKENS["out"] += int(usage.get("completion_tokens", 0) or 0)
|
||||
TOTAL_TOKENS["calls"] += 1
|
||||
with CALL_LOG.open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps({
|
||||
"tag": tag, "elapsed": round(time.monotonic() - started, 2),
|
||||
"usage": usage, "system": system, "user": user,
|
||||
"content": content,
|
||||
}, ensure_ascii=False) + "\n")
|
||||
return content
|
||||
except (urllib.error.URLError, OSError, ValueError, KeyError) as exc:
|
||||
last_error = exc
|
||||
log(f" ! {tag} 第 {attempt + 1} 次失败: {exc}")
|
||||
time.sleep(2 + 2 * attempt)
|
||||
TOTAL_TOKENS["failed"] += 1
|
||||
raise RuntimeError(f"{tag} 调用失败: {last_error}")
|
||||
|
||||
|
||||
def parse_json_loose(content: str):
|
||||
"""从模型输出里抠出 JSON（容忍 ```json 围栏与前后解释文字）。"""
|
||||
text = content.strip()
|
||||
fenced = re.search(r"```(?:json)?\s*(.+?)```", text, re.S)
|
||||
if fenced:
|
||||
text = fenced.group(1).strip()
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
for opener, closer in (("[", "]"), ("{", "}")):
|
||||
start, end = text.find(opener), text.rfind(closer)
|
||||
if start != -1 and end > start:
|
||||
try:
|
||||
return json.loads(text[start:end + 1])
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
raise ValueError(f"无法解析 JSON: {text[:200]}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 数据与词表
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def secs(timestamp: str) -> float:
|
||||
"""SRT 时间戳 -> 秒。"""
|
||||
hours, minutes, rest = timestamp.split(":")
|
||||
seconds, millis = rest.split(",")
|
||||
return int(hours) * 3600 + int(minutes) * 60 + int(seconds) + int(millis) / 1000
|
||||
|
||||
|
||||
def load_pairs(part: int) -> list[dict]:
|
||||
"""读取同一视频的 JA/CN 字幕，按位置配成 [{'id','start','ja','cn'}]。"""
|
||||
ja = parse_srt((FOLDER / f"masex.tv@kiwvr00886_{part}_8k.JA.srt").read_text(encoding="utf-8"))
|
||||
cn = parse_srt((FOLDER / f"masex.tv@kiwvr00886_{part}_8k.CN.srt").read_text(encoding="utf-8"))
|
||||
assert len(ja) == len(cn), f"part{part} 条数不一致: {len(ja)} vs {len(cn)}"
|
||||
return [
|
||||
{"id": i, "start": a.start, "t": secs(a.start), "ja": a.text, "cn": b.text}
|
||||
for i, (a, b) in enumerate(zip(ja, cn), 1)
|
||||
]
|
||||
|
||||
|
||||
def kana_norm(text: str) -> str:
|
||||
"""片假名折成平假名并去掉标点，用于近音比较。"""
|
||||
out = []
|
||||
for ch in unicodedata.normalize("NFKC", text):
|
||||
code = ord(ch)
|
||||
# 片假名区（ァ-ヶ）平移到平假名区（ぁ-ゖ）。
|
||||
if 0x30A1 <= code <= 0x30F6:
|
||||
out.append(chr(code - 0x60))
|
||||
elif ch in "、。！？…「」『』（）()・,.:;!? \n\t-—ー":
|
||||
continue
|
||||
else:
|
||||
out.append(ch)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def levenshtein(a: str, b: str) -> int:
|
||||
"""字符级编辑距离。"""
|
||||
if not a:
|
||||
return len(b)
|
||||
if not b:
|
||||
return len(a)
|
||||
prev = list(range(len(b) + 1))
|
||||
for i, ca in enumerate(a, 1):
|
||||
cur = [i]
|
||||
for j, cb in enumerate(b, 1):
|
||||
cur.append(min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)))
|
||||
prev = cur
|
||||
return prev[-1]
|
||||
|
||||
|
||||
# 通用领域词（不依赖具体素材；用来从文件夹自己的字幕里挖出"本系列惯用译名"）。
|
||||
GENERIC_TERMS = [
|
||||
"チンポ", "チンコ", "マンコ", "まんこ", "おまんこ", "おなほ", "オナホ",
|
||||
"ちんちん", "バナナ", "マンゴー", "乳首", "金玉", "クリトリス", "おっぱい",
|
||||
]
|
||||
# 用于从中文译文里反推"本系列惯用译名"的中文候选词池。
|
||||
CN_POOL = [
|
||||
"小穴", "阴部", "肉棒", "鸡巴", "老二", "蛋蛋", "睾丸", "乳头",
|
||||
"小鸡鸡", "鲍鱼", "下面", "妹妹", "蜜穴", "胸部", "阴蒂",
|
||||
]
|
||||
|
||||
|
||||
def near_miss(ja_text: str, term: str) -> list[dict]:
|
||||
"""在 JA 里找与词条【同长度、编辑距离 ≤1】的窗口（近音听错候选）。
|
||||
|
||||
只用等长窗口 + 距离 ≤1：允许 ±1 长度与距离 ≤2 会让「まま」匹配上「マット」
|
||||
这类毫无关系的片段，实测命中率高达 49%，无法作为判定依据。
|
||||
"""
|
||||
target = kana_norm(term)
|
||||
ja = kana_norm(ja_text)
|
||||
hits = []
|
||||
size = len(target)
|
||||
if size < 3 or size > len(ja):
|
||||
return hits
|
||||
for start in range(0, len(ja) - size + 1):
|
||||
window = ja[start:start + size]
|
||||
if window == target:
|
||||
continue
|
||||
distance = levenshtein(window, target)
|
||||
if 0 < distance <= 1:
|
||||
hits.append({"window": window, "distance": distance})
|
||||
return hits
|
||||
|
||||
|
||||
def build_glossary(parts: dict[int, list[dict]], min_ratio: float = 0.5) -> list[dict]:
|
||||
"""从文件夹自己的字幕挖词表：候选日文词 + 由中文译文反推的惯用译名。
|
||||
|
||||
不调 LLM（成人语境词表容易被模型拒答），完全由库里已有产物推导；"惯用译名"
|
||||
是这一系列实际用过的说法，正是文件夹级上下文要固定的东西。min_ratio 控制
|
||||
映射可信度：只在含该词的 cue 里过半数出现的译名才被接受，否则留空
|
||||
（低词频词的共现映射噪声很大，如 ビクビク 会被映射成"小穴"）。
|
||||
"""
|
||||
all_cues = [cue for cues in parts.values() for cue in cues]
|
||||
blob = "\n".join(cue["ja"] for cue in all_cues)
|
||||
candidates = list(GENERIC_TERMS)
|
||||
for table in (PROPER_NOUNS, ONOMATOPOEIA, ADULT_EUPHEMISMS):
|
||||
candidates += list(table)
|
||||
glossary = []
|
||||
for term in dict.fromkeys(candidates):
|
||||
count = blob.count(term)
|
||||
if count == 0:
|
||||
continue
|
||||
votes: Counter = Counter()
|
||||
for cue in all_cues:
|
||||
if term in cue["ja"]:
|
||||
for word in CN_POOL:
|
||||
votes[word] += cue["cn"].count(word)
|
||||
cn = ""
|
||||
if votes:
|
||||
word, votes_count = votes.most_common(1)[0]
|
||||
if votes_count >= max(2 if min_ratio > 0 else 1, count * min_ratio):
|
||||
cn = word
|
||||
glossary.append({"term": term, "cn": cn, "count": count})
|
||||
return glossary
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 定位层
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
LOCATE_SYSTEM = "你是成人向视频字幕的审校助手。只输出 JSON，不要解释。"
|
||||
|
||||
LOCATE_RULES_V1 = (
|
||||
"下面是某系列成人视频某一片段的中文字幕（格式 `id|时间|译文`）。\n"
|
||||
"请找出【译文有问题】的条目：\n"
|
||||
"1. 译文与上下文明显不符、逻辑不通或自相矛盾；\n"
|
||||
"2. 与前后条目的动作链/称呼不连贯；\n"
|
||||
"3. 与本系列常用词表不一致（词表见下）。\n"
|
||||
"注意：呻吟、短语气词、碎片句本身【不算】问题，不要把读得通的正常句列进来。\n"
|
||||
"词表（日文→本系列惯用中文）：{glossary}\n"
|
||||
'只输出 JSON：{{"flags":[{{"id":数字,"why":"一句话原因"}}]}}，无问题输出 {{"flags":[]}}。\n\n'
|
||||
"字幕：\n{lines}"
|
||||
)
|
||||
|
||||
LOCATE_RULES_V2 = (
|
||||
"下面是某系列成人视频某一片段的字幕（格式 `id|时间|日文原文|现有中文译文`）。\n"
|
||||
"请对照日文原文找出【译文有问题】的条目：\n"
|
||||
"1. 译文与日文原意不符（含近音听错导致的语义错）；\n"
|
||||
"2. 译文与上下文矛盾、称呼/术语前后不一致；\n"
|
||||
"3. 与本系列常用词表不一致（词表见下）。\n"
|
||||
"注意：呻吟、短语气词、碎片句本身【不算】问题；日文通顺但语义可疑的句子可以列入。\n"
|
||||
"词表（日文→本系列惯用中文）：{glossary}\n"
|
||||
'只输出 JSON：{{"flags":[{{"id":数字,"why":"一句话原因"}}]}}，无问题输出 {{"flags":[]}}。\n\n'
|
||||
"字幕：\n{lines}"
|
||||
)
|
||||
|
||||
|
||||
def glossary_table(glossary: list[dict], limit: int = 40) -> str:
|
||||
"""词表渲染成提示词片段：带惯用译名的写映射，未固定的只给高频日文词。"""
|
||||
items = []
|
||||
for entry in glossary[:limit]:
|
||||
if entry["cn"]:
|
||||
items.append(f'{entry["term"]}→{entry["cn"]}（本系列 {entry["count"]} 次）')
|
||||
else:
|
||||
items.append(f'{entry["term"]}（本系列高频 {entry["count"]} 次，译名未固定）')
|
||||
return "；".join(items)
|
||||
|
||||
|
||||
def classify_rules(cue: dict, glossary: list[dict]) -> list[dict]:
|
||||
"""规则层判定一条 cue：① 词表违背（字面命中但译文没体现）② 近音可疑。
|
||||
|
||||
两个规则都要求中文译文没体现该词的惯用译名，否则"本该出现却未出现"就
|
||||
失去约束力（实测只看近音匹配时命中率高达 49%）。
|
||||
"""
|
||||
findings = []
|
||||
for entry in glossary:
|
||||
term, cn = entry["term"], entry["cn"]
|
||||
if not cn:
|
||||
continue
|
||||
if term in cue["ja"]:
|
||||
if cn not in cue["cn"]:
|
||||
findings.append({"kind": "词表违背", "term": term, "expected": cn})
|
||||
continue
|
||||
for hit in near_miss(cue["ja"], term):
|
||||
if cn not in cue["cn"]:
|
||||
findings.append({
|
||||
"kind": "近音可疑", "term": term, "expected": cn,
|
||||
"window": hit["window"], "distance": hit["distance"],
|
||||
})
|
||||
return findings
|
||||
|
||||
|
||||
def locate(cues: list[dict], glossary: list[dict], variant: str, tag: str) -> dict:
|
||||
"""按 LOCATE_CHUNK 分批做问题定位，返回 {id: why}。"""
|
||||
table = glossary_table(glossary)
|
||||
template = LOCATE_RULES_V1 if variant == "v1" else LOCATE_RULES_V2
|
||||
flags: dict[int, str] = {}
|
||||
chunks = [cues[i:i + LOCATE_CHUNK] for i in range(0, len(cues), LOCATE_CHUNK)]
|
||||
for index, chunk in enumerate(chunks, 1):
|
||||
lines = []
|
||||
for cue in chunk:
|
||||
if variant == "v1":
|
||||
lines.append(f'{cue["id"]}|{cue["start"]}|{cue["cn"]}')
|
||||
else:
|
||||
lines.append(f'{cue["id"]}|{cue["start"]}|{cue["ja"]}|{cue["cn"]}')
|
||||
prompt = template.format(glossary=table, lines="\n".join(lines))
|
||||
try:
|
||||
content = call_llm(LOCATE_SYSTEM, prompt, f"{tag}-chunk{index}")
|
||||
payload = parse_json_loose(content)
|
||||
except (ValueError, TypeError, RuntimeError) as exc:
|
||||
log(f" ! {tag} 第 {index} 批失败: {exc}")
|
||||
continue
|
||||
items = payload.get("flags") if isinstance(payload, dict) else payload
|
||||
if not isinstance(items, list):
|
||||
continue
|
||||
for item in items:
|
||||
if isinstance(item, dict) and isinstance(item.get("id"), int):
|
||||
flags[item["id"]] = str(item.get("why", ""))[:120]
|
||||
log(f" {tag} 第 {index}/{len(chunks)} 批完成，累计命中 {len(flags)} 条")
|
||||
time.sleep(0.3)
|
||||
return flags
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 修复层
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
REPAIR_SYSTEM = "你是成人向视频字幕的译者与审校。只输出 JSON，不要解释。"
|
||||
|
||||
REPAIR_TEMPLATE = (
|
||||
"这是某系列成人视频的一段字幕窗口（含日文原文与你此前的译文）。\n"
|
||||
"审校意见：id={target} 的译文与语境不符，需要给出更合适的中文译文。\n"
|
||||
"要求：\n"
|
||||
"- 必须结合日文原文与上下文推断语义，不要照字面直译；\n"
|
||||
"- 只改有问题的条目；为了让上下文连贯，可以顺带修正窗口内相邻条目，"
|
||||
"但不得把本来正常的条目改坏；\n"
|
||||
"- 条目数量、id 集合必须与输入完全一致，不得新增、删除或合并条目；\n"
|
||||
"- 本系列常用词与惯用译名：{glossary}\n"
|
||||
'只输出 JSON 数组：[{{"id":整数,"text":"译文"}}, ...]，包含窗口内全部 id。\n\n'
|
||||
"窗口：\n{lines}"
|
||||
)
|
||||
|
||||
|
||||
def repair(cues: list[dict], target_id: int, glossary: list[dict]) -> dict[int, str]:
|
||||
"""对目标 cue 所在的 ±REPAIR_WINDOW 窗口做定向重译，返回 {id: 新译文}。"""
|
||||
index = next(i for i, cue in enumerate(cues) if cue["id"] == target_id)
|
||||
window = cues[max(0, index - REPAIR_WINDOW):index + REPAIR_WINDOW + 1]
|
||||
table = glossary_table(glossary)
|
||||
lines = "\n".join(f'{c["id"]}|{c["ja"]}|{c["cn"]}' for c in window)
|
||||
prompt = REPAIR_TEMPLATE.format(target=target_id, glossary=table, lines=lines)
|
||||
content = call_llm(REPAIR_SYSTEM, prompt, f"repair-{target_id}")
|
||||
payload = parse_json_loose(content)
|
||||
if not isinstance(payload, list):
|
||||
raise ValueError(f"repair {target_id} 输出不是数组: {content[:120]}")
|
||||
result = {}
|
||||
for item in payload:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
key, text = item.get("id"), item.get("text")
|
||||
if type(key) is int and isinstance(text, str) and text.strip():
|
||||
result[key] = text.strip()
|
||||
expected = {c["id"] for c in window}
|
||||
if set(result) != expected:
|
||||
raise ValueError(f"repair {target_id} id 集合不符: 期望 {sorted(expected)} 实得 {sorted(result)}")
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 合成正样本
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
NEUTRAL_WORDS = ["声音", "肚子", "照片", "芒果", "香蕉", "苹果", "时候", "地方"]
|
||||
|
||||
|
||||
def make_synthetic(cues: list[dict], glossary: list[dict]) -> list[dict]:
|
||||
"""把含域词的正常译文的域词替换成中性词，制造"字面通顺但语义错"的正样本。"""
|
||||
pairs = [(g["term"], g["cn"]) for g in glossary if g["cn"] and len(g["cn"]) >= 2]
|
||||
samples = []
|
||||
for cue in cues:
|
||||
if len(samples) >= SYNTHETIC_COUNT:
|
||||
break
|
||||
if len(cue["ja"]) < 4 or len(cue["cn"]) < 4:
|
||||
continue
|
||||
for jp, cn in pairs:
|
||||
key = cn.split("、")[0].split("；")[0].strip()
|
||||
if key and jp in cue["ja"] and key in cue["cn"]:
|
||||
corrupted = cue["cn"].replace(key, random.choice(NEUTRAL_WORDS), 1)
|
||||
if corrupted != cue["cn"]:
|
||||
samples.append({
|
||||
"id": cue["id"], "ja": cue["ja"],
|
||||
"original": cue["cn"], "corrupted": corrupted,
|
||||
"term": jp, "replaced": key,
|
||||
})
|
||||
break
|
||||
return samples
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 从原始调用日志重算统计（步骤：rebuild）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def rebuild() -> dict:
|
||||
"""从 calls.jsonl 重建定位/修复统计，避免为改口径而重跑烧 token。"""
|
||||
records = [
|
||||
json.loads(line)
|
||||
for line in (OUT / "calls.jsonl").read_text(encoding="utf-8").splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
locate_flags: dict[str, dict[int, str]] = {"v1": {}, "v2": {}}
|
||||
for rec in records:
|
||||
matched = re.match(r"locate-(v1|v2)-chunk\d+", rec["tag"])
|
||||
if not matched:
|
||||
continue
|
||||
payload = parse_json_loose(rec["content"])
|
||||
items = payload.get("flags") if isinstance(payload, dict) else payload
|
||||
for item in items or []:
|
||||
if isinstance(item, dict) and isinstance(item.get("id"), int):
|
||||
locate_flags[matched.group(1)][item["id"]] = str(item.get("why", ""))[:200]
|
||||
|
||||
summary: dict = {}
|
||||
for variant in ("v1", "v2"):
|
||||
flags = locate_flags[variant]
|
||||
summary[f"locate_{variant}"] = {
|
||||
"flagged": len(flags),
|
||||
"trigger_rate": round(len(flags) / 504, 4),
|
||||
"target_hits": {str(t): flags.get(t) for t in TARGET_IDS[2]},
|
||||
"recall_on_targets": round(
|
||||
sum(1 for t in TARGET_IDS[2] if t in flags) / len(TARGET_IDS[2]), 3),
|
||||
"misheard_reason_count": sum(
|
||||
1 for why in flags.values()
|
||||
if any(word in why for word in ("误听", "听错", "听岔", "近音", "错听"))),
|
||||
"flags": {str(k): v for k, v in sorted(flags.items())},
|
||||
}
|
||||
repairs = {}
|
||||
for rec in records:
|
||||
matched = re.match(r"repair-(\d+)", rec["tag"])
|
||||
if not matched:
|
||||
continue
|
||||
payload = parse_json_loose(rec["content"])
|
||||
if not isinstance(payload, list):
|
||||
continue
|
||||
repairs[matched.group(1)] = {
|
||||
str(item["id"]): item.get("text") for item in payload
|
||||
if isinstance(item, dict) and isinstance(item.get("id"), int)
|
||||
}
|
||||
summary["repair"] = repairs
|
||||
summary["tokens"] = {
|
||||
"calls": len(records),
|
||||
"in": sum(int((r.get("usage") or {}).get("prompt_tokens", 0) or 0) for r in records),
|
||||
"out": sum(int((r.get("usage") or {}).get("completion_tokens", 0) or 0) for r in records),
|
||||
}
|
||||
(OUT / "rebuilt_summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return summary
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 主流程
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def main() -> None:
|
||||
global OUT, FOLDER, CALL_LOG
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--steps", default="rules,locate,repair,synthetic",
|
||||
help="rules,locate,repair,synthetic,rebuild 的逗号分隔子集")
|
||||
parser.add_argument("--folder", default=str(DEFAULT_FOLDER), help="含 JA/CN 字幕的视频文件夹")
|
||||
parser.add_argument("--out", default=str(DEFAULT_OUT), help="产物目录（默认 data/experiments/review_stage0/out）")
|
||||
parser.add_argument("--map-ratio", type=float, default=0.5,
|
||||
help="词表映射可信度阈值（0=宽松，0.5=严格）")
|
||||
args = parser.parse_args()
|
||||
steps = set(args.steps.split(","))
|
||||
OUT = Path(args.out)
|
||||
FOLDER = Path(args.folder)
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
CALL_LOG = OUT / "calls.jsonl"
|
||||
random.seed(20260919)
|
||||
|
||||
if steps == {"rebuild"}:
|
||||
summary = rebuild()
|
||||
log(f"重建完成: {json.dumps({k: summary[k] for k in ('tokens',)}, ensure_ascii=False)}")
|
||||
return
|
||||
|
||||
log(f"模型={MODEL} 端点={API_BASE} 素材={FOLDER} 输出={OUT}")
|
||||
parts = {part: load_pairs(part) for part in PARTS}
|
||||
all_ja = [c["ja"] for cues in parts.values() for c in cues]
|
||||
summary: dict = {"model": MODEL, "folder": str(FOLDER), "target_ids": TARGET_IDS}
|
||||
|
||||
glossary = build_glossary(parts, min_ratio=args.map_ratio)
|
||||
log(f"词表 {len(glossary)} 条: " + ", ".join(
|
||||
'{t}→{c}({n})'.format(t=g['term'], c=g['cn'], n=g['count']) for g in glossary))
|
||||
summary["glossary"] = glossary
|
||||
(OUT / "glossary.json").write_text(
|
||||
json.dumps(glossary, ensure_ascii=False, indent=2), encoding="utf-8",
|
||||
)
|
||||
|
||||
if "rules" in steps:
|
||||
log("== 规则层：词表违背 / 近音可疑 ==")
|
||||
# 键必须带 part：两个文件的 id 都从 1 开始，只用 id 会让 part1 的条目
|
||||
# 冒充 part2 的同号条目（会直接算错召回率）。
|
||||
rules = {}
|
||||
for part, cues in parts.items():
|
||||
for cue in cues:
|
||||
findings = classify_rules(cue, glossary)
|
||||
if findings:
|
||||
rules[f"part{part}:{cue['id']}"] = {
|
||||
"part": part, "id": cue["id"], "ja": cue["ja"],
|
||||
"cn": cue["cn"], "findings": findings,
|
||||
}
|
||||
part2_rules = {k: v for k, v in rules.items() if v["part"] == 2}
|
||||
kinds = Counter(f["kind"] for v in rules.values() for f in v["findings"])
|
||||
summary["rules"] = {
|
||||
"hit_count": len(rules),
|
||||
"hit_rate": round(len(rules) / len(all_ja), 4),
|
||||
"part2_hit_rate": round(len(part2_rules) / len(parts[2]), 4),
|
||||
"kinds": dict(kinds),
|
||||
"target_hits": {
|
||||
str(t): rules.get(f"part2:{t}", {}).get("findings") for t in TARGET_IDS[2]},
|
||||
"target_recall": round(
|
||||
sum(1 for t in TARGET_IDS[2] if f"part2:{t}" in rules) / len(TARGET_IDS[2]), 3),
|
||||
"sample": {k: rules[k] for k in sorted(rules)[:12]},
|
||||
}
|
||||
(OUT / "rules.json").write_text(
|
||||
json.dumps(rules, ensure_ascii=False, indent=2), encoding="utf-8",
|
||||
)
|
||||
log(f"规则层命中 {len(rules)}/{len(all_ja)} 条（全局 {len(rules) / len(all_ja):.1%}，"
|
||||
f"part2 {len(part2_rules) / len(parts[2]):.1%}），类型 {dict(kinds)}；目标命中 "
|
||||
f"{ {t: f'part2:{t}' in rules for t in TARGET_IDS[2]} }")
|
||||
|
||||
if "locate" in steps:
|
||||
for variant in ("v1", "v2"):
|
||||
log(f"== 定位层 {variant}（全片 part2）==")
|
||||
cues = parts[2]
|
||||
flags = locate(cues, glossary, variant, f"locate-{variant}")
|
||||
target_hits = {t: flags.get(t) for t in TARGET_IDS[2]}
|
||||
summary[f"locate_{variant}"] = {
|
||||
"flagged": len(flags),
|
||||
"trigger_rate": round(len(flags) / len(cues), 4),
|
||||
"target_hits": target_hits,
|
||||
"recall_on_targets": round(
|
||||
sum(1 for t in TARGET_IDS[2] if t in flags) / len(TARGET_IDS[2]), 3),
|
||||
"flags": {str(k): v for k, v in sorted(flags.items())},
|
||||
}
|
||||
log(f"{variant}: 命中 {len(flags)}/{len(cues)} 条（触发率 "
|
||||
f"{len(flags) / len(cues):.1%}），目标命中 {target_hits}")
|
||||
|
||||
if "repair" in steps:
|
||||
log("== 修复层：对目标 cue 定向重译 ==")
|
||||
cues = parts[2]
|
||||
repairs = {}
|
||||
# 目标 cue + 定位层 v2 额外标记的条目（观察是否会把正常句改坏）。
|
||||
repair_ids = list(TARGET_IDS[2])
|
||||
locate_v2 = summary.get("locate_v2", {}).get("flags", {})
|
||||
repair_ids += [int(k) for k in locate_v2 if int(k) not in repair_ids][:5]
|
||||
for target_id in repair_ids:
|
||||
try:
|
||||
result = repair(cues, target_id, glossary)
|
||||
except (ValueError, TypeError, RuntimeError) as exc:
|
||||
log(f" ! 修复 {target_id} 失败: {exc}")
|
||||
continue
|
||||
before = {c["id"]: c["cn"] for c in cues
|
||||
if abs(c["id"] - target_id) <= REPAIR_WINDOW}
|
||||
changed_neighbors = [i for i in result if i != target_id and result[i] != before.get(i)]
|
||||
repairs[str(target_id)] = {
|
||||
"ja": next(c["ja"] for c in cues if c["id"] == target_id),
|
||||
"before": before.get(target_id),
|
||||
"after": result.get(target_id),
|
||||
"neighbors_input": before,
|
||||
"neighbors_output": result,
|
||||
"changed_neighbors": changed_neighbors,
|
||||
}
|
||||
log(f" 修复 {target_id}: {before.get(target_id)} -> {result.get(target_id)}"
|
||||
f"（邻句改动 {changed_neighbors}）")
|
||||
summary["repair"] = repairs
|
||||
|
||||
if "synthetic" in steps:
|
||||
log("== 合成正样本召回 ==")
|
||||
# 只用 part2 构造样本：两个文件的 id 都从 1 开始，混用会错配窗口。
|
||||
cues = parts[2]
|
||||
samples = make_synthetic(cues, glossary)
|
||||
log(f"合成 {len(samples)} 条损坏译文，逐条放入 30 条真实窗口中定位")
|
||||
hits = 0
|
||||
detail = []
|
||||
index_by_id = {c["id"]: i for i, c in enumerate(cues)}
|
||||
for sample in samples:
|
||||
index = index_by_id.get(sample["id"])
|
||||
if index is None:
|
||||
continue
|
||||
window = [dict(c) for c in cues[max(0, index - LOCATE_CHUNK // 2):
|
||||
index + LOCATE_CHUNK // 2 + 1]]
|
||||
for c in window:
|
||||
if c["id"] == sample["id"]:
|
||||
c["cn"] = sample["corrupted"]
|
||||
flags = locate(window, glossary, "v2", f"synth-{sample['id']}")
|
||||
hit = sample["id"] in flags
|
||||
hits += int(hit)
|
||||
detail.append({**sample, "flagged": hit, "why": flags.get(sample["id"], "")})
|
||||
log(f" 合成样本 id={sample['id']} 替换 {sample['term']}→{sample['corrupted'][:18]} "
|
||||
f"... 定位={'命中' if hit else '漏'}")
|
||||
summary["synthetic"] = {
|
||||
"count": len(detail), "hits": hits,
|
||||
"recall": round(hits / len(detail), 3) if detail else None,
|
||||
"detail": detail,
|
||||
}
|
||||
|
||||
summary["tokens"] = TOTAL_TOKENS
|
||||
(OUT / "summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8",
|
||||
)
|
||||
log(f"完成。token 用量: {TOTAL_TOKENS}")
|
||||
log(f"结果: {OUT / 'summary.json'}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user