cat-shark
a032855210
两部分同属字幕质量优化，共用 llm.py 翻译链路：
1) 长时寒暄幻觉词清洗（nodes/subtitle_cleanup.py）
对展示时长超过阈值（默认 15s，实测 30s 幻觉占位 vs 2s 真实词的分界）
且文本含收尾/开场寒暄（晚安、感谢观看等）的字幕，文本替换为 '-'，
由后续过滤流程移除；短时寒暄（如剧情中真实互道晚安）保留不误删。
写文件前调用 。
2) 专名/隐语不直译（nodes/proper_nouns.py）
片假名专名（人名/品牌/角色）与成人语境隐语是 LLM 误译重灾区：
- ジンゴ 被误译成'芒果'、カンタくん 被保留日文而非音译；
- マンゴー/バナナ/リンゴ/金玉/おまんこ/ちんちん 等在色情语境中
是生殖器官代称，字面直译严重错译。
整理三张规则表（专名/拟声词/成人隐语，用户提供隐语表），
检测原文命中后动态注入翻译提示词，
让 LLM 按正确语义处理。文档见 docs/proper_nouns.md。
测试（红→绿）：
- tests/test_hallucination_mask.py：长时掩码/短时保留/非寒暄保留/阈值边界
- tests/test_proper_nouns.py：专名命中/拟声词/成人隐语/真实数据集成
- 真实 LLM 集成测试（完整 1440 行翻译 + 清洗 + 专名注入）通过
110 lines
4.7 KiB
Python
110 lines
4.7 KiB
Python
"""专有名词（不应直译）处理规则测试（先红后绿）。
|
||
|
||
背景（实测 run 20260905115050）：字幕中片假名专名被 LLM 按读音硬译——
|
||
'ジンゴ' 被误译成 '芒果'（4500s"看，跟芒果摩擦好多"），参考正确为
|
||
'肉棒摩擦小穴' 等；人名 'カンタくん' 被保留日文而非音译。LLM 需被告知
|
||
这些专名/拟声词的正确处理方式。
|
||
|
||
方案（用户确认）：整理专名表，翻译时若原文命中则向提示词动态注入规则，
|
||
让 LLM 正确处理。本测试验证 `build_proper_noun_rule` 的命中与注入行为
|
||
（纯函数），以及真实数据场景的端到端效果（真实 LLM 集成）。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from nodes.proper_nouns import build_proper_noun_rule
|
||
|
||
WORKSPACE = Path(__file__).resolve().parent.parent
|
||
# 真实中文字幕产物（含"芒果"误译）。目录按实际路径调整。
|
||
PROD = Path("/home/cat/Downloads/39.105.149.197/202609051952/CJOD-255-长视频-单行字幕.zh-CN.20260905115050.srt")
|
||
# 真实日文原文 transcript（含 ジンゴ/カンタくん 等专名）。
|
||
TRANSCR = Path("/home/cat/Downloads/39.105.149.197/202609051737/run_51242078d76e/steps/asr/transcript.srt")
|
||
|
||
|
||
def test_rule_returns_none_when_no_proper_noun() -> None:
|
||
"""原文不含任何专名时，不注入规则（返回 None），避免干扰普通翻译。"""
|
||
plain = "今天天气真好。\n我们一起去散步吧。"
|
||
assert build_proper_noun_rule(plain) is None
|
||
|
||
|
||
def test_rule_injects_for_jingo() -> None:
|
||
"""原文含'ジンゴ'（角色/道具专名）时，注入'禁止译作芒果'的规则。"""
|
||
text = "クモ穴にパンパンになって ジンゴがここで味わいませんか"
|
||
rule = build_proper_noun_rule(text)
|
||
assert rule is not None
|
||
assert "ジンゴ" in rule
|
||
assert "芒果" in rule # 提示禁止硬译
|
||
assert "不要按读音或字面硬译" in rule # 提示禁止硬译
|
||
|
||
def test_rule_injects_for_kanta_kun() -> None:
|
||
"""原文含'カンタくん'（人名）时，注入'音译勿保留日文'的规则。"""
|
||
text = "ねえ、カンタくん、4つんばんになってください"
|
||
rule = build_proper_noun_rule(text)
|
||
assert rule is not None
|
||
assert "カンタくん" in rule
|
||
assert "康太君" in rule or "坎塔君" in rule
|
||
|
||
|
||
def test_rule_injects_onomatopoeia() -> None:
|
||
"""原文含拟声词'パンパン'时，注入'鼓胀、饱满'而非'砰砰'的规则。"""
|
||
text = "クモ穴にパンパンになって"
|
||
rule = build_proper_noun_rule(text)
|
||
assert rule is not None
|
||
assert "パンパン" in rule
|
||
assert "砰砰" in rule # 提示禁止直译
|
||
|
||
|
||
def test_rule_list_input() -> None:
|
||
"""传入列表（每批字幕行）也能命中。"""
|
||
lines = ["音楽", "ご来店ありがとうございます", "ジンゴがここで味わいませんか"]
|
||
rule = build_proper_noun_rule(lines)
|
||
assert rule is not None
|
||
assert "ジンゴ" in rule
|
||
|
||
|
||
def test_rule_injects_adult_euphemism_mango() -> None:
|
||
"""成人隐语'マンゴー'：应注入'小穴/鲍鱼'而非直译'芒果'。"""
|
||
text = "クモ穴にマンゴーをこすり合わせて"
|
||
rule = build_proper_noun_rule(text)
|
||
assert rule is not None
|
||
assert "マンゴー" in rule
|
||
assert "小穴" in rule or "鲍鱼" in rule
|
||
assert "芒果" in rule
|
||
|
||
|
||
def test_rule_injects_adult_euphemism_kintama() -> None:
|
||
"""成人隐语'金玉'：应注入'蛋蛋/睾丸'而非直译'金玉'。"""
|
||
text = "金玉が大きくなってきた"
|
||
rule = build_proper_noun_rule(text)
|
||
assert rule is not None
|
||
assert "金玉" in rule
|
||
assert "蛋蛋" in rule or "睾丸" in rule
|
||
|
||
|
||
def test_rule_injects_adult_euphemism_banana() -> None:
|
||
"""成人隐语'バナナ'：应注入'肉棒'而非直译'香蕉'。"""
|
||
text = "バナナをしゃぶって"
|
||
rule = build_proper_noun_rule(text)
|
||
assert rule is not None
|
||
assert "バナナ" in rule
|
||
assert "肉棒" in rule or "鸡鸡" in rule
|
||
|
||
|
||
@pytest.mark.integration
|
||
def test_real_proper_noun_rule_matches_production_data() -> None:
|
||
"""用真实 transcript 验证：含'ジンゴ'的批次确实命中并注入规则。"""
|
||
if not TRANSCR.is_file():
|
||
pytest.skip("缺少真实 transcript.srt，跳过")
|
||
from tests.realdata_contract import parse_srt_entries
|
||
|
||
entries = parse_srt_entries(TRANSCR.read_text(encoding="utf-8"))
|
||
# 找到含 ジンゴ 的批次（4500-4518s 那批）。
|
||
batch = [e["text"] for e in entries if 4490 <= e["start"] <= 4520]
|
||
rule = build_proper_noun_rule(batch)
|
||
assert rule is not None, "真实数据中含 ジンゴ，应命中专名规则"
|
||
assert "ジンゴ" in rule and "芒果" in rule |