cat-shark
3f4478523e
whisper 在单个窗口内卡住重复时会把一个单元写满整条 cue（实测 30 秒整、重复 74–446 次，如 `チン`×111）。这类正文会让下游 LLM 跟着循环、把输出预算耗在思考上， 最终 `content` 返回空串并报 `translation alignment failed … Expecting value: line 1 column 1 (char 0)`；它也可能直接渲染成超长字幕行。 - `nodes/subtitle_cleanup.py` 新增 `remove_repetition_entries`：**纯模式判据、无字符 词表**——展示时长 ≥15s 且同一 1–6 字单元连续重复 ≥6 次且覆盖正文 ≥70% 的 cue 整条 删除；真实短促呻吟（`ぇ`×15、`ああああああ`）靠时长区分，零误删。 - `nodes/whisper.py` 在转写产出处应用该清洗（VAD 开关都生效，它与套话幻觉无关）。 - 真实数据验证：本地全部真实转写 3605 条 cue 只删 5 条 30 秒伪影；对照实验同批次 伪影截短后 5/5 成功、原样 1/5。
333 lines
12 KiB
Python
333 lines
12 KiB
Python
"""nodes/subtitle_cleanup.py 的模块级测试（数据 → 测试过程 → 验证结果）。
|
|
|
|
被测模块：`nodes/subtitle_cleanup.py`（幻觉整条删除 + 短呻吟过滤），被
|
|
whisper（日语链路）与 llm-translate（中文链路）复用，纯函数可独立调用。
|
|
|
|
每个用例构造真实 SRT 文本，调用真实清理函数，并解析输出验证时间轴与序号。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from nodes.subtitle_cleanup import (
|
|
DEFAULT_MOAN_MAX_CHARS,
|
|
HALLUCINATION_TOKENS,
|
|
JAPANESE_HALLUCINATION_TOKENS,
|
|
clean_japanese_hallucinations,
|
|
clean_srt_text,
|
|
remove_hallucination_entries,
|
|
remove_repetition_entries,
|
|
remove_short_moan_entries,
|
|
)
|
|
from tests.shared.srt_entries import parse_srt_entries
|
|
|
|
|
|
def _srt(*cues: tuple[str, str, str]) -> str:
|
|
"""把 (起始, 结束, 文本) 列表拼成标准 SRT 文本，供各用例作为输入数据。"""
|
|
blocks = [
|
|
f"{i}\n{start} --> {end}\n{text}\n"
|
|
for i, (start, end, text) in enumerate(cues, 1)
|
|
]
|
|
return "\n".join(blocks)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 幻觉整条删除（中文词表 / 日语词表）
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_removes_long_hallucination_cue_entirely() -> None:
|
|
"""展示时长达到阈值的寒暄幻觉整条删除（时间轴不残留空 cue）。"""
|
|
# 数据：一条 20s 的"谢谢观看"（超过默认 15s 阈值）。
|
|
text = _srt(
|
|
("00:00:01,000", "00:00:02,000", "真实的对话"),
|
|
("00:00:03,000", "00:00:23,000", "谢谢观看"),
|
|
)
|
|
|
|
# 测试过程
|
|
cleaned = clean_srt_text(text)
|
|
|
|
# 验证结果：只剩真实对话，序号重排为 1，幻想的行完全消失。
|
|
entries = parse_srt_entries(cleaned)
|
|
assert [e["text"] for e in entries] == ["真实的对话"]
|
|
assert cleaned.startswith("1\n")
|
|
assert "谢谢观看" not in cleaned
|
|
|
|
|
|
def test_keeps_short_hallucination_when_inside_threshold() -> None:
|
|
"""展示时长低于阈值的相同词可能是剧情真实内容，必须保留。"""
|
|
# 数据：一条 2s 的"晚安"（剧情中真实互道晚安）。
|
|
text = _srt(("00:00:01,000", "00:00:03,000", "晚安"))
|
|
|
|
# 测试过程
|
|
cleaned = clean_srt_text(text)
|
|
|
|
# 验证结果：保留。
|
|
assert [e["text"] for e in parse_srt_entries(cleaned)] == ["晚安"]
|
|
|
|
|
|
def test_keeps_non_hallucination_long_cue() -> None:
|
|
"""长时但不是幻觉词的内容必须保留（只按词表删除）。"""
|
|
# 数据：一条 30s 的正常长台词。
|
|
text = _srt(("00:00:01,000", "00:00:31,000", "这是一段很长的真实独白内容"))
|
|
|
|
# 测试过程
|
|
cleaned = clean_srt_text(text)
|
|
|
|
# 验证结果
|
|
assert "这是一段很长的真实独白内容" in cleaned
|
|
|
|
|
|
def test_threshold_boundary_is_inclusive() -> None:
|
|
"""阈值边界按"≥ 阈值"删除（严格等于阈值即删除）。"""
|
|
# 数据：恰好 15s 的幻觉条目与 14.999s 的同类条目。
|
|
text = _srt(
|
|
("00:00:00,000", "00:00:15,000", "谢谢观看"),
|
|
("00:00:16,000", "00:00:30,999", "感谢观看"),
|
|
)
|
|
|
|
# 测试过程
|
|
cleaned = clean_srt_text(text, threshold_seconds=15.0)
|
|
|
|
# 验证结果：15s 的被删除，14.999s 的保留。
|
|
kept = [e["text"] for e in parse_srt_entries(cleaned)]
|
|
assert kept == ["感谢观看"]
|
|
|
|
|
|
def test_resequences_after_middle_removal() -> None:
|
|
"""删除中间条目后剩余条目从 1 连续编号，保持合法 SRT。"""
|
|
# 数据：三条，中间一条是长时幻觉。
|
|
text = _srt(
|
|
("00:00:01,000", "00:00:02,000", "第一条"),
|
|
("00:00:03,000", "00:00:25,000", "谢谢观看"),
|
|
("00:00:26,000", "00:00:27,000", "第三条"),
|
|
)
|
|
|
|
# 测试过程
|
|
cleaned = clean_srt_text(text)
|
|
|
|
# 验证结果：序号连续且内容为第一、三条。
|
|
assert cleaned.splitlines()[0] == "1"
|
|
assert "\n2\n" in cleaned
|
|
assert [e["text"] for e in parse_srt_entries(cleaned)] == ["第一条", "第三条"]
|
|
|
|
|
|
def test_japanese_hallucination_removed_and_short_kept() -> None:
|
|
"""日语词表：长时"おやすみなさい"删除，短时保留。"""
|
|
# 数据：一条 30s 日语幻觉 + 一条 3s 同词。
|
|
text = _srt(
|
|
("00:01:00,000", "00:01:30,000", "おやすみなさい"),
|
|
("00:02:00,000", "00:02:03,000", "おやすみなさい"),
|
|
)
|
|
|
|
# 测试过程
|
|
cleaned = clean_japanese_hallucinations(text)
|
|
|
|
# 验证结果：只保留短的那条。
|
|
entries = parse_srt_entries(cleaned)
|
|
assert len(entries) == 1
|
|
assert entries[0]["start"] == 120.0
|
|
|
|
|
|
def test_custom_token_list_is_honored() -> None:
|
|
"""自定义词表生效：只删除传入词命中的条目。"""
|
|
# 数据：两个不同的长时条目。
|
|
text = _srt(
|
|
("00:00:01,000", "00:00:20,000", "自定义幻觉词"),
|
|
("00:00:21,000", "00:00:40,000", "谢谢观看"),
|
|
)
|
|
|
|
# 测试过程：只传"自定义幻觉词"。
|
|
cleaned = remove_hallucination_entries(text, ("自定义幻觉词",))
|
|
|
|
# 验证结果：只删除自定义词，"谢谢观看"保留。
|
|
assert [e["text"] for e in parse_srt_entries(cleaned)] == ["谢谢观看"]
|
|
|
|
|
|
def test_default_tables_are_not_empty() -> None:
|
|
"""两张默认词表都非空（防止重构时误清空导致清理失效）。"""
|
|
# 数据：模块导出的词表常量。
|
|
# 测试过程与验证结果
|
|
assert len(HALLUCINATION_TOKENS) > 0
|
|
assert len(JAPANESE_HALLUCINATION_TOKENS) > 0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# 短呻吟过滤（decode_full 去噪）
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_removes_pure_moan_fragments() -> None:
|
|
"""纯呻吟碎片（あ…/ん?/はぁ…）整条删除。"""
|
|
# 数据：三条纯呻吟与一条真实短对话。
|
|
text = _srt(
|
|
("00:00:01,000", "00:00:02,000", "あ…"),
|
|
("00:00:03,000", "00:00:04,000", "ん?"),
|
|
("00:00:05,000", "00:00:06,000", "はぁ…"),
|
|
("00:00:07,000", "00:00:09,000", "そこ、だめ"),
|
|
)
|
|
|
|
# 测试过程
|
|
cleaned = remove_short_moan_entries(text)
|
|
|
|
# 验证结果：只剩真实短对话。
|
|
assert [e["text"] for e in parse_srt_entries(cleaned)] == ["そこ、だめ"]
|
|
|
|
|
|
def test_keeps_real_short_dialogue() -> None:
|
|
"""含真实假名（そ/や/ね）的短对话不命中判据，必须保留。"""
|
|
# 数据：四条真实短对话。
|
|
dialog = ["そこ", "やばい", "ねえ", "やだ"]
|
|
|
|
# 测试过程
|
|
cleaned = remove_short_moan_entries(_srt(*[
|
|
(f"00:00:0{i},000", f"00:00:0{i + 1},000", text)
|
|
for i, text in enumerate(dialog, 1)
|
|
]))
|
|
|
|
# 验证结果：全部保留。
|
|
assert [e["text"] for e in parse_srt_entries(cleaned)] == dialog
|
|
|
|
|
|
def test_moan_threshold_boundary() -> None:
|
|
"""有效假名数超过阈值（默认 3）的纯呻吟串保留，等于阈值的删除。"""
|
|
# 数据：3 个假名（删除）与 4 个假名（保留）。
|
|
text = _srt(
|
|
("00:00:01,000", "00:00:02,000", "あんあ"),
|
|
("00:00:03,000", "00:00:04,000", "あんあん"),
|
|
)
|
|
|
|
# 测试过程
|
|
cleaned = remove_short_moan_entries(text, max_chars=3)
|
|
|
|
# 验证结果
|
|
assert [e["text"] for e in parse_srt_entries(cleaned)] == ["あんあん"]
|
|
|
|
|
|
def test_moan_default_max_chars_constant() -> None:
|
|
"""默认阈值常量为 3（与文档约定一致，改动需同步文档）。"""
|
|
# 数据：模块常量。
|
|
# 测试过程与验证结果
|
|
assert DEFAULT_MOAN_MAX_CHARS == 3
|
|
|
|
|
|
def test_moan_filter_can_be_disabled() -> None:
|
|
"""max_chars=0 时关闭过滤，输入原样返回。"""
|
|
# 数据：一条纯呻吟。
|
|
text = _srt(("00:00:01,000", "00:00:02,000", "あ…"))
|
|
|
|
# 测试过程与验证结果
|
|
assert remove_short_moan_entries(text, max_chars=0) == text
|
|
|
|
|
|
def test_moan_removal_in_middle_resequences() -> None:
|
|
"""删除中间呻吟后剩余条目序号连续。"""
|
|
# 数据：真实对话、呻吟、真实对话。
|
|
text = _srt(
|
|
("00:00:01,000", "00:00:02,000", "行くよ"),
|
|
("00:00:03,000", "00:00:04,000", "ん…"),
|
|
("00:00:05,000", "00:00:06,000", "だめ"),
|
|
)
|
|
|
|
# 测试过程
|
|
cleaned = remove_short_moan_entries(text)
|
|
|
|
# 验证结果
|
|
assert [e["text"] for e in parse_srt_entries(cleaned)] == ["行くよ", "だめ"]
|
|
assert "\n2\n" in cleaned
|
|
|
|
|
|
def test_multiline_moan_entry_removed_as_one_cue() -> None:
|
|
"""多行呻吟条目整体作为一条 cue 删除（不残留半条）。"""
|
|
# 数据：两行纯呻吟组成一条 cue。
|
|
text = "1\n00:00:01,000 --> 00:00:03,000\nあ…\nん…\n\n2\n00:00:04,000 --> 00:00:05,000\nそこ\n"
|
|
|
|
# 测试过程
|
|
cleaned = remove_short_moan_entries(text)
|
|
|
|
# 验证结果：只剩第二条并重编号。
|
|
assert [e["text"] for e in parse_srt_entries(cleaned)] == ["そこ"]
|
|
assert cleaned.startswith("1\n")
|
|
|
|
|
|
# 真实转写抽样（data/repetition_cues.json）：30 秒窗口被同一单元填满的 whisper
|
|
# 重复伪影 + 真实短呻吟 + 真实台词，用于"重复伪影"判据的正反例。
|
|
_REPETITION_CUES = json.loads(
|
|
(Path(__file__).parent / "data" / "repetition_cues.json").read_text(encoding="utf-8")
|
|
)
|
|
|
|
|
|
def test_remove_repetition_entries_deletes_whisper_loops_only() -> None:
|
|
"""数据：真实 ASR 产物——3 条 30 秒重复伪影（重复 74/111/446 次）、
|
|
3 条 2–3 秒真实呻吟（重复 6–10 次）、1 条正常台词。
|
|
|
|
过程：调用 remove_repetition_entries 清理整份 SRT。
|
|
|
|
验证：只删 30 秒伪影，真实呻吟与台词原样保留，剩余 cue 序号连续。
|
|
"""
|
|
artifacts = _REPETITION_CUES["artifacts"]
|
|
moans = _REPETITION_CUES["moans"]
|
|
normal = _REPETITION_CUES["normal"]
|
|
srt = _srt(
|
|
*[(c["start"], c["end"], c["text"]) for c in artifacts],
|
|
*[(c["start"], c["end"], c["text"]) for c in moans],
|
|
(normal["start"], normal["end"], normal["text"]),
|
|
)
|
|
|
|
cleaned = remove_repetition_entries(srt)
|
|
|
|
for cue in artifacts:
|
|
assert cue["text"] not in cleaned
|
|
for cue in moans:
|
|
assert cue["text"] in cleaned
|
|
assert normal["text"] in cleaned
|
|
entries = parse_srt_entries(cleaned)
|
|
assert [e["text"] for e in entries] == [c["text"] for c in moans] + [normal["text"]]
|
|
# 序号/时间轴重建后从 1 连续编号，不留空号（合法 SRT）。
|
|
numbers = [line for line in cleaned.splitlines() if line.strip().isdigit()]
|
|
assert numbers == [str(i) for i in range(1, len(entries) + 1)]
|
|
|
|
|
|
def test_remove_repetition_entries_keeps_short_repeated_moan() -> None:
|
|
"""数据：3 秒内重复 15 次的真实呻吟（时长不足阈值）。
|
|
|
|
过程：调用 remove_repetition_entries。
|
|
|
|
验证：保留——时长阈值是"窗口被填满"的判据，短促重复属真实发声。
|
|
"""
|
|
srt = _srt(("00:00:01,000", "00:00:04,200", "ぇ" * 15))
|
|
|
|
cleaned = remove_repetition_entries(srt)
|
|
|
|
assert "ぇ" * 15 in cleaned
|
|
|
|
|
|
def test_remove_repetition_entries_keeps_mixed_long_line() -> None:
|
|
"""数据：30 秒长条但正文以正常台词为主，只有少量重复。
|
|
|
|
过程：调用 remove_repetition_entries。
|
|
|
|
验证：保留——重复片段未占正文 70% 以上，不构成重复伪影。
|
|
"""
|
|
text = "そうですね、それでいいと思いますよ" * 3 + "ああ"
|
|
srt = _srt(("00:00:01,000", "00:00:31,000", text))
|
|
|
|
cleaned = remove_repetition_entries(srt)
|
|
|
|
assert text in cleaned
|
|
|
|
|
|
def test_remove_repetition_entries_can_be_disabled() -> None:
|
|
"""数据：一条 30 秒重复伪影，阈值设为 0（关闭）。
|
|
|
|
过程：调用 remove_repetition_entries(threshold_seconds=0)。
|
|
|
|
验证：原样返回，便于按需走旧行为。
|
|
"""
|
|
artifact = _REPETITION_CUES["artifacts"][0]
|
|
srt = _srt((artifact["start"], artifact["end"], artifact["text"]))
|
|
|
|
assert remove_repetition_entries(srt, threshold_seconds=0) == srt
|