feat: llm-filter 两级过滤(规则层+五类分类+去重+长文本保护),工作流单线程 v4

- 规则层(不调 LLM):横线装饰/HTML 水印 token/URL/邮箱/单双 ASCII 字符直接删
- LLM 五类分类:garbage/overlay/noise 删,repeat/dialogue 留,未识别回退保留
- 按文本去重:相同文本只调一次 LLM(忽略空白/大小写),判定一致并省调用
- 长文本保护:≥min_keep_len 时 noise 不构成删除依据
- 真实任务 run_ac7f480a3ccb 验证:非规则误删 350→193(-45%),呻吟/对话保留
- ocr-subtitle 工作流 v4:pool 钉死单线程(1/1)
- 回归夹具 testdata/ocr_srt_run_ac7f480a3ccb.srt(真实 1666 条 OCR 输出)
This commit is contained in:
2026-08-17 23:20:16 +08:00
parent 2b3a650612
commit 5ffa2ac39e
5 changed files with 7230 additions and 70 deletions
+318 -24
View File
@@ -1,14 +1,30 @@
"""LLM 字幕过滤节点测试。
覆盖 SRT 解析/序列化、±N 上下文窗口组装(纯文本无时间戳、目标标记)、
LLM 调用(按 I/O 边界 mock urlopen)与删除判定、invoke 全链路与异常路径。
覆盖
- SRT 解析/序列化(多行、末条无空行、重新编号);
- 确定性规则层(横线装饰、HTML/水印 token、URL、单双 ASCII 字符直接删,不调 LLM);
- LLM 五类分类判断(garbage/overlay/noise 删,repeat/dialogue 留,未识别回退保留);
- 文本去重(相同文本只调一次 LLM,上下文取首次出现,判定结果一致);
- 长文本保护(≥min_keep_len 时 noise 类不删,需明确垃圾类别);
- 真实任务 run_ac7f480a3ccb 留存数据回归(OCR 1666 条:垃圾删除、对话保留、时间轴单调、去重后 LLM 调用数);
- invoke 全链路与异常路径。
"""
import json
import urllib.error
from pathlib import Path
from nodes.llm_filter import invoke, parse_srt, serialize_srt
import pytest
from nodes.llm_filter import (
_dedup_key,
_judge_category,
_rule_verdict,
_should_delete,
invoke,
parse_srt,
serialize_srt,
)
from wov_sdk.models import InvokeRequest
# 4 条字幕的 SRT:第 3 条为"答:"开头的无意义杂项,模拟 OCR 噪声。
@@ -19,6 +35,10 @@ _SRT = (
"4\n00:00:13,000 --> 00:00:16,000\n第二句正常字幕\n"
)
# 真实任务 run_ac7f480a3ccb 的 OCR 输出(1666 条,2026-08 单线程 v4 跑完整 2 小时视频)。
WORKSPACE = Path(__file__).resolve().parent.parent
REAL_SRT = WORKSPACE / "testdata" / "ocr_srt_run_ac7f480a3ccb.srt"
class FakeResponse:
"""模拟 urllib 响应:read() 返回 LLM 兼容接口的 JSON 载荷。"""
@@ -37,9 +57,9 @@ class FakeResponse:
class FakeLLM:
"""模拟 LLM 兼容接口:记录请求体,按策略返回"保留/删除"
"""模拟 LLM 兼容接口:记录请求体,按策略返回类别词
支持两种策略:contents(按队列顺序,用于单次直调 _judge_target
支持两种策略:contents(按队列顺序,用于单次直调 _judge_category
确定性测试)或 decision_fn(按请求体内容决策,用于并发 invoke 测试,
保证任何线程执行顺序下判定结果都确定)。
"""
@@ -70,12 +90,17 @@ def _patch_llm(monkeypatch, contents: list[str] | None = None, decision_fn=None)
def _decision_by_target(body) -> str:
"""按目标字幕内容决策:含"答:"判为删除,其余保留(与 _SRT 的噪声对应)"""
"""按目标字幕内容决策:含"答:"判为 garbage,其余为 dialogue"""
target = next(
line for line in body["messages"][1]["content"].splitlines()
if line.startswith("【目标】")
)
return "删除" if "答:" in target else "保留"
return "garbage" if "答:" in target else "dialogue"
# ---------------------------------------------------------------------------
# SRT 解析 / 序列化
# ---------------------------------------------------------------------------
def test_parse_srt_multiline_and_last_block() -> None:
@@ -105,14 +130,49 @@ def test_serialize_srt_renumbers() -> None:
)
def test_judge_target_window_and_keep(monkeypatch) -> None:
"""窗口只含纯文本(无时间戳)、目标带标记;模型答"保留"则返回 False。"""
from nodes.llm_filter import _judge_target
# ---------------------------------------------------------------------------
# 确定性规则层
# ---------------------------------------------------------------------------
def test_rule_verdict_deletes_garbage_patterns() -> None:
"""规则层直接删除:横线装饰、HTML/水印 token、URL/邮箱、单双 ASCII 字符。"""
tokens = {"html", "background", "___"}
for text in (
"---", "------", "------------------", "--- ---", "___", "= = =",
"", "", "HTML", "html", "Background", "background", "___",
"https://example.com/x", "www.example.com", "a@b.com",
"A", "V", "1", "DQ", "P4",
):
assert _rule_verdict(text, tokens) is True, text
# 空文本/纯空白也删除。
assert _rule_verdict(" ", tokens) is True
def test_rule_verdict_passes_cjk_and_dialogue() -> None:
"""规则层不误伤:CJK 单字、正常对话、内容性短句交给 LLM 判断(None)。"""
tokens = {"html"}
for text in ("", "", "谢谢你 松井小姐", "不这么做的话 可没法胜任患者的对象", "好舒服"):
assert _rule_verdict(text, tokens) is None, text
def test_rule_verdict_custom_overlay_tokens() -> None:
"""overlay_tokens 参数化:自定义 token 同样直接删除。"""
assert _rule_verdict("Cleaning", {"cleaning"}) is True
assert _rule_verdict("Cleaning", set()) is None
# ---------------------------------------------------------------------------
# LLM 分类判断
# ---------------------------------------------------------------------------
def test_judge_category_window_and_dialogue(monkeypatch) -> None:
"""窗口只含纯文本(无时间戳)、目标带标记;模型答 dialogue 则保留。"""
entries = parse_srt(_SRT)
fake = _patch_llm(monkeypatch, ["保留"])
fake = _patch_llm(monkeypatch, ["dialogue"])
# context_size=1,目标为第 2 条(index=1):窗口 0..2 共 3 行,目标在中间。
assert _judge_target(entries, 1, context_size=1, params={}) is False
assert _judge_category(entries, 1, context_size=1, params={}) == "dialogue"
body = fake.bodies[0]
lines = body["messages"][1]["content"].splitlines()
assert len(lines) == 3
@@ -125,32 +185,71 @@ def test_judge_target_window_and_keep(monkeypatch) -> None:
assert body["max_tokens"] == 16
def test_judge_target_delete(monkeypatch) -> None:
"""模型答"删除"时返回 True(判定该条无意义)。"""
from nodes.llm_filter import _judge_target
def test_judge_category_delete_classes(monkeypatch) -> None:
"""模型答 garbage/overlay/noise 时返回对应类别(删除类)。"""
entries = parse_srt(_SRT)
for cat in ("garbage", "overlay", "noise"):
_patch_llm(monkeypatch, [cat])
assert _judge_category(entries, 2, context_size=10, params={}) == cat
def test_judge_category_repeat_and_unknown_kept(monkeypatch) -> None:
"""repeat/dialogue 返回保留类;未识别输出(空/乱码/旧式"保留")回退 dialogue。"""
entries = parse_srt(_SRT)
for answer in ("repeat", "dialogue", "", "???", "保留"):
_patch_llm(monkeypatch, [answer])
got = _judge_category(entries, 2, context_size=10, params={})
assert got in ("repeat", "dialogue"), (answer, got)
# 旧式"删除"回答兼容为垃圾类。
_patch_llm(monkeypatch, ["删除"])
assert _judge_target(entries, 2, context_size=10, params={}) is True
assert _judge_category(entries, 2, context_size=10, params={}) == "garbage"
def test_judge_target_model_and_auth(monkeypatch) -> None:
def test_judge_category_model_and_auth(monkeypatch) -> None:
"""模型名从参数取;配置 API Key 时附带 Bearer 鉴权头。"""
from nodes.llm_filter import _judge_target
entries = parse_srt(_SRT)
monkeypatch.setenv("LLM_API_KEY", "sk-test")
fake = _patch_llm(monkeypatch, ["保留"])
assert _judge_target(entries, 0, context_size=10, params={"model": "m/1"}) is False
fake = _patch_llm(monkeypatch, ["dialogue"])
assert _judge_category(entries, 0, context_size=10, params={"model": "m/1"}) == "dialogue"
assert fake.bodies[0]["model"] == "m/1"
assert fake.headers[0]["Authorization"] == "Bearer sk-test"
# ---------------------------------------------------------------------------
# 去重键与删除判定
# ---------------------------------------------------------------------------
def test_dedup_key_normalizes_whitespace_and_case() -> None:
"""去重键:去掉全部空白并统一大小写,空格差异视为同一文本。"""
assert _dedup_key("不这么做的话 可没法胜任患者的对象") == _dedup_key(
"不这么做的话可没法胜任患者的对象"
)
assert _dedup_key("HTML") == _dedup_key("html")
assert _dedup_key("你好 世界") != _dedup_key("你好世界2")
def test_should_delete_long_text_noise_protected() -> None:
"""长文本保护:≥min_keep_len 时 noise 不删,garbage/overlay 仍删;短文本三类都删。"""
long_text = "不这么做的话 可没法胜任患者的对象"
assert _should_delete("noise", long_text, min_keep_len=12) is False
assert _should_delete("garbage", long_text, min_keep_len=12) is True
assert _should_delete("overlay", long_text, min_keep_len=12) is True
assert _should_delete("noise", "答:无意义杂项", min_keep_len=12) is True
assert _should_delete("repeat", long_text, min_keep_len=12) is False
assert _should_delete("dialogue", long_text, min_keep_len=12) is False
# ---------------------------------------------------------------------------
# invoke 全链路
# ---------------------------------------------------------------------------
def test_invoke_filters_and_renumbers(monkeypatch, tmp_path) -> None:
"""全链路(并发):按 LLM 判定删除无意义条,保留条重新编号输出。"""
"""全链路(并发):按 LLM 类别判定删除无意义条,保留条重新编号输出。"""
srt = tmp_path / "in.srt"
srt.write_text(_SRT, encoding="utf-8")
# 内容决策:目标字幕含"答:"判删除,其余保留(任何线程顺序下结果确定)。
# 内容决策:目标字幕含"答:"判 garbage,其余 dialogue(任何线程顺序下结果确定)。
_patch_llm(monkeypatch, decision_fn=_decision_by_target)
response = invoke(
InvokeRequest(
@@ -172,6 +271,79 @@ def test_invoke_filters_and_renumbers(monkeypatch, tmp_path) -> None:
assert "00:00:13,000 --> 00:00:16,000\n第二句正常字幕\n" in out
def test_invoke_rules_skip_llm(monkeypatch, tmp_path) -> None:
"""规则层命中时直接删除且不调用 LLM:横线/HTML/单字符全部清理。"""
srt = tmp_path / "in.srt"
srt.write_text(
"1\n00:00:01,000 --> 00:00:04,000\n---\n\n"
"2\n00:00:05,000 --> 00:00:08,000\nHTML\n\n"
"3\n00:00:09,000 --> 00:00:12,000\nV\n\n"
"4\n00:00:13,000 --> 00:00:16,000\n正常对话\n",
encoding="utf-8",
)
fake = _patch_llm(monkeypatch, ["dialogue"])
response = invoke(
InvokeRequest(
run_id="r", node_instance_id="",
inputs={"srt_uri": str(srt)},
params={},
output_dir=str(tmp_path / "out"),
)
)
assert response.status == "completed", response.error
assert response.outputs["kept"] == 1
assert response.outputs["removed"] == 3
assert len(fake.bodies) == 1 # 仅"正常对话"需 LLM,其余全走规则。
def test_invoke_dedup_single_llm_call(monkeypatch, tmp_path) -> None:
"""去重:相同文本(含空格变体)只调一次 LLM,判定结果一致。"""
srt = tmp_path / "in.srt"
srt.write_text(
"1\n00:00:01,000 --> 00:00:04,000\n好舒服\n\n"
"2\n00:00:05,000 --> 00:00:08,000\n好 舒 服\n\n"
"3\n00:00:09,000 --> 00:00:12,000\n好舒服\n\n"
"4\n00:00:13,000 --> 00:00:16,000\n别的台词\n",
encoding="utf-8",
)
fake = _patch_llm(monkeypatch, decision_fn=lambda body: "repeat")
response = invoke(
InvokeRequest(
run_id="r", node_instance_id="",
inputs={"srt_uri": str(srt)},
params={},
output_dir=str(tmp_path / "out"),
)
)
assert response.status == "completed", response.error
# 2 个唯一文本(好舒服/别的台词)→ 恰好 2 次 LLM 调用。
assert len(fake.bodies) == 2
# repeat 类保留 → 4 条全部保留。
assert response.outputs["kept"] == 4
assert response.outputs["removed"] == 0
def test_invoke_dedupe_disabled(monkeypatch, tmp_path) -> None:
"""dedupe=0 关闭去重:每个条目都调一次 LLM。"""
srt = tmp_path / "in.srt"
srt.write_text(
"1\n00:00:01,000 --> 00:00:04,000\n同一句\n\n"
"2\n00:00:05,000 --> 00:00:08,000\n同一句\n",
encoding="utf-8",
)
fake = _patch_llm(monkeypatch, decision_fn=lambda body: "dialogue")
response = invoke(
InvokeRequest(
run_id="r", node_instance_id="",
inputs={"srt_uri": str(srt)},
params={"dedupe": "0"},
output_dir=str(tmp_path / "out"),
)
)
assert response.status == "completed", response.error
assert len(fake.bodies) == 2
def test_invoke_context_size_param(monkeypatch, tmp_path) -> None:
"""context_size 参数生效:窗口大小=2×context_size+1(两端截断除外)。"""
srt = tmp_path / "in.srt"
@@ -195,6 +367,52 @@ def test_invoke_context_size_param(monkeypatch, tmp_path) -> None:
assert len(window) == 3
def test_invoke_overlay_tokens_param(monkeypatch, tmp_path) -> None:
"""overlay_tokens 参数:自定义 token 走规则层删除,不调用 LLM。"""
srt = tmp_path / "in.srt"
srt.write_text(
"1\n00:00:01,000 --> 00:00:04,000\nCleaning\n\n"
"2\n00:00:05,000 --> 00:00:08,000\n正常对话\n",
encoding="utf-8",
)
fake = _patch_llm(monkeypatch, decision_fn=lambda body: "dialogue")
response = invoke(
InvokeRequest(
run_id="r", node_instance_id="",
inputs={"srt_uri": str(srt)},
params={"overlay_tokens": ["cleaning"]},
output_dir=str(tmp_path / "out"),
)
)
assert response.status == "completed", response.error
assert response.outputs["removed"] == 1
assert response.outputs["kept"] == 1
assert len(fake.bodies) == 1 # 只判正常对话。
def test_invoke_overlay_tokens_json_string(monkeypatch, tmp_path) -> None:
"""overlay_tokens 以 JSON 字符串形式传入(工作流参数常见形态)同样生效。"""
srt = tmp_path / "in.srt"
srt.write_text(
"1\n00:00:01,000 --> 00:00:04,000\nXXLogo\n\n"
"2\n00:00:05,000 --> 00:00:08,000\n正常对话\n",
encoding="utf-8",
)
fake = _patch_llm(monkeypatch, ["dialogue"])
response = invoke(
InvokeRequest(
run_id="r", node_instance_id="",
inputs={"srt_uri": str(srt)},
params={"overlay_tokens": '["xxlogo", "xx"]'},
output_dir=str(tmp_path / "out"),
)
)
assert response.status == "completed", response.error
assert response.outputs["removed"] == 1
assert response.outputs["kept"] == 1
assert len(fake.bodies) == 1
def test_invoke_missing_input(tmp_path) -> None:
"""缺少 srt_uri 时返回失败。"""
response = invoke(
@@ -255,3 +473,79 @@ def test_invoke_empty_srt(monkeypatch, tmp_path) -> None:
assert response.outputs["kept"] == 0
assert response.outputs["removed"] == 0
assert fake.bodies == []
# ---------------------------------------------------------------------------
# 真实任务留存数据回归(testdata/ocr_srt_run_ac7f480a3ccb.srt
# ---------------------------------------------------------------------------
def test_real_run_rules_and_dialogue_regression(monkeypatch, tmp_path) -> None:
"""真实数据回归:规则层清理垃圾、对话保留、时间轴单调、去重生效。
夹具为 run_ac7f480a3ccb(单线程 v4 跑完整 2 小时视频)的 OCR 输出 1666 条。
假 LLM 对非规则条目一律答 dialogue:验证规则层删掉全部垃圾(横线/水印/
单字符),真实对话全部保留,且相同文本只调一次 LLM(去重)。
"""
srt = REAL_SRT
if not srt.is_file():
pytest.skip("缺少 testdata/ocr_srt_run_ac7f480a3ccb.srt,跳过回归测试")
fake = _patch_llm(monkeypatch, decision_fn=lambda body: "dialogue")
response = invoke(
InvokeRequest(
run_id="r", node_instance_id="",
inputs={"srt_uri": str(srt)},
params={},
output_dir=str(tmp_path / "out"),
)
)
assert response.status == "completed", response.error
entries = parse_srt(srt.read_text(encoding="utf-8"))
out = Path(response.outputs["srt_uri"]).read_text(encoding="utf-8")
# 规则层删除量 = 夹具中直接命中规则文本的条数(真实数据动态计算)。
# 规则层删除量 = 夹具中直接命中规则文本的条数(与 invoke 默认 token 集一致)。
from nodes.llm_filter import DEFAULT_OVERLAY_TOKENS
tokens = set(DEFAULT_OVERLAY_TOKENS)
rule_removed = sum(
1 for e in entries if _rule_verdict(e["text"], tokens) is True
)
assert response.outputs["removed"] == rule_removed
assert response.outputs["kept"] == len(entries) - rule_removed
# 垃圾全部清出输出(横线、HTML、单字符都不再出现;只检查文本字段,
# 排除 SRT 的序号行与时间轴行)。
for entry in parse_srt(out):
assert _rule_verdict(entry["text"], tokens) is not True, entry["text"]
# 真实对话保留(修复前被误删的自我介绍、请求对话、呻吟内容行)。
# 注意:片头标题卡(淫魔病院…)真实 LLM 判为 overlay(烧录标题),
# 不在必保留列表内;本测试假 LLM 一律答 dialogue,仅验证机制正确。
for kept_line in (
"谢谢你 松井小姐",
"我姓泷本 请多多指教",
"不这么做的话 可没法胜任患者的对象",
"你要看着北冈小姐的脸 你们俩相互看看嘛",
"好舒服",
):
assert kept_line in out, kept_line
# 时间轴单调递增(输出顺序即时间顺序)。
times = []
for m in __import__("re").finditer(
r"(\d{2}:\d{2}:\d{2},\d{3})\s*-->", out
):
t = m.group(1).replace(",", ".")
h, mi, s = t.split(":")
times.append(int(h) * 3600 + int(mi) * 60 + float(s))
assert all(a < b for a, b in zip(times, times[1:]))
# 去重:LLM 调用数 == 非规则条目中的唯一文本数(远小于条目数)。
unique_llm = len({
_dedup_key(e["text"])
for e in entries
if _rule_verdict(e["text"], tokens) is None
})
assert len(fake.bodies) == unique_llm
assert len(fake.bodies) < len(entries) # 去重确实省调用。