feat: llm-filter 两级过滤(规则层+五类分类+去重+长文本保护),工作流单线程 v4

- 规则层(不调 LLM):横线装饰/HTML 水印 token/URL/邮箱/单双 ASCII 字符直接删
- LLM 五类分类:garbage/overlay/noise 删,repeat/dialogue 留,未识别回退保留
- 按文本去重:相同文本只调一次 LLM(忽略空白/大小写),判定一致并省调用
- 长文本保护:≥min_keep_len 时 noise 不构成删除依据
- 真实任务 run_ac7f480a3ccb 验证:非规则误删 350→193(-45%),呻吟/对话保留
- ocr-subtitle 工作流 v4:pool 钉死单线程(1/1)
- 回归夹具 testdata/ocr_srt_run_ac7f480a3ccb.srt(真实 1666 条 OCR 输出)
This commit is contained in:
2026-08-17 23:20:16 +08:00
parent 2b3a650612
commit 5ffa2ac39e
5 changed files with 7230 additions and 70 deletions
+45
View File
@@ -0,0 +1,45 @@
# -*- coding: utf-8 -*-
"""用修复后的 llm_filter 对 run_ac7f480a3ccb 的真实 OCR 输出重新过滤。
- 加载 .envLLM Key/模型配置)
- 复用节点 invoke 真实代码路径(与调度器执行完全一致)
- 池参数按该 run 的 workflow v4 配置(单线程 pool 1/1
- 产物写入 run 的 filter 步骤目录(filtered.srt
"""
import sys
import time
from pathlib import Path
from dotenv import load_dotenv
load_dotenv(".env")
from wov_sdk.models import InvokeRequest
from nodes.llm_filter import invoke
RUN = Path("data/storage/runs/run_ac7f480a3ccb")
SRT_URI = RUN / "steps/ocr/subtitle.srt"
OUT_DIR = RUN / "steps/filter"
print(f"输入: {SRT_URI}", flush=True)
print(f"输出目录: {OUT_DIR}", flush=True)
started = time.monotonic()
response = invoke(
InvokeRequest(
run_id="run_ac7f480a3ccb",
node_instance_id="",
inputs={"srt_uri": str(SRT_URI)},
# 与 workflow v4(单线程)保持一致。
params={"pool_min_workers": 1, "pool_max_workers": 1},
output_dir=str(OUT_DIR),
)
)
elapsed = time.monotonic() - started
print(f"耗时 {elapsed:.0f}s", flush=True)
if response.status != "completed":
print(f"FAILED: {response.error}", flush=True)
sys.exit(1)
print(f"kept={response.outputs['kept']} removed={response.outputs['removed']}", flush=True)
print(f"产物: {response.outputs['srt_uri']}", flush=True)