feat: llm-filter 两级过滤(规则层+五类分类+去重+长文本保护),工作流单线程 v4
- 规则层(不调 LLM):横线装饰/HTML 水印 token/URL/邮箱/单双 ASCII 字符直接删 - LLM 五类分类:garbage/overlay/noise 删,repeat/dialogue 留,未识别回退保留 - 按文本去重:相同文本只调一次 LLM(忽略空白/大小写),判定一致并省调用 - 长文本保护:≥min_keep_len 时 noise 不构成删除依据 - 真实任务 run_ac7f480a3ccb 验证:非规则误删 350→193(-45%),呻吟/对话保留 - ocr-subtitle 工作流 v4:pool 钉死单线程(1/1) - 回归夹具 testdata/ocr_srt_run_ac7f480a3ccb.srt(真实 1666 条 OCR 输出)
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""用修复后的 llm_filter 对 run_ac7f480a3ccb 的真实 OCR 输出重新过滤。
|
||||
|
||||
- 加载 .env(LLM Key/模型配置)
|
||||
- 复用节点 invoke 真实代码路径(与调度器执行完全一致)
|
||||
- 池参数按该 run 的 workflow v4 配置(单线程 pool 1/1)
|
||||
- 产物写入 run 的 filter 步骤目录(filtered.srt)
|
||||
"""
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv(".env")
|
||||
|
||||
from wov_sdk.models import InvokeRequest
|
||||
from nodes.llm_filter import invoke
|
||||
|
||||
RUN = Path("data/storage/runs/run_ac7f480a3ccb")
|
||||
SRT_URI = RUN / "steps/ocr/subtitle.srt"
|
||||
OUT_DIR = RUN / "steps/filter"
|
||||
|
||||
print(f"输入: {SRT_URI}", flush=True)
|
||||
print(f"输出目录: {OUT_DIR}", flush=True)
|
||||
|
||||
started = time.monotonic()
|
||||
response = invoke(
|
||||
InvokeRequest(
|
||||
run_id="run_ac7f480a3ccb",
|
||||
node_instance_id="",
|
||||
inputs={"srt_uri": str(SRT_URI)},
|
||||
# 与 workflow v4(单线程)保持一致。
|
||||
params={"pool_min_workers": 1, "pool_max_workers": 1},
|
||||
output_dir=str(OUT_DIR),
|
||||
)
|
||||
)
|
||||
elapsed = time.monotonic() - started
|
||||
print(f"耗时 {elapsed:.0f}s", flush=True)
|
||||
if response.status != "completed":
|
||||
print(f"FAILED: {response.error}", flush=True)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"kept={response.outputs['kept']} removed={response.outputs['removed']}", flush=True)
|
||||
print(f"产物: {response.outputs['srt_uri']}", flush=True)
|
||||
Reference in New Issue
Block a user