feat: llm-filter 节点级断点存档
- 每条 LLM 判定成功立即追加 filter_partial.jsonl(多线程加锁串行化) - 失败/中断后重跑只重判未判定条目,已判定结果复用,与 OCR 存档同机制 - 附带上下文净化回归重跑脚本(run_011d01f19999)
This commit is contained in:
+48
-6
@@ -33,6 +33,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
@@ -71,6 +72,39 @@ _ALL_CATEGORIES = (
|
||||
)
|
||||
DELETE_CATEGORIES = {CATEGORY_GARBAGE, CATEGORY_OVERLAY, CATEGORY_NOISE}
|
||||
|
||||
# 判定存档文件名:位于节点 output_dir,每行 {"index": 条目标引, "category": 类别}。
|
||||
# 每条 LLM 判定成功即追加一行;进程被杀/节点失败(如 429 限流)后重跑时,
|
||||
# 只对未判定的条目重新调用 LLM,已判定结果直接复用(类似 OCR 的断点存档)。
|
||||
_PARTIAL_NAME = "filter_partial.jsonl"
|
||||
|
||||
# 判定存档追加写锁:多线程判定并发完成时串行化追加,避免行交错。
|
||||
_partial_lock = threading.Lock()
|
||||
|
||||
|
||||
def _load_partial(output_dir: Path) -> dict[int, str]:
|
||||
"""读取判定存档,返回 {条目标引: 类别};无存档/损坏行跳过。"""
|
||||
path = output_dir / _PARTIAL_NAME
|
||||
if not path.is_file():
|
||||
return {}
|
||||
result: dict[int, str] = {}
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
# 进程被杀时可能残留半行写入:跳过,对应条目视为未判定。
|
||||
continue
|
||||
result[int(item["index"])] = str(item["category"])
|
||||
return result
|
||||
|
||||
|
||||
def _append_partial(output_dir: Path, index: int, category: str) -> None:
|
||||
"""线程安全地把一条判定结果追加到存档(成功判定后立即落盘)。"""
|
||||
with _partial_lock:
|
||||
with (output_dir / _PARTIAL_NAME).open("a", encoding="utf-8") as fh:
|
||||
fh.write(json.dumps({"index": index, "category": category}, ensure_ascii=False) + "\n")
|
||||
|
||||
# 规则层正则:横线装饰(含全角/半角横线、下划线、中点、句点等符号组合)。
|
||||
_DASH_RE = re.compile(r"^[\s\-—_~=•・。..、]+$")
|
||||
# 规则层正则:URL / 邮箱。
|
||||
@@ -295,7 +329,11 @@ def invoke(request: InvokeRequest) -> InvokeResponse:
|
||||
llm_needed = [i for i, verdict in enumerate(rule_verdicts) if verdict is None]
|
||||
|
||||
# 阶段 2:LLM 分类层(去重:相同文本只判一次,上下文取首次出现)。
|
||||
cat_by_index: dict[int, str] = {}
|
||||
# 节点级断点存档:output_dir 提前建好,重跑时只重判未判定条目。
|
||||
output_dir = Path(request.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
partial = _load_partial(output_dir)
|
||||
cat_by_index: dict[int, str] = dict(partial)
|
||||
if llm_needed:
|
||||
if dedupe:
|
||||
first_of_key: dict[str, int] = {}
|
||||
@@ -304,20 +342,24 @@ def invoke(request: InvokeRequest) -> InvokeResponse:
|
||||
key = _dedup_key(entries[i]["text"])
|
||||
if key not in first_of_key:
|
||||
first_of_key[key] = i
|
||||
pool_indices.append(i)
|
||||
# 断点续跑:该键首次出现已在存档判定过则跳过(结果复用)。
|
||||
if i not in partial:
|
||||
pool_indices.append(i)
|
||||
else:
|
||||
pool_indices = llm_needed
|
||||
|
||||
# 断点续跑:只处理未判定的条目。
|
||||
pool_indices = [i for i in llm_needed if i not in partial]
|
||||
# 单条判断的工作函数:返回类别词;overlay_tokens 用于上下文净化。
|
||||
def judge_one(index: int) -> str:
|
||||
try:
|
||||
return _judge_category(entries, index, context_size, request.params, overlay_tokens)
|
||||
category = _judge_category(entries, index, context_size, request.params, overlay_tokens)
|
||||
except urllib.error.HTTPError as exc:
|
||||
# 限流/服务端错误:通知线程池临时降低最大并发,避免持续超配额。
|
||||
if exc.code == 429 or 500 <= exc.code < 600:
|
||||
pool.report_failure()
|
||||
raise
|
||||
|
||||
# 判定成功立即落盘(断点存档):失败/中断后重跑不重复调用已判定条目。
|
||||
_append_partial(output_dir, index, category)
|
||||
return category
|
||||
# 进度日志:打印已判定条数、总数、平均处理速度(条/s)、最近窗口
|
||||
# 平均单条耗时与当前线程数(与 OCR 节点同一回调协议)。
|
||||
def log_progress(done: int, total: int, rate: float, avg_time: float, workers: int) -> None:
|
||||
|
||||
Reference in New Issue
Block a user