feat: 任务断点续跑/暂停中断/LLM 过滤优化与调度容错
调度与状态机： - 修复 PAUSED 任务被拾起后复活执行（点击暂停反而开始任务）：next_queued_run 只取 QUEUED，execute_run 以 PAUSED 进入直接返回，暂停必须显式 resume - 重启恢复：启动时 recover_interrupted_runs 把遗留 RUNNING 置 QUEUED（保留产物） - 暂停信号 paused.flag：暂停接口写、继续/重试清除，OCR 逐帧检查秒级中断， 节点内被暂停保持 PAUSED 不误报 FAILED - 调度轮询容错：_loop 异常不杀死线程（曾致任务永久停留 QUEUED） subtitle-ocr 节点级断点： - ocr_partial.jsonl 逐帧存档，重启/暂停后只处理未处理帧，产物与一次跑完一致 - 进度日志携带窗口平均耗时与线程数；取消后抑制进度日志井喷 llm-filter 过滤质量与限流自适应： - 上下文净化：喂给 LLM 的是过滤后的字幕（规则层垃圾从上下文剔除） - 正则确定性过滤：裸网址域名、HTML/水印模式直接删除 - 429/5xx 指数退避重试；worker 限流错误 report_failure 内存临时降最大线程数 并缩容（无错误窗口回升），失败条目降并发后重试一轮 - 保留长文本保护（noise 不删 ≥min_keep_len 文本，LLM 判定不稳的必要兜底） 前端： - 工作流编排页支持选择工作流编辑（加载最新/历史版本）、版本历史面板、 新建/编辑双模式；管理后台编辑跳转 workflow.html?edit=<id> 工作流：ocr-subtitle v7（filter pool_max_workers=20、pool_fast_threshold=1）
This commit is contained in:
+130
-3
@@ -51,18 +51,96 @@ def test_pool_map_ordered_results() -> None:
|
||||
|
||||
|
||||
def test_pool_on_progress_callback() -> None:
|
||||
"""进度回调：每次完成触发一次，携带已完成数/总数/速度。"""
|
||||
progress: list[tuple[int, int, float]] = []
|
||||
"""进度回调：每次完成触发一次，携带已完成数/总数/速度/平均耗时/线程数。"""
|
||||
progress: list[tuple[int, int, float, float, int]] = []
|
||||
pool = AdaptiveThreadPool(
|
||||
worker=lambda item: item,
|
||||
on_progress=lambda done, total, rate: progress.append((done, total, rate)),
|
||||
on_progress=lambda done, total, rate, avg_time, workers: progress.append(
|
||||
(done, total, rate, avg_time, workers)
|
||||
),
|
||||
)
|
||||
pool.map([10, 20, 30])
|
||||
assert [item[0] for item in progress] == [1, 2, 3] # 已完成数递增。
|
||||
assert all(item[1] == 3 for item in progress) # 总数固定。
|
||||
assert all(item[2] > 0 for item in progress) # 速度为正值。
|
||||
# 窗口未满时平均耗时回退为累计平均（>0）；线程数 ∈ [1, 上限]。
|
||||
assert all(item[3] > 0 for item in progress)
|
||||
assert all(1 <= item[4] <= pool.max_workers for item in progress)
|
||||
|
||||
|
||||
def test_pool_progress_reports_window_avg_after_first_window() -> None:
|
||||
"""窗口评估后：回调携带最近窗口平均耗时（扩缩容依据）与扩容后的线程数。
|
||||
|
||||
覆盖 `_current_avg_time` 两个分支：窗口评估前回退累计平均，评估后使用
|
||||
最近窗口平均（0.0s，响应远快于 fast_threshold 0.3s → 线程 +1）。
|
||||
"""
|
||||
clock = FakeClock()
|
||||
seen: list[tuple[int, int, float, float, int]] = []
|
||||
pool = AdaptiveThreadPool(
|
||||
worker=lambda item: item,
|
||||
min_workers=1, max_workers=16,
|
||||
window_seconds=10.0, fast_threshold=0.3,
|
||||
clock=clock,
|
||||
on_progress=lambda done, total, rate, avg_time, workers: seen.append(
|
||||
(done, total, rate, avg_time, workers)
|
||||
),
|
||||
)
|
||||
clock.advance(11) # 首个任务完成即越过窗口 → 触发评估。
|
||||
pool.map(list(range(5)))
|
||||
# 第一个完成的任务回调在窗口评估前：回退累计平均（>0）。
|
||||
assert seen[0][3] > 0
|
||||
# 窗口评估后：回调携带最近窗口平均（≈0.0），且线程已扩容到 2。
|
||||
assert any(item[3] == 0.0 for item in seen)
|
||||
assert any(item[4] == 2 for item in seen)
|
||||
assert pool.max_concurrency == 2
|
||||
def test_pool_cancel_suppresses_progress() -> None:
|
||||
"""worker 触发 cancel（如检测到暂停信号）后：剩余任务不再触发进度回调。
|
||||
|
||||
暂停场景：队列中剩余的大量帧会逐帧快速失败退出，若每完成一项都打印
|
||||
进度日志，会在数秒内打出上万行日志；cancel 后抑制后续进度回调。
|
||||
"""
|
||||
progress: list[tuple[int, int, float, float, int]] = []
|
||||
|
||||
def worker(item):
|
||||
if item == 1:
|
||||
pool.cancel() # 模拟某帧检测到暂停信号。
|
||||
return item
|
||||
|
||||
pool = AdaptiveThreadPool(
|
||||
worker=worker,
|
||||
on_progress=lambda done, total, rate, avg_time, workers: progress.append(
|
||||
(done, total, rate, avg_time, workers)
|
||||
),
|
||||
)
|
||||
pool.map([0, 1, 2, 3])
|
||||
# 只有 cancel 之前的任务（item=0）触发了进度回调。
|
||||
assert len(progress) == 1
|
||||
assert progress[0][1] == 4 # 总数仍是 4。
|
||||
|
||||
|
||||
def test_pool_cancel_resets_between_maps() -> None:
|
||||
"""取消状态按批（map）重置：下一批任务进度回调恢复正常。"""
|
||||
progress: list[tuple[int, int, float, float, int]] = []
|
||||
|
||||
def worker(item):
|
||||
if item == "stop":
|
||||
pool.cancel()
|
||||
return item
|
||||
|
||||
pool = AdaptiveThreadPool(
|
||||
worker=worker,
|
||||
on_progress=lambda done, total, rate, avg_time, workers: progress.append(
|
||||
(done, total, rate, avg_time, workers)
|
||||
),
|
||||
)
|
||||
pool.map(["a", "stop", "b"])
|
||||
assert len(progress) == 1 # 只有 cancel 前的 a 触发回调。
|
||||
pool.map(["c", "d"])
|
||||
# 新一批恢复回调（done 从 1 重新计数）：共 3 次回调（1 + 2）。
|
||||
assert [item[0] for item in progress] == [1, 1, 2]
|
||||
|
||||
def test_pool_map_empty() -> None:
|
||||
|
||||
"""空输入：不启动任务，直接返回空列表。"""
|
||||
pool = AdaptiveThreadPool(worker=lambda item: item)
|
||||
assert pool.map([]) == []
|
||||
@@ -146,3 +224,52 @@ def test_pool_survives_mixed_grow_shrink() -> None:
|
||||
)
|
||||
out = pool.map(list(range(60)))
|
||||
assert out == list(range(60))
|
||||
|
||||
|
||||
def test_pool_report_failure_lowers_effective_max() -> None:
|
||||
"""消费错误（如 API 限流）临时降低有效最大线程数，下限为 min_workers。
|
||||
|
||||
自适应：并发打到配额线触发 429 时，report_failure 收紧有效上限，
|
||||
后续请求减少从而避开持续限流。
|
||||
"""
|
||||
pool = AdaptiveThreadPool(worker=lambda item: item, min_workers=1, max_workers=16)
|
||||
assert pool._effective_max_workers == 16
|
||||
pool.report_failure()
|
||||
assert pool._effective_max_workers == 15
|
||||
for _ in range(30):
|
||||
pool.report_failure()
|
||||
assert pool._effective_max_workers == 1 # 下限 min_workers。
|
||||
|
||||
|
||||
def test_pool_effective_max_recovers_after_clean_window() -> None:
|
||||
"""连续无错误窗口后有效上限逐步回升到 max_workers。"""
|
||||
clock = FakeClock()
|
||||
pool = AdaptiveThreadPool(
|
||||
worker=lambda item: item, min_workers=1, max_workers=16,
|
||||
window_seconds=10.0, fast_threshold=0.3, clock=clock,
|
||||
)
|
||||
pool.report_failure() # 有效上限 16 -> 15。
|
||||
clock.advance(11)
|
||||
pool._tick(0.01) # 错误所在窗口：上限不恢复。
|
||||
assert pool._effective_max_workers == 15
|
||||
clock.advance(11)
|
||||
pool._tick(0.01) # 下一个干净窗口：恢复 +1。
|
||||
assert pool._effective_max_workers == 16
|
||||
pool._tick(0.01) # 窗口未满早退，上限不变。
|
||||
assert pool._effective_max_workers == 16
|
||||
|
||||
|
||||
def test_pool_decide_uses_effective_max() -> None:
|
||||
"""扩容上限按有效最大线程数：错误窗口内即使响应快也不超过收紧后的上限。"""
|
||||
clock = FakeClock()
|
||||
pool = AdaptiveThreadPool(
|
||||
worker=lambda item: item, min_workers=1, max_workers=16,
|
||||
window_seconds=10.0, fast_threshold=0.3, clock=clock,
|
||||
)
|
||||
pool.report_failure() # 有效上限 16 -> 15。
|
||||
pool._resize(15)
|
||||
pool._window_failures = 1 # 本窗口内仍有错误 → 不恢复上限。
|
||||
clock.advance(11)
|
||||
pool._tick(0.01) # 响应快，但 15 已是有效上限 → 不扩。
|
||||
assert pool._target_workers == 15
|
||||
assert pool._effective_max_workers == 15
|
||||
|
||||
@@ -196,15 +196,22 @@ def test_pause_resume_run_api() -> None:
|
||||
run_id = uploaded.json()["id"]
|
||||
assert uploaded.json()["status"] == "QUEUED"
|
||||
|
||||
from wov_app.config import STORAGE_DIR
|
||||
flag = STORAGE_DIR / "runs" / run_id / "paused.flag"
|
||||
|
||||
paused = client.post(f"/api/runs/{run_id}/pause")
|
||||
assert paused.status_code == 200
|
||||
assert paused.json() == {"id": run_id, "status": "PAUSED"}
|
||||
assert client.get(f"/api/runs/{run_id}").json()["status"] == "PAUSED"
|
||||
# 暂停时写入暂停信号文件，供运行中的节点（如 OCR）逐帧检查并中止。
|
||||
assert flag.exists()
|
||||
|
||||
resumed = client.post(f"/api/runs/{run_id}/resume")
|
||||
assert resumed.status_code == 200
|
||||
assert resumed.json() == {"id": run_id, "status": "QUEUED"}
|
||||
assert client.get(f"/api/runs/{run_id}").json()["status"] == "QUEUED"
|
||||
# 继续时清除暂停信号，避免误触发节点内暂停。
|
||||
assert not flag.exists()
|
||||
|
||||
# 非 PAUSED 任务不可继续。
|
||||
assert client.post(f"/api/runs/{run_id}/resume").status_code == 422
|
||||
|
||||
+40
-3
@@ -238,7 +238,12 @@ def test_db_migration_adds_param_overrides(tmp_path) -> None:
|
||||
|
||||
|
||||
def test_pause_resume_run(tmp_path) -> None:
|
||||
"""验证 pause_run/resume_run 的状态流转与 PAUSED 任务可被调度器取到。"""
|
||||
"""验证 pause_run/resume_run 的状态流转与 PAUSED 任务不被调度器自动拾起。
|
||||
|
||||
修复回归：PAUSED 任务若被 next_queued_run 取到，execute_run 会把它复活为
|
||||
RUNNING 继续执行——"点击暂停反而开始任务"。暂停必须由用户显式 resume
|
||||
（PAUSED → QUEUED）后调度器才重新执行。
|
||||
"""
|
||||
db = Database(tmp_path / "wov.db")
|
||||
db.upsert_workflow({"id": "demo", "name": "Demo", "published": 1, "latest_version": 1})
|
||||
now = "2026-01-01T00:00:00+00:00"
|
||||
@@ -255,13 +260,45 @@ def test_pause_resume_run(tmp_path) -> None:
|
||||
)
|
||||
db.pause_run("run_p", now)
|
||||
assert db.get_run("run_p")["status"] == "PAUSED"
|
||||
# PAUSED 任务会被 next_queued_run 取到（等待续跑）。
|
||||
assert db.next_queued_run()["id"] == "run_p"
|
||||
# 已暂停的任务不会被调度器拾起（等待用户显式 resume）。
|
||||
assert db.next_queued_run() is None
|
||||
db.resume_run("run_p", now)
|
||||
assert db.get_run("run_p")["status"] == "QUEUED"
|
||||
assert db.next_queued_run()["id"] == "run_p"
|
||||
|
||||
|
||||
def test_recover_interrupted_runs(tmp_path) -> None:
|
||||
"""重启恢复：遗留 RUNNING 任务恢复为 QUEUED（保留产物供断点续跑）。
|
||||
|
||||
进程被杀/重启时 RUNNING 任务不会自动收尾，若保持 RUNNING 将永久孤儿
|
||||
（next_queued_run 不拾起、暂停后又被 execute_run 复活）。恢复为 QUEUED
|
||||
后调度器会从产物表断点续跑；用户主动暂停的 PAUSED 任务保持不变。
|
||||
"""
|
||||
db = Database(tmp_path / "wov.db")
|
||||
db.upsert_workflow({"id": "demo", "name": "Demo", "published": 1, "latest_version": 1})
|
||||
now = "2026-01-01T00:00:00+00:00"
|
||||
for run_id, status in (("run_orphan", "RUNNING"), ("run_paused", "PAUSED"),
|
||||
("run_done", "COMPLETED")):
|
||||
db.create_run(
|
||||
{
|
||||
"id": run_id,
|
||||
"workflow_id": "demo",
|
||||
"workflow_version": 1,
|
||||
"status": status,
|
||||
"progress": 0.5,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
recovered = db.recover_interrupted_runs("2026-01-02T00:00:00+00:00")
|
||||
assert recovered == 1 # 只有 RUNNING 被恢复。
|
||||
assert db.get_run("run_orphan")["status"] == "QUEUED"
|
||||
assert db.get_run("run_orphan")["updated_at"] == "2026-01-02T00:00:00+00:00"
|
||||
assert db.get_run("run_paused")["status"] == "PAUSED"
|
||||
assert db.get_run("run_done")["status"] == "COMPLETED"
|
||||
# 恢复后调度器可拾起并断点续跑。
|
||||
assert db.next_queued_run()["id"] == "run_orphan"
|
||||
|
||||
def test_restore_run_outputs(tmp_path) -> None:
|
||||
"""验证从产物重建节点输出（断点续跑的依据）。"""
|
||||
db = Database(tmp_path / "wov.db")
|
||||
|
||||
+161
-1
@@ -17,6 +17,7 @@ from pathlib import Path
|
||||
import pytest
|
||||
|
||||
from nodes.llm_filter import (
|
||||
DEFAULT_OVERLAY_TOKENS,
|
||||
_dedup_key,
|
||||
_judge_category,
|
||||
_rule_verdict,
|
||||
@@ -81,7 +82,31 @@ class FakeLLM:
|
||||
payload = json.dumps({"choices": [{"message": {"content": content}}]}).encode()
|
||||
return FakeResponse(payload)
|
||||
|
||||
class FlakyLLM:
|
||||
"""模拟限流/服务端错误：前 failures 次抛 HTTPError(429/503)，之后正常返回。
|
||||
|
||||
用于验证 _judge_category 的退避重试：429/5xx 时让出时间重试，配合
|
||||
自适应线程池"慢响应减线程"弹性，让并发自动回落到限流配额内。
|
||||
"""
|
||||
|
||||
def __init__(self, failures: int, code: int = 429, answer: str = "dialogue") -> None:
|
||||
self.failures = failures
|
||||
self.code = code
|
||||
self.answer = answer
|
||||
self.calls = 0
|
||||
self.bodies: list[dict] = []
|
||||
|
||||
def __call__(self, request, timeout=None):
|
||||
self.calls += 1
|
||||
self.bodies.append(json.loads(request.data.decode("utf-8")))
|
||||
if self.calls <= self.failures:
|
||||
raise urllib.error.HTTPError(
|
||||
request.full_url, self.code, "flaky", {}, None
|
||||
)
|
||||
payload = json.dumps(
|
||||
{"choices": [{"message": {"content": self.answer}}]}
|
||||
).encode()
|
||||
return FakeResponse(payload)
|
||||
def _patch_llm(monkeypatch, contents: list[str] | None = None, decision_fn=None) -> FakeLLM:
|
||||
"""替换 nodes.llm_filter 的 urlopen 为 FakeLLM 并返回实例。"""
|
||||
fake = FakeLLM(contents=contents, decision_fn=decision_fn)
|
||||
@@ -191,6 +216,34 @@ def test_judge_category_delete_classes(monkeypatch) -> None:
|
||||
for cat in ("garbage", "overlay", "noise"):
|
||||
_patch_llm(monkeypatch, [cat])
|
||||
assert _judge_category(entries, 2, context_size=10, params={}) == cat
|
||||
def test_judge_category_sanitizes_noise_context(monkeypatch) -> None:
|
||||
"""上下文净化：规则层可确定性识别的垃圾（横线/HTML 等）在喂给 LLM 前
|
||||
替换为 [噪音] 占位，避免污染对目标条目的场景判断。
|
||||
|
||||
修复回归：OCR 输出中相邻字幕混有大量覆盖层垃圾（---------------、HTML、
|
||||
Marketing 等），原样进入 LLM 上下文会让模型误判整段为"水印覆盖层"，
|
||||
把相邻的真实对话误删（run_011d01f19999 中 190 条含 ≥4 汉字的对话被删）。
|
||||
"""
|
||||
entries = [
|
||||
{"start": "00:00:01,000", "end": "00:00:02,000", "text": "正常对话一"},
|
||||
{"start": "00:00:03,000", "end": "00:00:04,000", "text": "---------------"},
|
||||
{"start": "00:00:05,000", "end": "00:00:06,000", "text": "HTML"},
|
||||
{"start": "00:00:07,000", "end": "00:00:08,000", "text": "我是目标对话"},
|
||||
{"start": "00:00:09,000", "end": "00:00:10,000", "text": "---"},
|
||||
{"start": "00:00:11,000", "end": "00:00:12,000", "text": "正常对话二"},
|
||||
]
|
||||
fake = _patch_llm(monkeypatch, ["dialogue"])
|
||||
assert _judge_category(entries, 3, context_size=10, params={}) == "dialogue"
|
||||
content = fake.bodies[0]["messages"][1]["content"]
|
||||
lines = content.splitlines()
|
||||
# 上下文只含过滤后的字幕：确定性垃圾条目被剔除，原文不进入 LLM。
|
||||
assert lines == ["正常对话一", "【目标】我是目标对话", "正常对话二"]
|
||||
assert "---------------" not in content
|
||||
assert "HTML" not in content
|
||||
assert "---" not in content
|
||||
# 系统提示词明确说明上下文已过滤装饰/水印符号。
|
||||
assert "过滤" in fake.bodies[0]["messages"][0]["content"]
|
||||
|
||||
|
||||
|
||||
def test_judge_category_repeat_and_unknown_kept(monkeypatch) -> None:
|
||||
@@ -215,6 +268,47 @@ def test_judge_category_model_and_auth(monkeypatch) -> None:
|
||||
assert fake.headers[0]["Authorization"] == "Bearer sk-test"
|
||||
|
||||
|
||||
def test_judge_category_retries_on_429(monkeypatch) -> None:
|
||||
"""429 限流时指数退避重试，最终成功判定（配合弹性把并发压回配额内）。"""
|
||||
entries = parse_srt(_SRT)
|
||||
fake = FlakyLLM(failures=2, answer="dialogue")
|
||||
monkeypatch.setattr("nodes.llm_filter.time.sleep", lambda s: None)
|
||||
monkeypatch.setattr("nodes.llm_filter.urllib.request.urlopen", fake)
|
||||
assert _judge_category(entries, 2, context_size=10, params={}) == "dialogue"
|
||||
assert fake.calls == 3 # 失败 2 次 + 成功 1 次。
|
||||
|
||||
|
||||
def test_judge_category_retries_on_5xx(monkeypatch) -> None:
|
||||
"""服务端 5xx 同样退避重试。"""
|
||||
entries = parse_srt(_SRT)
|
||||
fake = FlakyLLM(failures=1, code=503, answer="dialogue")
|
||||
monkeypatch.setattr("nodes.llm_filter.time.sleep", lambda s: None)
|
||||
monkeypatch.setattr("nodes.llm_filter.urllib.request.urlopen", fake)
|
||||
assert _judge_category(entries, 2, context_size=10, params={}) == "dialogue"
|
||||
assert fake.calls == 2
|
||||
|
||||
|
||||
def test_judge_category_gives_up_after_retries(monkeypatch) -> None:
|
||||
"""重试耗尽仍失败时抛错：该条判定失败，任务失败后可重新处理数据。"""
|
||||
entries = parse_srt(_SRT)
|
||||
fake = FlakyLLM(failures=99)
|
||||
monkeypatch.setattr("nodes.llm_filter.time.sleep", lambda s: None)
|
||||
monkeypatch.setattr("nodes.llm_filter.urllib.request.urlopen", fake)
|
||||
with pytest.raises(urllib.error.HTTPError):
|
||||
_judge_category(entries, 2, context_size=10, params={})
|
||||
assert fake.calls == 3 # 最多尝试 3 次。
|
||||
|
||||
|
||||
def test_judge_category_other_errors_no_retry(monkeypatch) -> None:
|
||||
"""非 429/5xx 错误（如 400）不重试，直接抛出。"""
|
||||
entries = parse_srt(_SRT)
|
||||
fake = FlakyLLM(failures=1, code=400, answer="dialogue")
|
||||
monkeypatch.setattr("nodes.llm_filter.time.sleep", lambda s: None)
|
||||
monkeypatch.setattr("nodes.llm_filter.urllib.request.urlopen", fake)
|
||||
with pytest.raises(urllib.error.HTTPError):
|
||||
_judge_category(entries, 2, context_size=10, params={})
|
||||
assert fake.calls == 1 # 只调用一次。
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 去重键与删除判定
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -229,8 +323,22 @@ def test_dedup_key_normalizes_whitespace_and_case() -> None:
|
||||
assert _dedup_key("你好 世界") != _dedup_key("你好世界2")
|
||||
|
||||
|
||||
def test_should_delete_by_category_only() -> None:
|
||||
"""删除判定：garbage/overlay/noise 删，repeat/dialogue 留（短文本无保护）。"""
|
||||
assert _should_delete("garbage", "x", min_keep_len=12) is True
|
||||
assert _should_delete("overlay", "x", min_keep_len=12) is True
|
||||
assert _should_delete("noise", "x", min_keep_len=12) is True
|
||||
assert _should_delete("repeat", "x", min_keep_len=12) is False
|
||||
assert _should_delete("dialogue", "x", min_keep_len=12) is False
|
||||
|
||||
|
||||
def test_should_delete_long_text_noise_protected() -> None:
|
||||
"""长文本保护：≥min_keep_len 时 noise 不删，garbage/overlay 仍删；短文本三类都删。"""
|
||||
"""长文本保护：≥min_keep_len 时 noise 不删（LLM 判定不稳的兜底），
|
||||
garbage/overlay 仍删；短文本 noise 可删。
|
||||
|
||||
回归：移除保护后 run_011d01f19999 新增误删 124 条真实长对话
|
||||
（'很棒的表情呢 看 拍下来了吗' 等被 LLM 误判 noise），恢复保护。
|
||||
"""
|
||||
long_text = "不这么做的话 可没法胜任患者的对象"
|
||||
assert _should_delete("noise", long_text, min_keep_len=12) is False
|
||||
assert _should_delete("garbage", long_text, min_keep_len=12) is True
|
||||
@@ -240,6 +348,25 @@ def test_should_delete_long_text_noise_protected() -> None:
|
||||
assert _should_delete("dialogue", long_text, min_keep_len=12) is False
|
||||
|
||||
|
||||
def test_rule_verdict_removes_domain_and_html_watermark() -> None:
|
||||
"""正则确定性过滤：网址域名/HTML 水印等"一定需要移除"的模式直接删除。
|
||||
|
||||
覆盖真实案例：'98室[巴花堂] 水火地址 489155.com'（广告）、
|
||||
'HTML code for a simple blue background'（OCR 识别出的网页水印）。
|
||||
"""
|
||||
tokens = set(DEFAULT_OVERLAY_TOKENS)
|
||||
for text in (
|
||||
"98室[巴花堂] 水火地址 489155.com",
|
||||
"HTML code for a simple blue background",
|
||||
"http://example.com/path",
|
||||
"联系我们 admin@example.com",
|
||||
):
|
||||
assert _rule_verdict(text, tokens) is True, text
|
||||
# 正常对话不含垃圾模式 → 交 LLM 判断。
|
||||
assert _rule_verdict("青沼君 好可爱", tokens) is None
|
||||
assert _rule_verdict("再见了 再见", tokens) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# invoke 全链路
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -454,6 +581,39 @@ def test_invoke_llm_error(monkeypatch, tmp_path) -> None:
|
||||
)
|
||||
assert response.status == "failed"
|
||||
assert "llm down" in response.error
|
||||
def test_invoke_retries_rate_limited_entries(monkeypatch, tmp_path) -> None:
|
||||
"""判定遇 429（限流）时临时降并发并对失败条目重试，最终正常完成。
|
||||
|
||||
自适应：多线程并发打到 SiliconFlow 配额线触发 429 时，worker 通知线程池
|
||||
report_failure 临时收紧最大并发，失败条目在收紧后重试一轮，避免整体失败
|
||||
（run_011d01f19999 在 20 并发下因 429 重试耗尽而 FAILED 的修复）。
|
||||
"""
|
||||
srt = tmp_path / "in.srt"
|
||||
srt.write_text(
|
||||
"1\n00:00:01,000 --> 00:00:04,000\n第一句对话\n\n"
|
||||
"2\n00:00:05,000 --> 00:00:08,000\n---------------\n\n"
|
||||
"3\n00:00:09,000 --> 00:00:12,000\n第二句对话\n\n"
|
||||
"4\n00:00:13,000 --> 00:00:16,000\nHTML\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
# 前 3 次调用都 429（重试耗尽抛错），之后成功：首次 map 部分条目失败，
|
||||
# 二次重试在降并发后成功。
|
||||
fake = FlakyLLM(failures=3, answer="dialogue")
|
||||
monkeypatch.setattr("nodes.llm_filter.time.sleep", lambda s: None)
|
||||
monkeypatch.setattr("nodes.llm_filter.urllib.request.urlopen", fake)
|
||||
response = invoke(
|
||||
InvokeRequest(
|
||||
run_id="r", node_instance_id="",
|
||||
inputs={"srt_uri": str(srt)},
|
||||
params={},
|
||||
output_dir=str(tmp_path / "out"),
|
||||
)
|
||||
)
|
||||
assert response.status == "completed", response.error
|
||||
kept = parse_srt((tmp_path / "out" / "filtered.srt").read_text(encoding="utf-8"))
|
||||
assert [e["text"] for e in kept] == ["第一句对话", "第二句对话"]
|
||||
assert fake.calls >= 5 # 首次 map + 二次重试均有调用。
|
||||
|
||||
|
||||
|
||||
def test_invoke_empty_srt(monkeypatch, tmp_path) -> None:
|
||||
|
||||
@@ -497,3 +497,45 @@ def test_ocr_passes_short_text_through(monkeypatch, tmp_path) -> None:
|
||||
assert response.status == "completed", response.error
|
||||
srt = Path(response.outputs["srt_uri"]).read_text(encoding="utf-8")
|
||||
assert "SUB 001" in srt
|
||||
|
||||
|
||||
def test_ocr_interrupts_on_pause_flag(monkeypatch, tmp_path) -> None:
|
||||
"""暂停信号：任务被暂停（paused.flag 存在）时 OCR 立即中断。
|
||||
|
||||
output_dir = <storage>/runs/<run_id>/steps/ocr，run 根目录为其父父目录；
|
||||
调度器/API 在暂停时向 run 根写入 paused.flag，节点逐帧检查到即中止：
|
||||
不调用 vlm-ocr、不写该帧断点存档，invoke 返回 failed（由调度器识别为
|
||||
"被暂停"并保持 PAUSED，等待 resume 后从断点续跑）。
|
||||
"""
|
||||
image = tmp_path / "s0.png"
|
||||
image.write_bytes(TEXT_IMG.read_bytes())
|
||||
frames = [(i * 2.0, image) for i in range(4)]
|
||||
|
||||
# 暂停信号位于 run 根目录（steps/ocr 的父父目录）。
|
||||
run_root = tmp_path / "run_root"
|
||||
out_dir = run_root / "steps" / "ocr"
|
||||
out_dir.mkdir(parents=True)
|
||||
(run_root / "paused.flag").write_text("", encoding="utf-8")
|
||||
|
||||
calls: list[str] = []
|
||||
|
||||
def fake_vlm(node_id, request):
|
||||
calls.append(request.inputs["image_uri"])
|
||||
return InvokeResponse(status="completed", outputs={"text": "SUB 001"})
|
||||
|
||||
monkeypatch.setattr("wov_app.registry.invoke", fake_vlm)
|
||||
manifest = _frames_manifest(tmp_path, frames)
|
||||
response = ocr_invoke(
|
||||
InvokeRequest(
|
||||
run_id="run_paused_flag",
|
||||
node_instance_id="",
|
||||
inputs={"frames_manifest": str(manifest)},
|
||||
params={},
|
||||
output_dir=str(out_dir),
|
||||
)
|
||||
)
|
||||
# 节点以"被暂停"失败：调度器会据此保持 PAUSED 而不标 FAILED。
|
||||
assert response.status == "failed"
|
||||
assert "暂停" in (response.error or "")
|
||||
assert calls == [] # 一帧都没有真正 OCR。
|
||||
assert not (out_dir / "ocr_partial.jsonl").exists() # 未处理帧不入存档。
|
||||
|
||||
@@ -612,6 +612,132 @@ def test_execute_paused_run_not_run(tmp_path, monkeypatch) -> None:
|
||||
assert called == []
|
||||
|
||||
|
||||
def test_execute_paused_run_stays_paused(tmp_path, monkeypatch) -> None:
|
||||
"""PAUSED 任务不被 execute_run 复活：不置 RUNNING、不执行任何节点。
|
||||
|
||||
修复回归：PAUSED 任务被调度器拾起后曾先置 RUNNING 再检查，节点循环
|
||||
读到的是刚改的 RUNNING 状态，"暂停检查"永远不成立 → 任务被复活继续跑
|
||||
（用户观察到的"点击暂停反而开始任务"）。修复后 PAUSED 直接返回保持暂停。
|
||||
"""
|
||||
db = _db(tmp_path)
|
||||
db.upsert_workflow({"id": "flow", "name": "Flow", "published": 1, "latest_version": 1})
|
||||
db.create_workflow_version("flow", 1, _two_node_definition().to_dict())
|
||||
now = "2026-01-01T00:00:00+00:00"
|
||||
db.create_run(
|
||||
{
|
||||
"id": "run_paused",
|
||||
"workflow_id": "flow",
|
||||
"workflow_version": 1,
|
||||
"status": "PAUSED",
|
||||
"progress": 0.5,
|
||||
"current_node_id": "a",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
called = []
|
||||
|
||||
def fake_invoke(node_type, request):
|
||||
called.append(node_type)
|
||||
return InvokeResponse(status="completed", outputs={})
|
||||
|
||||
monkeypatch.setattr(registry, "invoke", fake_invoke)
|
||||
scheduler = WorkflowScheduler(db, tmp_path / "storage")
|
||||
scheduler.execute_run("run_paused")
|
||||
# 状态保持 PAUSED（未被置为 RUNNING），节点一个都不执行。
|
||||
assert db.get_run("run_paused")["status"] == "PAUSED"
|
||||
assert called == []
|
||||
|
||||
def test_execute_paused_during_node_keeps_paused(tmp_path, monkeypatch) -> None:
|
||||
"""节点内被暂停（节点检测到暂停信号后中止）：保持 PAUSED 不标 FAILED。
|
||||
|
||||
节点内暂停响应：subtitle-ocr 检查到 paused.flag 后中止并返回失败；
|
||||
调度器捕获节点异常时应检查任务状态——若已被置为 PAUSED（用户点了暂停），
|
||||
则保持 PAUSED 等待 resume 从断点续跑，而不是覆盖为 FAILED。
|
||||
"""
|
||||
db = _db(tmp_path)
|
||||
db.upsert_workflow({"id": "flow", "name": "Flow", "published": 1, "latest_version": 1})
|
||||
db.create_workflow_version("flow", 1, _two_node_definition().to_dict())
|
||||
now = "2026-01-01T00:00:00+00:00"
|
||||
db.create_run(
|
||||
{
|
||||
"id": "run_paused",
|
||||
"workflow_id": "flow",
|
||||
"workflow_version": 1,
|
||||
"status": "QUEUED",
|
||||
"progress": 0,
|
||||
"input_uri": str(tmp_path / "in.txt"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
|
||||
def fake_invoke(node_type, request):
|
||||
# 模拟节点内暂停：任务已被置 PAUSED，节点随后中止并抛异常。
|
||||
db.pause_run("run_paused", now)
|
||||
raise RuntimeError("OCR interrupted by pause")
|
||||
|
||||
monkeypatch.setattr(registry, "invoke", fake_invoke)
|
||||
WorkflowScheduler(db, tmp_path / "storage").execute_run("run_paused")
|
||||
# 保持 PAUSED，不标 FAILED、不写 error（等待用户 resume 断点续跑）。
|
||||
run = db.get_run("run_paused")
|
||||
assert run["status"] == "PAUSED"
|
||||
assert run["error"] is None
|
||||
|
||||
|
||||
def test_scheduler_loop_survives_poll_exception(tmp_path, monkeypatch) -> None:
|
||||
"""调度轮询遇异常不退出线程：下一轮继续执行排队任务。
|
||||
|
||||
修复回归：_loop 中 next_queued_run/execute_run 的未捕获异常曾杀死调度
|
||||
线程（worker 进程只剩 uvicorn 主线程），任务永远停留在 QUEUED——
|
||||
run_011d01f19999 实际发生：回退 QUEUED 后调度器不再拾起，新配置
|
||||
（pool_max_workers=20）因此从未执行。
|
||||
"""
|
||||
db = _db(tmp_path)
|
||||
input_file = tmp_path / "input.txt"
|
||||
input_file.write_text("resilient", encoding="utf-8")
|
||||
_register_echo()
|
||||
db.upsert_workflow({"id": "flow", "name": "Flow", "published": 1, "latest_version": 1})
|
||||
db.create_workflow_version("flow", 1, _echo_definition().to_dict())
|
||||
now = "2026-01-01T00:00:00+00:00"
|
||||
db.create_run(
|
||||
{
|
||||
"id": "run_resilient",
|
||||
"workflow_id": "flow",
|
||||
"workflow_version": 1,
|
||||
"status": "QUEUED",
|
||||
"progress": 0,
|
||||
"input_uri": str(input_file),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
|
||||
# 第一次轮询抛异常（模拟数据库抖动等），后续正常。
|
||||
calls = {"n": 0}
|
||||
real_next = db.next_queued_run
|
||||
|
||||
def flaky_next():
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
raise RuntimeError("transient db error")
|
||||
return real_next()
|
||||
|
||||
monkeypatch.setattr(db, "next_queued_run", flaky_next)
|
||||
scheduler = WorkflowScheduler(db, tmp_path / "storage", interval_seconds=0.05)
|
||||
scheduler.start()
|
||||
try:
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
if db.get_run("run_resilient")["status"] in {"COMPLETED", "FAILED"}:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
finally:
|
||||
scheduler.stop()
|
||||
# 第一次异常后调度线程仍存活，第二轮回合把任务执行完成。
|
||||
assert db.get_run("run_resilient")["status"] == "COMPLETED"
|
||||
assert calls["n"] >= 2
|
||||
|
||||
def test_execute_pause_after_last_node_keeps_paused(tmp_path, monkeypatch) -> None:
|
||||
"""验证全部节点完成但运行中被暂停时保持 PAUSED；续跑补做收尾后完成。"""
|
||||
db = _db(tmp_path)
|
||||
|
||||
@@ -213,3 +213,83 @@ class TestSubtitleOcrOrderUnderThreading:
|
||||
# ⑤ 已知真实内容存在于结果中（确认结果的代表性条目）。
|
||||
for line in ("北冈小姐", "这是特别病房患者的病历表", "应该已经察觉到 至今为止的一切了吧"):
|
||||
assert line in confirmed, line
|
||||
|
||||
|
||||
def test_ocr_resumes_from_partial_checkpoint(monkeypatch, tmp_path) -> None:
|
||||
"""节点级断点：预写部分帧存档后运行，只处理未处理帧，产物与全量一致。
|
||||
|
||||
模拟中断时已落盘的 ocr_partial.jsonl（前 100 帧已处理）：invoke 应只对
|
||||
未处理帧调用 vlm-ocr，并把存档文本与新增文本合并，最终 SRT 与全量一次
|
||||
跑完逐字节一致——重启不浪费已处理的帧。
|
||||
"""
|
||||
manifest, texts_by_frame = _load_full_data()
|
||||
confirmed = CONFIRMED_SRT.read_text(encoding="utf-8")
|
||||
out_dir = tmp_path / "resume"
|
||||
partial_path = out_dir / "ocr_partial.jsonl"
|
||||
partial_path.parent.mkdir(parents=True)
|
||||
lines = []
|
||||
for i in range(100):
|
||||
# 存档按 0-based 帧序号记录；manifest[i] 的帧号 = i+1。
|
||||
lines.append(
|
||||
json.dumps({"frame": i, "text": texts_by_frame[i + 1]}, ensure_ascii=False)
|
||||
)
|
||||
if i == 50:
|
||||
lines.append("") # 空行：_load_partial 必须跳过，不视为一条记录。
|
||||
if i == 51:
|
||||
lines.append("broken-json-line") # 损坏行（进程被杀残留）：跳过。
|
||||
partial_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
from nodes.subtitle_ocr import invoke as ocr_invoke
|
||||
|
||||
fake = FakeVlmOcrApi(texts_by_frame, seed=99)
|
||||
monkeypatch.setattr("wov_app.registry.invoke", fake)
|
||||
response = ocr_invoke(
|
||||
InvokeRequest(
|
||||
run_id="resume_test",
|
||||
node_instance_id="",
|
||||
inputs={"frames_manifest": str(FULL_MANIFEST)},
|
||||
params={"pool_min_workers": 4, "pool_max_workers": 4},
|
||||
output_dir=str(out_dir),
|
||||
)
|
||||
)
|
||||
assert response.status == "completed", response.error
|
||||
srt = Path(response.outputs["srt_uri"]).read_text(encoding="utf-8")
|
||||
assert srt == confirmed # 存档 + 新增合并结果与全量一致。
|
||||
# 只处理未处理帧：前 100 帧不再调用 vlm-ocr。
|
||||
|
||||
# 场景 2：先用真实逻辑完整跑一遍（生成存档），再以同一目录续跑——
|
||||
# 第二次 pending 为空，完全不调用 vlm-ocr，产物与第一次逐字节一致
|
||||
# （覆盖 _load_partial 全恢复路径，存档文本与处理逻辑天然一致）。
|
||||
out_dir_all = tmp_path / "resume_all"
|
||||
fake_first = FakeVlmOcrApi(texts_by_frame, seed=7)
|
||||
monkeypatch.setattr("wov_app.registry.invoke", fake_first)
|
||||
resp_first = ocr_invoke(
|
||||
InvokeRequest(
|
||||
run_id="resume_all_test",
|
||||
node_instance_id="",
|
||||
inputs={"frames_manifest": str(FULL_MANIFEST)},
|
||||
params={"pool_min_workers": 4, "pool_max_workers": 4},
|
||||
output_dir=str(out_dir_all),
|
||||
)
|
||||
)
|
||||
assert resp_first.status == "completed", resp_first.error
|
||||
srt_first = Path(resp_first.outputs["srt_uri"]).read_text(encoding="utf-8")
|
||||
assert srt_first == confirmed
|
||||
assert len(fake_first.completed_frames) == TOTAL_FRAMES
|
||||
|
||||
fake_second = FakeVlmOcrApi(texts_by_frame, seed=8)
|
||||
monkeypatch.setattr("wov_app.registry.invoke", fake_second)
|
||||
resp_second = ocr_invoke(
|
||||
InvokeRequest(
|
||||
run_id="resume_all_test",
|
||||
node_instance_id="",
|
||||
inputs={"frames_manifest": str(FULL_MANIFEST)},
|
||||
params={"pool_min_workers": 4, "pool_max_workers": 4},
|
||||
output_dir=str(out_dir_all),
|
||||
)
|
||||
)
|
||||
assert resp_second.status == "completed", resp_second.error
|
||||
srt_second = Path(resp_second.outputs["srt_uri"]).read_text(encoding="utf-8")
|
||||
assert srt_second == srt_first # 存档恢复与一次跑完逐字节一致。
|
||||
assert len(fake_second.completed_frames) == 0 # 一帧都不重新调用。
|
||||
assert len(fake.completed_frames) == TOTAL_FRAMES - 100
|
||||
|
||||
Reference in New Issue
Block a user