cat-shark
b16e0e9f3c
调度与状态机： - 修复 PAUSED 任务被拾起后复活执行（点击暂停反而开始任务）：next_queued_run 只取 QUEUED，execute_run 以 PAUSED 进入直接返回，暂停必须显式 resume - 重启恢复：启动时 recover_interrupted_runs 把遗留 RUNNING 置 QUEUED（保留产物） - 暂停信号 paused.flag：暂停接口写、继续/重试清除，OCR 逐帧检查秒级中断， 节点内被暂停保持 PAUSED 不误报 FAILED - 调度轮询容错：_loop 异常不杀死线程（曾致任务永久停留 QUEUED） subtitle-ocr 节点级断点： - ocr_partial.jsonl 逐帧存档，重启/暂停后只处理未处理帧，产物与一次跑完一致 - 进度日志携带窗口平均耗时与线程数；取消后抑制进度日志井喷 llm-filter 过滤质量与限流自适应： - 上下文净化：喂给 LLM 的是过滤后的字幕（规则层垃圾从上下文剔除） - 正则确定性过滤：裸网址域名、HTML/水印模式直接删除 - 429/5xx 指数退避重试；worker 限流错误 report_failure 内存临时降最大线程数 并缩容（无错误窗口回升），失败条目降并发后重试一轮 - 保留长文本保护（noise 不删 ≥min_keep_len 文本，LLM 判定不稳的必要兜底） 前端： - 工作流编排页支持选择工作流编辑（加载最新/历史版本）、版本历史面板、 新建/编辑双模式；管理后台编辑跳转 workflow.html?edit=<id> 工作流：ocr-subtitle v7（filter pool_max_workers=20、pool_fast_threshold=1）
276 lines
10 KiB
Python
276 lines
10 KiB
Python
"""自适应线程池测试。
|
|
|
|
覆盖决策函数（增/减/保持/边界）、map 顺序返回、worker 异常隔离，
|
|
以及"10s 窗口内平均响应 < 0.3s 加线程 / > 1.0s 减线程"的弹性行为
|
|
（通过注入假时钟做确定性验证）。
|
|
"""
|
|
|
|
import time
|
|
|
|
from nodes.adaptive_pool import AdaptiveThreadPool, decide
|
|
|
|
|
|
class FakeClock:
|
|
"""可手动拨动的假时钟，用于确定性验证弹性窗口逻辑。"""
|
|
|
|
def __init__(self, now: float = 0.0) -> None:
|
|
self.now = now
|
|
|
|
def __call__(self) -> float:
|
|
return self.now
|
|
|
|
def advance(self, seconds: float) -> None:
|
|
self.now += seconds
|
|
|
|
|
|
def test_decide_increase_when_fast() -> None:
|
|
"""平均响应低于 fast_threshold 且未达上限：线程数 +1。"""
|
|
assert decide(1, 0.1, 1, 16, 0.3, 1.0) == 2
|
|
|
|
|
|
def test_decide_decrease_when_slow() -> None:
|
|
"""平均响应高于 slow_threshold 且高于下限：线程数 -1。"""
|
|
assert decide(3, 2.0, 1, 16, 0.3, 1.0) == 2
|
|
|
|
|
|
def test_decide_keep_when_mid() -> None:
|
|
"""平均响应介于两阈值之间：保持不变。"""
|
|
assert decide(2, 0.5, 1, 16, 0.3, 1.0) == 2
|
|
|
|
|
|
def test_decide_bounds() -> None:
|
|
"""已达上限不再增、已达下限不再减。"""
|
|
assert decide(16, 0.1, 1, 16, 0.3, 1.0) == 16
|
|
assert decide(1, 2.0, 1, 16, 0.3, 1.0) == 1
|
|
|
|
|
|
def test_pool_map_ordered_results() -> None:
|
|
"""map 按输入顺序返回结果，worker 简单映射。"""
|
|
pool = AdaptiveThreadPool(worker=lambda item: item * 2)
|
|
assert pool.map([1, 2, 3, 4]) == [2, 4, 6, 8]
|
|
|
|
|
|
def test_pool_on_progress_callback() -> None:
|
|
"""进度回调：每次完成触发一次，携带已完成数/总数/速度/平均耗时/线程数。"""
|
|
progress: list[tuple[int, int, float, float, int]] = []
|
|
pool = AdaptiveThreadPool(
|
|
worker=lambda item: item,
|
|
on_progress=lambda done, total, rate, avg_time, workers: progress.append(
|
|
(done, total, rate, avg_time, workers)
|
|
),
|
|
)
|
|
pool.map([10, 20, 30])
|
|
assert [item[0] for item in progress] == [1, 2, 3] # 已完成数递增。
|
|
assert all(item[1] == 3 for item in progress) # 总数固定。
|
|
assert all(item[2] > 0 for item in progress) # 速度为正值。
|
|
# 窗口未满时平均耗时回退为累计平均（>0）；线程数 ∈ [1, 上限]。
|
|
assert all(item[3] > 0 for item in progress)
|
|
assert all(1 <= item[4] <= pool.max_workers for item in progress)
|
|
|
|
|
|
def test_pool_progress_reports_window_avg_after_first_window() -> None:
|
|
"""窗口评估后：回调携带最近窗口平均耗时（扩缩容依据）与扩容后的线程数。
|
|
|
|
覆盖 `_current_avg_time` 两个分支：窗口评估前回退累计平均，评估后使用
|
|
最近窗口平均（0.0s，响应远快于 fast_threshold 0.3s → 线程 +1）。
|
|
"""
|
|
clock = FakeClock()
|
|
seen: list[tuple[int, int, float, float, int]] = []
|
|
pool = AdaptiveThreadPool(
|
|
worker=lambda item: item,
|
|
min_workers=1, max_workers=16,
|
|
window_seconds=10.0, fast_threshold=0.3,
|
|
clock=clock,
|
|
on_progress=lambda done, total, rate, avg_time, workers: seen.append(
|
|
(done, total, rate, avg_time, workers)
|
|
),
|
|
)
|
|
clock.advance(11) # 首个任务完成即越过窗口 → 触发评估。
|
|
pool.map(list(range(5)))
|
|
# 第一个完成的任务回调在窗口评估前：回退累计平均（>0）。
|
|
assert seen[0][3] > 0
|
|
# 窗口评估后：回调携带最近窗口平均（≈0.0），且线程已扩容到 2。
|
|
assert any(item[3] == 0.0 for item in seen)
|
|
assert any(item[4] == 2 for item in seen)
|
|
assert pool.max_concurrency == 2
|
|
def test_pool_cancel_suppresses_progress() -> None:
|
|
"""worker 触发 cancel（如检测到暂停信号）后：剩余任务不再触发进度回调。
|
|
|
|
暂停场景：队列中剩余的大量帧会逐帧快速失败退出，若每完成一项都打印
|
|
进度日志，会在数秒内打出上万行日志；cancel 后抑制后续进度回调。
|
|
"""
|
|
progress: list[tuple[int, int, float, float, int]] = []
|
|
|
|
def worker(item):
|
|
if item == 1:
|
|
pool.cancel() # 模拟某帧检测到暂停信号。
|
|
return item
|
|
|
|
pool = AdaptiveThreadPool(
|
|
worker=worker,
|
|
on_progress=lambda done, total, rate, avg_time, workers: progress.append(
|
|
(done, total, rate, avg_time, workers)
|
|
),
|
|
)
|
|
pool.map([0, 1, 2, 3])
|
|
# 只有 cancel 之前的任务（item=0）触发了进度回调。
|
|
assert len(progress) == 1
|
|
assert progress[0][1] == 4 # 总数仍是 4。
|
|
|
|
|
|
def test_pool_cancel_resets_between_maps() -> None:
|
|
"""取消状态按批（map）重置：下一批任务进度回调恢复正常。"""
|
|
progress: list[tuple[int, int, float, float, int]] = []
|
|
|
|
def worker(item):
|
|
if item == "stop":
|
|
pool.cancel()
|
|
return item
|
|
|
|
pool = AdaptiveThreadPool(
|
|
worker=worker,
|
|
on_progress=lambda done, total, rate, avg_time, workers: progress.append(
|
|
(done, total, rate, avg_time, workers)
|
|
),
|
|
)
|
|
pool.map(["a", "stop", "b"])
|
|
assert len(progress) == 1 # 只有 cancel 前的 a 触发回调。
|
|
pool.map(["c", "d"])
|
|
# 新一批恢复回调（done 从 1 重新计数）：共 3 次回调（1 + 2）。
|
|
assert [item[0] for item in progress] == [1, 1, 2]
|
|
|
|
def test_pool_map_empty() -> None:
|
|
|
|
"""空输入：不启动任务，直接返回空列表。"""
|
|
pool = AdaptiveThreadPool(worker=lambda item: item)
|
|
assert pool.map([]) == []
|
|
|
|
|
|
def test_pool_worker_exception_isolated() -> None:
|
|
"""worker 抛异常时以异常对象作为结果，不拖垮整体。"""
|
|
def boom(item):
|
|
raise RuntimeError("boom")
|
|
|
|
pool = AdaptiveThreadPool(worker=boom)
|
|
results = pool.map([1, 2])
|
|
assert len(results) == 2
|
|
assert all(isinstance(result, RuntimeError) for result in results)
|
|
|
|
|
|
def test_pool_grows_when_fast() -> None:
|
|
"""10s 窗口内平均响应 < 0.3s：线程数从 1 增至 2（弹性扩容）。"""
|
|
clock = FakeClock()
|
|
pool = AdaptiveThreadPool(
|
|
worker=lambda item: item,
|
|
min_workers=1, max_workers=16,
|
|
window_seconds=10.0, fast_threshold=0.3,
|
|
clock=clock,
|
|
)
|
|
# 拨快时钟越过窗口：首个任务完成即触发评估 → 平均响应≈0 < 0.3 → +1 线程。
|
|
clock.advance(11)
|
|
pool.map(list(range(4)))
|
|
assert pool.max_concurrency == 2
|
|
|
|
|
|
def test_pool_shrink_when_slow() -> None:
|
|
"""窗口平均响应 > 1.0s：线程数从 2 减至 1（弹性退避）。"""
|
|
clock = FakeClock()
|
|
pool = AdaptiveThreadPool(
|
|
worker=lambda item: item,
|
|
min_workers=1, max_workers=16,
|
|
window_seconds=10.0, fast_threshold=0.3, slow_threshold=1.0,
|
|
clock=clock,
|
|
)
|
|
pool._resize(2) # 先扩到 2 个线程。
|
|
clock.advance(11)
|
|
pool._tick(2.0) # 窗口内平均 2.0 > 1.0 → 缩回 1。
|
|
deadline = time.monotonic() + 2
|
|
while len(pool._threads) > 1 and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
assert len(pool._threads) == 1
|
|
pool._stop.set()
|
|
|
|
|
|
def test_resize_shrink_idempotent() -> None:
|
|
"""回归：重复缩容到同一目标不会重复放哨兵（曾因并发缩容毒死全部线程而死锁）。"""
|
|
pool = AdaptiveThreadPool(worker=lambda item: item, min_workers=1, max_workers=8)
|
|
pool._resize(3)
|
|
assert pool._target_workers == 3
|
|
pool._resize(2)
|
|
pool._resize(2) # 目标已是 2：幂等，不再放哨兵。
|
|
assert pool._target_workers == 2
|
|
# 只有 1 个线程被哨兵退出，最终存活 2 个。
|
|
deadline = time.monotonic() + 2
|
|
while len(pool._threads) > 2 and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
assert len(pool._threads) == 2
|
|
pool._stop.set()
|
|
|
|
|
|
def test_pool_survives_mixed_grow_shrink() -> None:
|
|
"""回归：扩容+缩容混合场景 map 必须完成且保序（修复前会死锁挂起）。"""
|
|
clock = FakeClock()
|
|
state = {"count": 0}
|
|
|
|
def worker(item):
|
|
state["count"] += 1
|
|
clock.advance(0.06 if state["count"] <= 20 else 0.6)
|
|
return item
|
|
|
|
pool = AdaptiveThreadPool(
|
|
worker=worker, min_workers=1, max_workers=4,
|
|
window_seconds=0.5, fast_threshold=0.2, slow_threshold=0.4,
|
|
clock=clock,
|
|
)
|
|
out = pool.map(list(range(60)))
|
|
assert out == list(range(60))
|
|
|
|
|
|
def test_pool_report_failure_lowers_effective_max() -> None:
|
|
"""消费错误（如 API 限流）临时降低有效最大线程数，下限为 min_workers。
|
|
|
|
自适应：并发打到配额线触发 429 时，report_failure 收紧有效上限，
|
|
后续请求减少从而避开持续限流。
|
|
"""
|
|
pool = AdaptiveThreadPool(worker=lambda item: item, min_workers=1, max_workers=16)
|
|
assert pool._effective_max_workers == 16
|
|
pool.report_failure()
|
|
assert pool._effective_max_workers == 15
|
|
for _ in range(30):
|
|
pool.report_failure()
|
|
assert pool._effective_max_workers == 1 # 下限 min_workers。
|
|
|
|
|
|
def test_pool_effective_max_recovers_after_clean_window() -> None:
|
|
"""连续无错误窗口后有效上限逐步回升到 max_workers。"""
|
|
clock = FakeClock()
|
|
pool = AdaptiveThreadPool(
|
|
worker=lambda item: item, min_workers=1, max_workers=16,
|
|
window_seconds=10.0, fast_threshold=0.3, clock=clock,
|
|
)
|
|
pool.report_failure() # 有效上限 16 -> 15。
|
|
clock.advance(11)
|
|
pool._tick(0.01) # 错误所在窗口：上限不恢复。
|
|
assert pool._effective_max_workers == 15
|
|
clock.advance(11)
|
|
pool._tick(0.01) # 下一个干净窗口：恢复 +1。
|
|
assert pool._effective_max_workers == 16
|
|
pool._tick(0.01) # 窗口未满早退，上限不变。
|
|
assert pool._effective_max_workers == 16
|
|
|
|
|
|
def test_pool_decide_uses_effective_max() -> None:
|
|
"""扩容上限按有效最大线程数：错误窗口内即使响应快也不超过收紧后的上限。"""
|
|
clock = FakeClock()
|
|
pool = AdaptiveThreadPool(
|
|
worker=lambda item: item, min_workers=1, max_workers=16,
|
|
window_seconds=10.0, fast_threshold=0.3, clock=clock,
|
|
)
|
|
pool.report_failure() # 有效上限 16 -> 15。
|
|
pool._resize(15)
|
|
pool._window_failures = 1 # 本窗口内仍有错误 → 不恢复上限。
|
|
clock.advance(11)
|
|
pool._tick(0.01) # 响应快，但 15 已是有效上限 → 不扩。
|
|
assert pool._target_workers == 15
|
|
assert pool._effective_max_workers == 15
|