"""自适应线程池测试。 覆盖决策函数(增/减/保持/边界)、map 顺序返回、worker 异常隔离, 以及"10s 窗口内平均响应 < 0.3s 加线程 / > 1.0s 减线程"的弹性行为 (通过注入假时钟做确定性验证)。 """ import time from nodes.adaptive_pool import AdaptiveThreadPool, decide class FakeClock: """可手动拨动的假时钟,用于确定性验证弹性窗口逻辑。""" def __init__(self, now: float = 0.0) -> None: self.now = now def __call__(self) -> float: return self.now def advance(self, seconds: float) -> None: self.now += seconds def test_decide_increase_when_fast() -> None: """平均响应低于 fast_threshold 且未达上限:线程数 +1。""" assert decide(1, 0.1, 1, 16, 0.3, 1.0) == 2 def test_decide_decrease_when_slow() -> None: """平均响应高于 slow_threshold 且高于下限:线程数 -1。""" assert decide(3, 2.0, 1, 16, 0.3, 1.0) == 2 def test_decide_keep_when_mid() -> None: """平均响应介于两阈值之间:保持不变。""" assert decide(2, 0.5, 1, 16, 0.3, 1.0) == 2 def test_decide_bounds() -> None: """已达上限不再增、已达下限不再减。""" assert decide(16, 0.1, 1, 16, 0.3, 1.0) == 16 assert decide(1, 2.0, 1, 16, 0.3, 1.0) == 1 def test_pool_map_ordered_results() -> None: """map 按输入顺序返回结果,worker 简单映射。""" pool = AdaptiveThreadPool(worker=lambda item: item * 2) assert pool.map([1, 2, 3, 4]) == [2, 4, 6, 8] def test_pool_on_progress_callback() -> None: """进度回调:每次完成触发一次,携带已完成数/总数/速度/平均耗时/线程数。""" progress: list[tuple[int, int, float, float, int]] = [] pool = AdaptiveThreadPool( worker=lambda item: item, on_progress=lambda done, total, rate, avg_time, workers: progress.append( (done, total, rate, avg_time, workers) ), ) pool.map([10, 20, 30]) assert [item[0] for item in progress] == [1, 2, 3] # 已完成数递增。 assert all(item[1] == 3 for item in progress) # 总数固定。 assert all(item[2] > 0 for item in progress) # 速度为正值。 # 窗口未满时平均耗时回退为累计平均(>0);线程数 ∈ [1, 上限]。 assert all(item[3] > 0 for item in progress) assert all(1 <= item[4] <= pool.max_workers for item in progress) def test_pool_progress_reports_window_avg_after_first_window() -> None: """窗口评估后:回调携带最近窗口平均耗时(扩缩容依据)与扩容后的线程数。 覆盖 `_current_avg_time` 两个分支:窗口评估前回退累计平均,评估后使用 最近窗口平均(0.0s,响应远快于 fast_threshold 0.3s → 线程 +1)。 """ clock = FakeClock() seen: list[tuple[int, int, float, float, int]] = [] pool = AdaptiveThreadPool( worker=lambda item: item, min_workers=1, max_workers=16, window_seconds=10.0, fast_threshold=0.3, clock=clock, on_progress=lambda done, total, rate, avg_time, workers: seen.append( (done, total, rate, avg_time, workers) ), ) clock.advance(11) # 首个任务完成即越过窗口 → 触发评估。 pool.map(list(range(5))) # 第一个完成的任务回调在窗口评估前:回退累计平均(>0)。 assert seen[0][3] > 0 # 窗口评估后:回调携带最近窗口平均(≈0.0),且线程已扩容到 2。 assert any(item[3] == 0.0 for item in seen) assert any(item[4] == 2 for item in seen) assert pool.max_concurrency == 2 def test_pool_cancel_suppresses_progress() -> None: """worker 触发 cancel(如检测到暂停信号)后:剩余任务不再触发进度回调。 暂停场景:队列中剩余的大量帧会逐帧快速失败退出,若每完成一项都打印 进度日志,会在数秒内打出上万行日志;cancel 后抑制后续进度回调。 """ progress: list[tuple[int, int, float, float, int]] = [] def worker(item): if item == 1: pool.cancel() # 模拟某帧检测到暂停信号。 return item pool = AdaptiveThreadPool( worker=worker, on_progress=lambda done, total, rate, avg_time, workers: progress.append( (done, total, rate, avg_time, workers) ), ) pool.map([0, 1, 2, 3]) # 只有 cancel 之前的任务(item=0)触发了进度回调。 assert len(progress) == 1 assert progress[0][1] == 4 # 总数仍是 4。 def test_pool_cancel_resets_between_maps() -> None: """取消状态按批(map)重置:下一批任务进度回调恢复正常。""" progress: list[tuple[int, int, float, float, int]] = [] def worker(item): if item == "stop": pool.cancel() return item pool = AdaptiveThreadPool( worker=worker, on_progress=lambda done, total, rate, avg_time, workers: progress.append( (done, total, rate, avg_time, workers) ), ) pool.map(["a", "stop", "b"]) assert len(progress) == 1 # 只有 cancel 前的 a 触发回调。 pool.map(["c", "d"]) # 新一批恢复回调(done 从 1 重新计数):共 3 次回调(1 + 2)。 assert [item[0] for item in progress] == [1, 1, 2] def test_pool_map_empty() -> None: """空输入:不启动任务,直接返回空列表。""" pool = AdaptiveThreadPool(worker=lambda item: item) assert pool.map([]) == [] def test_pool_worker_exception_isolated() -> None: """worker 抛异常时以异常对象作为结果,不拖垮整体。""" def boom(item): raise RuntimeError("boom") pool = AdaptiveThreadPool(worker=boom) results = pool.map([1, 2]) assert len(results) == 2 assert all(isinstance(result, RuntimeError) for result in results) def test_pool_grows_when_fast() -> None: """10s 窗口内平均响应 < 0.3s:线程数从 1 增至 2(弹性扩容)。""" clock = FakeClock() pool = AdaptiveThreadPool( worker=lambda item: item, min_workers=1, max_workers=16, window_seconds=10.0, fast_threshold=0.3, clock=clock, ) # 拨快时钟越过窗口:首个任务完成即触发评估 → 平均响应≈0 < 0.3 → +1 线程。 clock.advance(11) pool.map(list(range(4))) assert pool.max_concurrency == 2 def test_pool_shrink_when_slow() -> None: """窗口平均响应 > 1.0s:线程数从 2 减至 1(弹性退避)。""" clock = FakeClock() pool = AdaptiveThreadPool( worker=lambda item: item, min_workers=1, max_workers=16, window_seconds=10.0, fast_threshold=0.3, slow_threshold=1.0, clock=clock, ) pool._resize(2) # 先扩到 2 个线程。 clock.advance(11) pool._tick(2.0) # 窗口内平均 2.0 > 1.0 → 缩回 1。 deadline = time.monotonic() + 2 while len(pool._threads) > 1 and time.monotonic() < deadline: time.sleep(0.01) assert len(pool._threads) == 1 pool._stop.set() def test_resize_shrink_idempotent() -> None: """回归:重复缩容到同一目标不会重复放哨兵(曾因并发缩容毒死全部线程而死锁)。""" pool = AdaptiveThreadPool(worker=lambda item: item, min_workers=1, max_workers=8) pool._resize(3) assert pool._target_workers == 3 pool._resize(2) pool._resize(2) # 目标已是 2:幂等,不再放哨兵。 assert pool._target_workers == 2 # 只有 1 个线程被哨兵退出,最终存活 2 个。 deadline = time.monotonic() + 2 while len(pool._threads) > 2 and time.monotonic() < deadline: time.sleep(0.01) assert len(pool._threads) == 2 pool._stop.set() def test_pool_survives_mixed_grow_shrink() -> None: """回归:扩容+缩容混合场景 map 必须完成且保序(修复前会死锁挂起)。""" clock = FakeClock() state = {"count": 0} def worker(item): state["count"] += 1 clock.advance(0.06 if state["count"] <= 20 else 0.6) return item pool = AdaptiveThreadPool( worker=worker, min_workers=1, max_workers=4, window_seconds=0.5, fast_threshold=0.2, slow_threshold=0.4, clock=clock, ) out = pool.map(list(range(60))) assert out == list(range(60)) def test_pool_report_failure_lowers_effective_max() -> None: """消费错误(如 API 限流)临时降低有效最大线程数,下限为 min_workers。 自适应:并发打到配额线触发 429 时,report_failure 收紧有效上限, 后续请求减少从而避开持续限流。 """ pool = AdaptiveThreadPool(worker=lambda item: item, min_workers=1, max_workers=16) assert pool._effective_max_workers == 16 pool.report_failure() assert pool._effective_max_workers == 15 for _ in range(30): pool.report_failure() assert pool._effective_max_workers == 1 # 下限 min_workers。 def test_pool_effective_max_recovers_after_clean_window() -> None: """连续无错误窗口后有效上限逐步回升到 max_workers。""" clock = FakeClock() pool = AdaptiveThreadPool( worker=lambda item: item, min_workers=1, max_workers=16, window_seconds=10.0, fast_threshold=0.3, clock=clock, ) pool.report_failure() # 有效上限 16 -> 15。 clock.advance(11) pool._tick(0.01) # 错误所在窗口:上限不恢复。 assert pool._effective_max_workers == 15 clock.advance(11) pool._tick(0.01) # 下一个干净窗口:恢复 +1。 assert pool._effective_max_workers == 16 pool._tick(0.01) # 窗口未满早退,上限不变。 assert pool._effective_max_workers == 16 def test_pool_decide_uses_effective_max() -> None: """扩容上限按有效最大线程数:错误窗口内即使响应快也不超过收紧后的上限。""" clock = FakeClock() pool = AdaptiveThreadPool( worker=lambda item: item, min_workers=1, max_workers=16, window_seconds=10.0, fast_threshold=0.3, clock=clock, ) pool.report_failure() # 有效上限 16 -> 15。 pool._resize(15) pool._window_failures = 1 # 本窗口内仍有错误 → 不恢复上限。 clock.advance(11) pool._tick(0.01) # 响应快,但 15 已是有效上限 → 不扩。 assert pool._target_workers == 15 assert pool._effective_max_workers == 15