fix: 批量任务残留PENDING仍误标COMPLETED——置完成前校验无残留+崩溃恢复批量任务
事故(batch_969fabe74b83 等 3 个任务实测):批量引擎串行处理到 9.9GB 大视频时中断,_run_job 无条件收尾把任务置 COMPLETED,留下"N 个 PENDING 待处理却已完成"的僵尸状态——3 个任务遗留的 PENDING 完全相同(kiwvr-887/ mdvr-413/savr-1077 共 10 个大视频),且均卡在大文件前从未被真正处理。 - batch.py: _run_job 置 COMPLETED 前校验全部非 SKIPPED 视频已结束 (无 PENDING/PAUSED 残留),否则保持 RUNNING 交引擎下一轮续跑 - db.py/main.py: 新增 recover_interrupted_batch_jobs,重启时把 RUNNING 批量任务恢复 QUEUED(否则停在 RUNNING 永远不会被再次拾起) - tests: 新增 2 条 TDD 回归(残留 PENDING 不置 COMPLETED、批量崩溃恢复) - scripts/fix_zombie_batch_jobs.py: 历史僵尸数据修复脚本(置回 QUEUED 续跑) - AGENTS.md: 补充完成任务判定与崩溃恢复约定
This commit is contained in:
@@ -634,6 +634,69 @@ def test_batch_worker_run_deleted_by_scheduler_returns(tmp_path, monkeypatch) ->
|
||||
assert video["status"] == "PENDING"
|
||||
|
||||
|
||||
def test_batch_worker_pending_leftover_never_marks_completed(tmp_path, monkeypatch) -> None:
|
||||
"""残留未处理 PENDING 视频时,任务**不得**置 COMPLETED(状态机缺陷回归)。
|
||||
|
||||
真实事故(batch_969fabe74b83 等 3 个任务):引擎串行处理到 9.9GB 大视频时
|
||||
execute_run 异常中断,_process_video 返回但该视频未被标 FAILED/COMPLETED
|
||||
(保持 PENDING),_run_job 循环照常走完剩余 SKIPPED 后**无条件**收尾置
|
||||
COMPLETED——留下"10 个待处理却已完成"的僵尸状态。
|
||||
|
||||
本测试用假调度器复现:第一个视频 execute_run 抛异常且不标任何状态
|
||||
(run 被删除的极端情形同路径),第二个视频正常;断言任务必须保持
|
||||
RUNNING(而非 COMPLETED),等待引擎下一次拾起重跑剩余 PENDING。
|
||||
"""
|
||||
db = _db(tmp_path)
|
||||
_seed_echo_workflow(db)
|
||||
# a.mp4 待处理(处理中崩溃);b.mp4 旁放好字幕 → 创建即 SKIPPED,不参与处理。
|
||||
folder = _video_folder(tmp_path, names=("a.mp4", "b.mp4"))
|
||||
(folder / "b.CN.srt").write_text("1\n00:00:00,000 --> 00:00:01,000\nb\n", encoding="utf-8")
|
||||
job = _make_job(db, folder)
|
||||
videos0 = {Path(v["video_path"]).name: v for v in db.list_batch_videos(job["id"])}
|
||||
assert videos0["b.mp4"]["status"] == "SKIPPED" # 前置:旁挂字幕命中跳过
|
||||
|
||||
class _CrashScheduler:
|
||||
"""模拟进程在第一个(唯一待处理)视频处理中被杀:run 消失、视频保持 PENDING。"""
|
||||
|
||||
def __init__(self, db: Database, work_dir: Path) -> None:
|
||||
self.db = db
|
||||
|
||||
def execute_run(self, run_id: str) -> None:
|
||||
# 等同进程崩溃后 run 记录残留被清/丢失,视频仍是 PENDING。
|
||||
self.db.delete_run(run_id)
|
||||
|
||||
monkeypatch.setattr("wov_app.batch.WorkflowScheduler", _CrashScheduler)
|
||||
worker = BatchWorker(db, interval_seconds=0.05)
|
||||
# 第一轮:a 崩溃残留 PENDING → 任务必须仍 RUNNING(允许续跑),不得 COMPLETED。
|
||||
worker._process_job(job)
|
||||
job = db.get_batch_job(job["id"])
|
||||
videos = {Path(v["video_path"]).name: v for v in db.list_batch_videos(job["id"])}
|
||||
# 缺陷行为:job 被置 COMPLETED(无条件收尾);期望保持 RUNNING。
|
||||
assert videos["a.mp4"]["status"] == "PENDING"
|
||||
assert videos["b.mp4"]["status"] == "SKIPPED"
|
||||
assert job["status"] == "RUNNING", (
|
||||
f"残留 PENDING 时任务被误置为 {job['status']}(缺陷:无条件收尾置 COMPLETED)"
|
||||
)
|
||||
|
||||
# 第二轮(引擎重启后再次拾起 RUNNING 任务):a 这次正常完成 → 全部完成才 COMPLETED。
|
||||
monkeypatch.setattr("wov_app.batch.WorkflowScheduler", _CrashScheduler)
|
||||
# 放开假调度器的崩溃:第二次调用时不再删 run,让真实调度器跑通。
|
||||
class _RecoveringScheduler:
|
||||
"""第二轮:不再崩溃,把 run 置 COMPLETED(由引擎收尾放产物)。"""
|
||||
|
||||
def __init__(self, db: Database, work_dir: Path) -> None:
|
||||
self.db = db
|
||||
|
||||
def execute_run(self, run_id: str) -> None:
|
||||
self.db.update_run(run_id, status="COMPLETED", updated_at=_now_iso())
|
||||
|
||||
monkeypatch.setattr("wov_app.batch.WorkflowScheduler", _RecoveringScheduler)
|
||||
worker._process_job(job)
|
||||
videos = {Path(v["video_path"]).name: v for v in db.list_batch_videos(job["id"])}
|
||||
assert videos["a.mp4"]["status"] == "COMPLETED"
|
||||
assert db.get_batch_job(job["id"])["status"] == "COMPLETED"
|
||||
|
||||
|
||||
def test_batch_worker_already_completed_runs_finalized(tmp_path) -> None:
|
||||
"""run 已完成但视频未标记(收尾前中断):直接放置产物、清理并标记完成。
|
||||
|
||||
|
||||
Reference in New Issue
Block a user