fix: 批量任务残留PENDING仍误标COMPLETED——置完成前校验无残留+崩溃恢复批量任务

事故(batch_969fabe74b83 等 3 个任务实测):批量引擎串行处理到 9.9GB
大视频时中断,_run_job 无条件收尾把任务置 COMPLETED,留下"N 个 PENDING
待处理却已完成"的僵尸状态——3 个任务遗留的 PENDING 完全相同(kiwvr-887/
mdvr-413/savr-1077 共 10 个大视频),且均卡在大文件前从未被真正处理。

- batch.py: _run_job 置 COMPLETED 前校验全部非 SKIPPED 视频已结束
  (无 PENDING/PAUSED 残留),否则保持 RUNNING 交引擎下一轮续跑
- db.py/main.py: 新增 recover_interrupted_batch_jobs,重启时把 RUNNING
  批量任务恢复 QUEUED(否则停在 RUNNING 永远不会被再次拾起)
- tests: 新增 2 条 TDD 回归(残留 PENDING 不置 COMPLETED、批量崩溃恢复)
- scripts/fix_zombie_batch_jobs.py: 历史僵尸数据修复脚本(置回 QUEUED 续跑)
- AGENTS.md: 补充完成任务判定与崩溃恢复约定
This commit is contained in:
2026-09-07 11:25:33 +08:00
parent 4d2a1912da
commit c063aed1f1
7 changed files with 219 additions and 0 deletions
+33
View File
@@ -299,6 +299,39 @@ def test_recover_interrupted_runs(tmp_path) -> None:
# 恢复后调度器可拾起并断点续跑。
assert db.next_queued_run()["id"] == "run_orphan"
def test_recover_interrupted_batch_jobs(tmp_path) -> None:
"""重启恢复:遗留 RUNNING 的批量任务恢复为 QUEUED,供引擎重新拾起。
批量引擎处理视频(尤其大文件)时进程被杀/重启,批量任务停在 RUNNING。
若保持 RUNNINGnext_queued_batch_job 只拾取 QUEUED,任务永远不会被再次
驱动 → 未处理完的 PENDING 视频永久残留(batch_969fabe74b83 事故链路之一)。
恢复为 QUEUED 后引擎重新拾起续跑;PAUSED 批量任务保持不变等待显式 resume。
"""
db = Database(tmp_path / "wov.db")
now = "2026-01-01T00:00:00+00:00"
def add_job(job_id: str, status: str) -> None:
"""创建指定状态的批量任务记录。"""
db.create_batch_job(
{
"id": job_id, "folder_path": "/tmp/videos", "workflow_id": "demo",
"recursive": 1, "status": status, "progress": 0.5, "total": 3,
"done": 1, "failed": 0, "error": None,
"created_at": now, "updated_at": now,
}
)
add_job("job_orphan", "RUNNING")
add_job("job_paused", "PAUSED")
add_job("job_done", "COMPLETED")
recovered = db.recover_interrupted_batch_jobs("2026-01-02T00:00:00+00:00")
assert recovered == 1 # 只有 RUNNING 被恢复。
assert db.get_batch_job("job_orphan")["status"] == "QUEUED"
assert db.get_batch_job("job_paused")["status"] == "PAUSED"
assert db.get_batch_job("job_done")["status"] == "COMPLETED"
# 恢复后批量引擎可拾起并从断点续跑。
assert db.next_queued_batch_job()["id"] == "job_orphan"
def test_restore_run_outputs(tmp_path) -> None:
"""验证从产物重建节点输出(断点续跑的依据)。"""
db = Database(tmp_path / "wov.db")