fix: 批量任务残留PENDING仍误标COMPLETED——置完成前校验无残留+崩溃恢复批量任务
事故(batch_969fabe74b83 等 3 个任务实测):批量引擎串行处理到 9.9GB 大视频时中断,_run_job 无条件收尾把任务置 COMPLETED,留下"N 个 PENDING 待处理却已完成"的僵尸状态——3 个任务遗留的 PENDING 完全相同(kiwvr-887/ mdvr-413/savr-1077 共 10 个大视频),且均卡在大文件前从未被真正处理。 - batch.py: _run_job 置 COMPLETED 前校验全部非 SKIPPED 视频已结束 (无 PENDING/PAUSED 残留),否则保持 RUNNING 交引擎下一轮续跑 - db.py/main.py: 新增 recover_interrupted_batch_jobs,重启时把 RUNNING 批量任务恢复 QUEUED(否则停在 RUNNING 永远不会被再次拾起) - tests: 新增 2 条 TDD 回归(残留 PENDING 不置 COMPLETED、批量崩溃恢复) - scripts/fix_zombie_batch_jobs.py: 历史僵尸数据修复脚本(置回 QUEUED 续跑) - AGENTS.md: 补充完成任务判定与崩溃恢复约定
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
"""修复僵尸批量任务脚本:把误标 COMPLETED 但仍有 PENDING 视频的任务置回 QUEUED。
|
||||
|
||||
背景(batch_969fabe74b83 事故):批量引擎处理大视频时中断,_run_job 无条件
|
||||
收尾把任务置 COMPLETED,留下"N 个 PENDING 待处理却已完成"的僵尸状态。
|
||||
代码已修复(置 COMPLETED 前校验无 PENDING 残留 + 重启恢复 RUNNING 批量任务),
|
||||
本脚本用于修复**历史遗留**的 3 个僵尸任务数据:
|
||||
- batch_969fabe74b83(10 个 PENDING,0 完成)
|
||||
- batch_1febe532a7cd(10 个 PENDING,11 完成)
|
||||
- batch_73c2b723456a(10 个 PENDING,6 完成)
|
||||
|
||||
修复方式:仅把这 3 个任务置回 QUEUED 并清空误导的 total/progress,保留
|
||||
SKIPPED/COMPLETED 明细与 run 记录;批量引擎(新代码)重启后会重新拾起,
|
||||
从剩余 PENDING 视频续跑,全部处理完才置 COMPLETED。
|
||||
|
||||
安全约束:只更新明确列出的 3 个 job_id;其余任务(含正常 COMPLETED 的
|
||||
fee6681/ac8ca4f)不动。执行前打印将变更的任务与明细统计供确认。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from wov_app.db import Database
|
||||
|
||||
# 待修复的僵尸任务(经诊断确认:COMPLETED 但仍有 PENDING 残留)。
|
||||
ZOMBIE_JOBS = [
|
||||
"batch_969fabe74b83", # 0 完成 / 10 PENDING(最严重,从未真正处理)
|
||||
"batch_1febe532a7cd", # 11 完成 / 10 PENDING
|
||||
"batch_73c2b723456a", # 6 完成 / 10 PENDING
|
||||
]
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
"""返回当前 UTC 时间的 ISO 格式字符串。"""
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def main(db_path: str, apply: bool = False) -> None:
|
||||
"""诊断(默认)或修复(--apply)僵尸批量任务。"""
|
||||
db = Database(Path(db_path))
|
||||
print(f"数据库: {db_path}\n")
|
||||
for job_id in ZOMBIE_JOBS:
|
||||
job = db.get_batch_job(job_id)
|
||||
if job is None:
|
||||
print(f" [跳过] {job_id}: 任务不存在")
|
||||
continue
|
||||
pending = sum(1 for v in db.list_batch_videos(job_id) if v["status"] == "PENDING")
|
||||
completed = sum(1 for v in db.list_batch_videos(job_id) if v["status"] == "COMPLETED")
|
||||
# 安全校验:只修复"COMPLETED 但仍有 PENDING"的僵尸状态;已正常完成的跳过。
|
||||
if job["status"] != "COMPLETED" or pending == 0:
|
||||
print(f" [跳过] {job_id}: status={job['status']}, PENDING={pending},非僵尸状态")
|
||||
continue
|
||||
print(f" [待修] {job_id}: status={job['status']} → QUEUED, "
|
||||
f"COMPLETED={completed}, PENDING={pending}")
|
||||
if apply:
|
||||
db.update_batch_job(
|
||||
job_id, status="QUEUED", progress=0, total=pending,
|
||||
done=completed, failed=int(job["failed"] or 0),
|
||||
current_video=None, error=None, updated_at=_now_iso(),
|
||||
)
|
||||
print(f" ✓ 已置回 QUEUED(引擎将续跑剩余 {pending} 个 PENDING)")
|
||||
if not apply:
|
||||
print("\n以上为预览。确认无误后加 --apply 实际修复。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(sys.argv[1] if len(sys.argv) > 1 else "data/wov.db", "--apply" in sys.argv)
|
||||
Reference in New Issue
Block a user