feat: 文件夹批量处理引擎(后端)

- BatchWorker 单线程轮询 batch_jobs 表,处理 source=batch 的运行,
  与主调度器互不抢占(next_queued_run 排除 batch 来源)
- 直接读取用户所选文件夹下的视频逐个执行流水线,不上传到工作目录;
  中间态与产物落在视频旁同名文件夹,batch.done.json 完成标记去重
- 支持暂停/继续、失败容错(单视频失败不阻塞后续)、删除任务只清库
- 孤儿清理跳过 source=batch 运行,防止误删用户视频文件夹
- workflow_runs 新增 source 列(upload/batch),旧库自动迁移
This commit is contained in:
2026-08-23 16:25:09 +08:00
parent b16e0e9f3c
commit 711867e79f
10 changed files with 2056 additions and 9 deletions
+32
View File
@@ -120,6 +120,38 @@ def test_cleaner_removes_run_with_empty_steps_dir(tmp_path) -> None:
assert not steps.exists()
def test_cleaner_skips_batch_runs(tmp_path) -> None:
"""批量处理运行(source=batch)跳过清理:绝不删除用户视频文件夹。
批量 run 的产物在用户视频旁的同名文件夹里,不在主存储目录下;若按普通
孤儿逻辑处理,_has_files 检查不到会误删记录,_remove_run 还会连带删除
input_uri 的父目录(用户的整个视频文件夹)。
"""
db = _db(tmp_path)
video_folder = tmp_path / "my_videos"
video_folder.mkdir()
video = video_folder / "a.mp4"
video.write_bytes(b"real")
# 过期、COMPLETED、主存储无任何产物文件——普通任务会被清理的条件全满足。
db.upsert_workflow({"id": "flow", "name": "F", "published": 1, "latest_version": 1})
db.create_run(
{
"id": "run_batch",
"workflow_id": "flow",
"workflow_version": 1,
"status": "COMPLETED",
"progress": 1,
"input_uri": str(video),
"source": "batch",
"created_at": "2020-01-01T00:00:00+00:00",
"updated_at": "2020-01-01T00:00:00+00:00",
}
)
cleaner = OrphanCleaner(db, tmp_path / "storage", grace_seconds=3600)
assert cleaner.clean_once() == 0
assert db.get_run("run_batch") is not None
assert video_folder.exists()
assert video.exists()
def test_cleaner_default_config_and_defensive_branches(tmp_path, monkeypatch) -> None:
"""验证默认配置构造、缺失/非法时间与缺失任务记录的防御分支。"""
db = _db(tmp_path)