feat: 文件夹批量处理引擎(后端)

- BatchWorker 单线程轮询 batch_jobs 表,处理 source=batch 的运行,
  与主调度器互不抢占(next_queued_run 排除 batch 来源)
- 直接读取用户所选文件夹下的视频逐个执行流水线,不上传到工作目录;
  中间态与产物落在视频旁同名文件夹,batch.done.json 完成标记去重
- 支持暂停/继续、失败容错(单视频失败不阻塞后续)、删除任务只清库
- 孤儿清理跳过 source=batch 运行,防止误删用户视频文件夹
- workflow_runs 新增 source 列(upload/batch),旧库自动迁移
This commit is contained in:
2026-08-23 16:25:09 +08:00
parent b16e0e9f3c
commit 711867e79f
10 changed files with 2056 additions and 9 deletions
+100
View File
@@ -329,3 +329,103 @@ def test_restore_run_outputs(tmp_path) -> None:
"extract": {"frames_manifest": "frames.json"}
}
assert db.restore_run_outputs("run_none") == {}
def test_run_source_column_default_and_next_queued(tmp_path) -> None:
"""source 列默认 upload;主调度器不拾取 batch 来源的运行。"""
db = Database(tmp_path / "wov.db")
db.upsert_workflow({"id": "demo", "name": "Demo", "published": 1, "latest_version": 1})
now = "2026-01-01T00:00:00+00:00"
db.create_run(
{
"id": "run_upload",
"workflow_id": "demo",
"workflow_version": 1,
"status": "QUEUED",
"progress": 0,
"created_at": now,
"updated_at": now,
}
)
db.create_run(
{
"id": "run_batch",
"workflow_id": "demo",
"workflow_version": 1,
"status": "QUEUED",
"progress": 0,
"source": "batch",
"created_at": now,
"updated_at": now,
}
)
# 默认 source 为 upload,可显式指定 batch。
assert db.get_run("run_upload")["source"] == "upload"
assert db.get_run("run_batch")["source"] == "batch"
# 主调度器只取非 batch 运行,批量运行由批量引擎单独拾起。
assert db.next_queued_run()["id"] == "run_upload"
def test_db_migration_adds_source_column(tmp_path) -> None:
"""旧库迁移:缺少 source 列的库打开后自动补列并默认 upload。"""
import sqlite3
db_path = tmp_path / "old.db"
conn = sqlite3.connect(db_path)
conn.execute(
"CREATE TABLE workflow_runs (id TEXT PRIMARY KEY, workflow_id TEXT NOT NULL,"
" workflow_version INTEGER NOT NULL, status TEXT NOT NULL, current_node_id TEXT,"
" progress REAL NOT NULL DEFAULT 0, error TEXT, input_uri TEXT,"
" param_overrides TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL)"
)
conn.commit()
conn.close()
Database(db_path)
conn = sqlite3.connect(db_path)
columns = [row[1] for row in conn.execute("PRAGMA table_info(workflow_runs)")]
conn.close()
assert "source" in columns
def test_batch_jobs_and_videos_crud(tmp_path) -> None:
"""批量任务/视频明细的增删改查与排队查询。"""
db = Database(tmp_path / "wov.db")
now = "2026-01-01T00:00:00+00:00"
db.create_batch_job(
{
"id": "batch_1", "folder_path": "/videos", "workflow_id": "demo",
"recursive": 1, "status": "QUEUED", "progress": 0, "total": 2,
"done": 0, "failed": 0, "current_video": None, "error": None,
"created_at": now, "updated_at": now,
}
)
assert db.next_queued_batch_job()["id"] == "batch_1"
assert db.list_batch_job_ids() == ["batch_1"]
assert db.list_batch_jobs()[0]["total"] == 2
db.create_batch_video(
{
"id": "bv_1", "job_id": "batch_1", "video_path": "/videos/a.mp4",
"work_dir": "/videos/a", "run_id": None, "status": "PENDING",
"error": None, "created_at": now, "updated_at": now,
}
)
db.update_batch_video("bv_1", status="COMPLETED", updated_at=now)
assert db.get_batch_video("bv_1")["status"] == "COMPLETED"
assert len(db.list_batch_videos("batch_1")) == 1
# 未知字段更新被忽略(不会报错也不会改状态)。
db.update_batch_video("bv_1", bogus=1, updated_at=now)
db.update_batch_job("batch_1", bogus=1, updated_at=now)
assert db.get_batch_video("bv_1")["status"] == "COMPLETED"
db.update_batch_job("batch_1", status="COMPLETED", done=1, failed=0, progress=1.0, updated_at=now)
job = db.get_batch_job("batch_1")
assert job["status"] == "COMPLETED" and job["done"] == 1
# 完成后不再排队。
assert db.next_queued_batch_job() is None
db.delete_batch_job("batch_1")
assert db.get_batch_job("batch_1") is None
assert db.get_batch_video("bv_1") is None