feat: 文件夹批量处理引擎(后端)
- BatchWorker 单线程轮询 batch_jobs 表,处理 source=batch 的运行, 与主调度器互不抢占(next_queued_run 排除 batch 来源) - 直接读取用户所选文件夹下的视频逐个执行流水线,不上传到工作目录; 中间态与产物落在视频旁同名文件夹,batch.done.json 完成标记去重 - 支持暂停/继续、失败容错(单视频失败不阻塞后续)、删除任务只清库 - 孤儿清理跳过 source=batch 运行,防止误删用户视频文件夹 - workflow_runs 新增 source 列(upload/batch),旧库自动迁移
This commit is contained in:
+1003
File diff suppressed because it is too large
Load Diff
@@ -329,3 +329,103 @@ def test_restore_run_outputs(tmp_path) -> None:
|
||||
"extract": {"frames_manifest": "frames.json"}
|
||||
}
|
||||
assert db.restore_run_outputs("run_none") == {}
|
||||
|
||||
|
||||
def test_run_source_column_default_and_next_queued(tmp_path) -> None:
|
||||
"""source 列默认 upload;主调度器不拾取 batch 来源的运行。"""
|
||||
db = Database(tmp_path / "wov.db")
|
||||
db.upsert_workflow({"id": "demo", "name": "Demo", "published": 1, "latest_version": 1})
|
||||
now = "2026-01-01T00:00:00+00:00"
|
||||
db.create_run(
|
||||
{
|
||||
"id": "run_upload",
|
||||
"workflow_id": "demo",
|
||||
"workflow_version": 1,
|
||||
"status": "QUEUED",
|
||||
"progress": 0,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
db.create_run(
|
||||
{
|
||||
"id": "run_batch",
|
||||
"workflow_id": "demo",
|
||||
"workflow_version": 1,
|
||||
"status": "QUEUED",
|
||||
"progress": 0,
|
||||
"source": "batch",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
# 默认 source 为 upload,可显式指定 batch。
|
||||
assert db.get_run("run_upload")["source"] == "upload"
|
||||
assert db.get_run("run_batch")["source"] == "batch"
|
||||
# 主调度器只取非 batch 运行,批量运行由批量引擎单独拾起。
|
||||
assert db.next_queued_run()["id"] == "run_upload"
|
||||
|
||||
|
||||
def test_db_migration_adds_source_column(tmp_path) -> None:
|
||||
"""旧库迁移:缺少 source 列的库打开后自动补列并默认 upload。"""
|
||||
import sqlite3
|
||||
|
||||
db_path = tmp_path / "old.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute(
|
||||
"CREATE TABLE workflow_runs (id TEXT PRIMARY KEY, workflow_id TEXT NOT NULL,"
|
||||
" workflow_version INTEGER NOT NULL, status TEXT NOT NULL, current_node_id TEXT,"
|
||||
" progress REAL NOT NULL DEFAULT 0, error TEXT, input_uri TEXT,"
|
||||
" param_overrides TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL)"
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
Database(db_path)
|
||||
conn = sqlite3.connect(db_path)
|
||||
columns = [row[1] for row in conn.execute("PRAGMA table_info(workflow_runs)")]
|
||||
conn.close()
|
||||
assert "source" in columns
|
||||
|
||||
|
||||
def test_batch_jobs_and_videos_crud(tmp_path) -> None:
|
||||
"""批量任务/视频明细的增删改查与排队查询。"""
|
||||
db = Database(tmp_path / "wov.db")
|
||||
now = "2026-01-01T00:00:00+00:00"
|
||||
db.create_batch_job(
|
||||
{
|
||||
"id": "batch_1", "folder_path": "/videos", "workflow_id": "demo",
|
||||
"recursive": 1, "status": "QUEUED", "progress": 0, "total": 2,
|
||||
"done": 0, "failed": 0, "current_video": None, "error": None,
|
||||
"created_at": now, "updated_at": now,
|
||||
}
|
||||
)
|
||||
assert db.next_queued_batch_job()["id"] == "batch_1"
|
||||
assert db.list_batch_job_ids() == ["batch_1"]
|
||||
assert db.list_batch_jobs()[0]["total"] == 2
|
||||
|
||||
db.create_batch_video(
|
||||
{
|
||||
"id": "bv_1", "job_id": "batch_1", "video_path": "/videos/a.mp4",
|
||||
"work_dir": "/videos/a", "run_id": None, "status": "PENDING",
|
||||
"error": None, "created_at": now, "updated_at": now,
|
||||
}
|
||||
)
|
||||
db.update_batch_video("bv_1", status="COMPLETED", updated_at=now)
|
||||
assert db.get_batch_video("bv_1")["status"] == "COMPLETED"
|
||||
assert len(db.list_batch_videos("batch_1")) == 1
|
||||
|
||||
# 未知字段更新被忽略(不会报错也不会改状态)。
|
||||
db.update_batch_video("bv_1", bogus=1, updated_at=now)
|
||||
db.update_batch_job("batch_1", bogus=1, updated_at=now)
|
||||
assert db.get_batch_video("bv_1")["status"] == "COMPLETED"
|
||||
|
||||
db.update_batch_job("batch_1", status="COMPLETED", done=1, failed=0, progress=1.0, updated_at=now)
|
||||
job = db.get_batch_job("batch_1")
|
||||
assert job["status"] == "COMPLETED" and job["done"] == 1
|
||||
# 完成后不再排队。
|
||||
assert db.next_queued_batch_job() is None
|
||||
|
||||
db.delete_batch_job("batch_1")
|
||||
assert db.get_batch_job("batch_1") is None
|
||||
assert db.get_batch_video("bv_1") is None
|
||||
|
||||
@@ -120,6 +120,38 @@ def test_cleaner_removes_run_with_empty_steps_dir(tmp_path) -> None:
|
||||
assert not steps.exists()
|
||||
|
||||
|
||||
def test_cleaner_skips_batch_runs(tmp_path) -> None:
|
||||
"""批量处理运行(source=batch)跳过清理:绝不删除用户视频文件夹。
|
||||
|
||||
批量 run 的产物在用户视频旁的同名文件夹里,不在主存储目录下;若按普通
|
||||
孤儿逻辑处理,_has_files 检查不到会误删记录,_remove_run 还会连带删除
|
||||
input_uri 的父目录(用户的整个视频文件夹)。
|
||||
"""
|
||||
db = _db(tmp_path)
|
||||
video_folder = tmp_path / "my_videos"
|
||||
video_folder.mkdir()
|
||||
video = video_folder / "a.mp4"
|
||||
video.write_bytes(b"real")
|
||||
# 过期、COMPLETED、主存储无任何产物文件——普通任务会被清理的条件全满足。
|
||||
db.upsert_workflow({"id": "flow", "name": "F", "published": 1, "latest_version": 1})
|
||||
db.create_run(
|
||||
{
|
||||
"id": "run_batch",
|
||||
"workflow_id": "flow",
|
||||
"workflow_version": 1,
|
||||
"status": "COMPLETED",
|
||||
"progress": 1,
|
||||
"input_uri": str(video),
|
||||
"source": "batch",
|
||||
"created_at": "2020-01-01T00:00:00+00:00",
|
||||
"updated_at": "2020-01-01T00:00:00+00:00",
|
||||
}
|
||||
)
|
||||
cleaner = OrphanCleaner(db, tmp_path / "storage", grace_seconds=3600)
|
||||
assert cleaner.clean_once() == 0
|
||||
assert db.get_run("run_batch") is not None
|
||||
assert video_folder.exists()
|
||||
assert video.exists()
|
||||
def test_cleaner_default_config_and_defensive_branches(tmp_path, monkeypatch) -> None:
|
||||
"""验证默认配置构造、缺失/非法时间与缺失任务记录的防御分支。"""
|
||||
db = _db(tmp_path)
|
||||
|
||||
Reference in New Issue
Block a user