feat: 文件夹批量处理引擎（后端）
- BatchWorker 单线程轮询 batch_jobs 表，处理 source=batch 的运行， 与主调度器互不抢占（next_queued_run 排除 batch 来源） - 直接读取用户所选文件夹下的视频逐个执行流水线，不上传到工作目录； 中间态与产物落在视频旁同名文件夹，batch.done.json 完成标记去重 - 支持暂停/继续、失败容错（单视频失败不阻塞后续）、删除任务只清库 - 孤儿清理跳过 source=batch 运行，防止误删用户视频文件夹 - workflow_runs 新增 source 列（upload/batch），旧库自动迁移
This commit is contained in:
+199
-6
@@ -73,6 +73,7 @@ class Database:
|
||||
error TEXT,
|
||||
input_uri TEXT,
|
||||
param_overrides TEXT,
|
||||
source TEXT NOT NULL DEFAULT 'upload',
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
FOREIGN KEY(workflow_id) REFERENCES workflows(id)
|
||||
@@ -91,6 +92,39 @@ class Database:
|
||||
UNIQUE(run_id, name),
|
||||
FOREIGN KEY(run_id) REFERENCES workflow_runs(id)
|
||||
);
|
||||
|
||||
-- 批量处理任务表：一次"文件夹批量处理"对应一条记录，记录目标
|
||||
-- 文件夹、所选工作流与整体状态。批量引擎与 Web 页面共用。
|
||||
CREATE TABLE IF NOT EXISTS batch_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
folder_path TEXT NOT NULL,
|
||||
workflow_id TEXT NOT NULL,
|
||||
recursive INTEGER NOT NULL DEFAULT 1,
|
||||
status TEXT NOT NULL,
|
||||
progress REAL NOT NULL DEFAULT 0,
|
||||
total INTEGER NOT NULL DEFAULT 0,
|
||||
done INTEGER NOT NULL DEFAULT 0,
|
||||
failed INTEGER NOT NULL DEFAULT 0,
|
||||
current_video TEXT,
|
||||
error TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- 批量视频明细表：一次批量任务处理的每个视频一条记录，保存其
|
||||
-- 对应的工作流 run（断点续跑复用 workflow_runs 的产物状态）。
|
||||
CREATE TABLE IF NOT EXISTS batch_videos (
|
||||
id TEXT PRIMARY KEY,
|
||||
job_id TEXT NOT NULL,
|
||||
video_path TEXT NOT NULL,
|
||||
work_dir TEXT NOT NULL,
|
||||
run_id TEXT,
|
||||
status TEXT NOT NULL,
|
||||
error TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
FOREIGN KEY(job_id) REFERENCES batch_jobs(id)
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
@@ -101,7 +135,10 @@ class Database:
|
||||
]
|
||||
if "param_overrides" not in columns:
|
||||
conn.execute("ALTER TABLE workflow_runs ADD COLUMN param_overrides TEXT")
|
||||
|
||||
# 旧库迁移：workflow_runs 补充 source 列（upload=网页上传 / batch=批量处理），
|
||||
# 批量引擎与主调度器据此隔离任务，避免互相抢占。
|
||||
if "source" not in columns:
|
||||
conn.execute("ALTER TABLE workflow_runs ADD COLUMN source TEXT NOT NULL DEFAULT 'upload'")
|
||||
def upsert_workflow(self, workflow: dict[str, Any]) -> None:
|
||||
"""插入或更新工作流概要信息。"""
|
||||
with self._connect() as conn:
|
||||
@@ -210,15 +247,20 @@ class Database:
|
||||
return versions
|
||||
|
||||
def create_run(self, run: dict[str, Any]) -> None:
|
||||
"""创建一条排队中的工作流运行记录。"""
|
||||
"""创建一条排队中的工作流运行记录。
|
||||
|
||||
source 标识任务来源：upload（网页上传，默认）由主调度器执行；
|
||||
batch（文件夹批量处理）由批量引擎执行，input_uri 直接指向本地视频。
|
||||
"""
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO workflow_runs (
|
||||
id, workflow_id, workflow_version, status, current_node_id,
|
||||
progress, error, input_uri, param_overrides, created_at, updated_at
|
||||
progress, error, input_uri, param_overrides, source,
|
||||
created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
run["id"],
|
||||
@@ -232,11 +274,11 @@ class Database:
|
||||
json.dumps(run["param_overrides"], ensure_ascii=False)
|
||||
if run.get("param_overrides")
|
||||
else None,
|
||||
run.get("source", "upload"),
|
||||
run["created_at"],
|
||||
run["updated_at"],
|
||||
),
|
||||
)
|
||||
|
||||
def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
"""按 ID 读取任务运行记录。"""
|
||||
with self._connect() as conn:
|
||||
@@ -308,12 +350,14 @@ class Database:
|
||||
只取 QUEUED：PAUSED 任务必须由用户显式 resume（转回 QUEUED）后调度器
|
||||
才重新执行。修复回归——此前把 PAUSED 也当可执行任务拾起，execute_run
|
||||
会先置 RUNNING 再检查暂停，导致"点击暂停反而开始任务"。
|
||||
同时排除 source=batch 的批量运行：批量任务由批量引擎使用视频旁的
|
||||
同名文件夹作为 storage 执行，主调度器拾起会用错存储目录。
|
||||
"""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM workflow_runs
|
||||
WHERE status = 'QUEUED'
|
||||
WHERE status = 'QUEUED' AND source != 'batch'
|
||||
ORDER BY created_at ASC
|
||||
LIMIT 1
|
||||
"""
|
||||
@@ -416,3 +460,152 @@ class Database:
|
||||
with self._connect() as conn:
|
||||
conn.execute("DELETE FROM artifacts WHERE run_id = ?", (run_id,))
|
||||
conn.execute("DELETE FROM workflow_runs WHERE id = ?", (run_id,))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# 批量处理任务（batch_jobs / batch_videos）数据访问。
|
||||
# 批量引擎与批量管理页共用这些方法，规则与 workflow_runs 一致。
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def create_batch_job(self, job: dict[str, Any]) -> None:
|
||||
"""插入一条批量处理任务记录。"""
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO batch_jobs (
|
||||
id, folder_path, workflow_id, recursive, status, progress,
|
||||
total, done, failed, current_video, error, created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
job["id"],
|
||||
job["folder_path"],
|
||||
job["workflow_id"],
|
||||
int(job.get("recursive", 1)),
|
||||
job["status"],
|
||||
float(job.get("progress", 0)),
|
||||
int(job.get("total", 0)),
|
||||
int(job.get("done", 0)),
|
||||
int(job.get("failed", 0)),
|
||||
job.get("current_video"),
|
||||
job.get("error"),
|
||||
job["created_at"],
|
||||
job["updated_at"],
|
||||
),
|
||||
)
|
||||
|
||||
def get_batch_job(self, job_id: str) -> dict[str, Any] | None:
|
||||
"""按 ID 读取批量任务记录。"""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute("SELECT * FROM batch_jobs WHERE id = ?", (job_id,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def list_batch_jobs(self, limit: int = 50) -> list[dict[str, Any]]:
|
||||
"""按创建时间倒序返回最近的批量任务。"""
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM batch_jobs ORDER BY created_at DESC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def list_batch_job_ids(self) -> list[str]:
|
||||
"""返回全部批量任务 ID，供孤儿清理区分批量运行使用。"""
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute("SELECT id FROM batch_jobs").fetchall()
|
||||
return [row["id"] for row in rows]
|
||||
|
||||
def next_queued_batch_job(self) -> dict[str, Any] | None:
|
||||
"""按创建时间返回最早一条排队（QUEUED）的批量任务。
|
||||
|
||||
批量引擎单线程顺序处理，同一时刻只执行一个批量任务。
|
||||
"""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM batch_jobs
|
||||
WHERE status = 'QUEUED'
|
||||
ORDER BY created_at ASC
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def update_batch_job(self, job_id: str, **fields: Any) -> None:
|
||||
"""更新批量任务字段，同时刷新 updated_at；未知字段会被忽略。"""
|
||||
allowed = {
|
||||
"status",
|
||||
"progress",
|
||||
"total",
|
||||
"done",
|
||||
"failed",
|
||||
"current_video",
|
||||
"error",
|
||||
}
|
||||
updates = {key: value for key, value in fields.items() if key in allowed}
|
||||
if not updates:
|
||||
return
|
||||
updates["updated_at"] = fields.get("updated_at")
|
||||
assignments = ", ".join(f"{key} = ?" for key in updates)
|
||||
values = list(updates.values()) + [job_id]
|
||||
with self._connect() as conn:
|
||||
conn.execute(f"UPDATE batch_jobs SET {assignments} WHERE id = ?", values)
|
||||
|
||||
def delete_batch_job(self, job_id: str) -> None:
|
||||
"""删除批量任务记录及其全部视频明细（不含 workflow_runs）。"""
|
||||
with self._connect() as conn:
|
||||
conn.execute("DELETE FROM batch_videos WHERE job_id = ?", (job_id,))
|
||||
conn.execute("DELETE FROM batch_jobs WHERE id = ?", (job_id,))
|
||||
|
||||
def create_batch_video(self, item: dict[str, Any]) -> None:
|
||||
"""插入一条批量视频明细记录。"""
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO batch_videos (
|
||||
id, job_id, video_path, work_dir, run_id, status, error,
|
||||
created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
item["id"],
|
||||
item["job_id"],
|
||||
item["video_path"],
|
||||
item["work_dir"],
|
||||
item.get("run_id"),
|
||||
item["status"],
|
||||
item.get("error"),
|
||||
item["created_at"],
|
||||
item["updated_at"],
|
||||
),
|
||||
)
|
||||
|
||||
def get_batch_video(self, video_id: str) -> dict[str, Any] | None:
|
||||
"""按 ID 读取批量视频明细。"""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM batch_videos WHERE id = ?", (video_id,)
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def list_batch_videos(self, job_id: str) -> list[dict[str, Any]]:
|
||||
"""按创建时间返回一次批量任务的全部视频明细。"""
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM batch_videos WHERE job_id = ? ORDER BY created_at",
|
||||
(job_id,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def update_batch_video(self, video_id: str, **fields: Any) -> None:
|
||||
"""更新批量视频字段，同时刷新 updated_at；未知字段会被忽略。"""
|
||||
allowed = {"run_id", "status", "error"}
|
||||
updates = {key: value for key, value in fields.items() if key in allowed}
|
||||
if not updates:
|
||||
return
|
||||
updates["updated_at"] = fields.get("updated_at")
|
||||
assignments = ", ".join(f"{key} = ?" for key in updates)
|
||||
values = list(updates.values()) + [video_id]
|
||||
with self._connect() as conn:
|
||||
conn.execute(f"UPDATE batch_videos SET {assignments} WHERE id = ?", values)
|
||||
|
||||
Reference in New Issue
Block a user