feat: 批量分块流水线、本地模型显存让渡与任务列表分工

批量引擎改为「分块流水线」:视频按 WOV_BATCH_STAGE_GROUP_SIZE(默认 8)分组,
组内按 DAG 拓扑序跑完全部视频(全部 extract → 全部 ASR → 全部翻译 → 全部 ASS)
再进入下一组,本地模型每组只加载一次、卸载一次,而不是每个视频来回加载卸载;
产物仍按组增量落到视频旁。调度器新增 execute_run(run_id, stop_after=节点):
该节点完成后任务保持 RUNNING 不收尾,下一次调用从产物表跳过已完成节点继续,
用于实现阶段边界。

- nodes/llm.py:翻译节点结束释放本机 Ollama 显存(node 参数 unload_after >
  LLM_UNLOAD_AFTER > 本机 loopback 端点默认卸载,云端端点不卸载;卸载失败只告警),
  新增 keep_model.flag 语义(阶段内保持常驻)与 release_local_model();
  新增节点内暂停(按批 20 行检查 paused.flag,抛 PauseRequested,调度器保持 PAUSED)。
- src/wov_app/batch.py:分组阶段执行与阶段末统一释放显存;失败视频只在它失败
  节点的那个阶段重试(避免 LLM 已常驻时重跑 ASR 抢显存);任务没有明细时保持
  QUEUED 等登记完成、仍有未完成视频时置回 QUEUED 自愈(原先留 RUNNING 会卡死:
  引擎只拾取 QUEUED,任务停在“运行中但没人推进”);无失败视频时删除任务级空目录;
  每个阶段开始前清理 paused.flag / keep_model.flag,避免强杀残留影响后续阶段。
- src/wov_app/config.py:新增 WOV_BATCH_STAGE_GROUP_SIZE(设为 1 即旧的每视频全链路)。
- 任务列表与批量页分工:GET /api/runs 默认排除 source=batch(一个批量任务会产生
  N 条单视频 run,会把 20 条窗口占满;且任务管理页的暂停/重试/删除对批量 run
  语义不成立),需要排查时用 include_batch=1;作为补偿批量页详情新增阶段列
  (阶段 i/N · 中文标签,由该视频 run 的 current_node_id 在 DAG 拓扑序中的位置
  推导,节点类型映射中文标签)。阶段只有节点边界粒度,句级进度不落库、只在日志。
- 顺带纳入此前未提交的批量僵尸状态恢复:recover_interrupted_batch_jobs 除 RUNNING
  外也把「COMPLETED 但仍含未结束视频」的任务置回 QUEUED;fix_zombie_batch_jobs.py
  改为按条件扫描并支持 --apply 预览;批量页明细只列本批真正处理过的视频。

测试新增/更新:分块流水线调用顺序(组内按节点跑完再下一组)、每组只释放一次模型、
阶段内保持常驻标志、翻译按批暂停、失败视频不跨阶段推进、任务无明细/中途登记视频时
置回 QUEUED、任务工作空间与残留信号清理、任务列表默认过滤批量 run、详情阶段字段、
前端阶段列渲染;全量 507 passed(唯一失败为既有素材缺失的 integration 用例)。
This commit is contained in:
2026-09-18 10:31:52 +08:00
parent 7a7212f70c
commit dcdc5e8604
25 changed files with 1606 additions and 192 deletions
+337 -1
View File
@@ -8,12 +8,14 @@
from __future__ import annotations
import json
from contextlib import contextmanager
from pathlib import Path
import pytest
from wov_app import registry
from wov_app.batch import (
KEEP_MODEL_FLAG,
MARKER_NAME,
SUBTITLE_EXTENSIONS,
VIDEO_EXTENSIONS,
@@ -26,7 +28,7 @@ from wov_app.batch import (
scan_videos,
)
from wov_app.db import Database
from wov_sdk.models import WorkflowDefinition
from wov_sdk.models import InvokeRequest, InvokeResponse, WorkflowDefinition
@pytest.fixture(autouse=True)
@@ -70,6 +72,78 @@ def _published_db(tmp_path: Path, workflow_id: str = "wf") -> Database:
return db
def _staged_definition() -> WorkflowDefinition:
"""三节点分阶段链路:prep(echo) → translate(llm) → post(echo),产物为 srt。"""
return WorkflowDefinition.from_dict({
"name": "分阶段流程",
"version": 1,
"nodes": [
{"id": "prep", "node_type": "echo", "params": {"node_tag": "prep"},
"inputs": {"file_uri": "input.video_uri"}},
{"id": "translate", "node_type": "llm-translate", "params": {"node_tag": "translate"},
"inputs": {"file_uri": "prep.file_uri"}},
{"id": "post", "node_type": "echo", "params": {"node_tag": "post"},
"inputs": {"file_uri": "translate.file_uri"}},
],
"edges": [{"from": "prep", "to": "translate"}, {"from": "translate", "to": "post"}],
"entry_inputs": {"video_uri": "file"},
"final_outputs": {"cn_srt": "post.file_uri"},
})
def _staged_db(tmp_path: Path, workflow_id: str = "wf") -> Database:
"""建好已发布的三节点分阶段工作流库。"""
db = Database(tmp_path / "wov.db")
db.upsert_workflow({
"id": workflow_id, "name": "分阶段流程", "description": "", "published": 1,
"latest_version": 1,
})
db.create_workflow_version(workflow_id, 1, _staged_definition().to_dict())
return db
@contextmanager
def _recording_nodes(
trace: list[tuple[str, str]],
llm_flag_state: list[bool] | None = None,
fail_stage: tuple[str, str] | None = None,
flag_trace: list[tuple[str, bool]] | None = None,
):
"""把 echo / llm-translate 节点换成记录调用顺序的假节点(覆盖真实注册表条目)。
假节点把上游传来的视频名写成 payload.srt 透传给下一节点,因此每个阶段都
知道自己在处理哪个视频;记录 (节点标签, 视频名),llm 节点额外记录
keep_model.flag 是否存在;fail_stage 指定的 (标签, 视频名) 组合返回失败。
"""
registry.register_all()
def handler(request: InvokeRequest) -> InvokeResponse:
tag = str(request.params.get("node_tag"))
source = str(request.inputs.get("file_uri") or "")
if source and Path(source).suffix.lower() in VIDEO_EXTENSIONS:
# 首阶段的输入就是视频文件,后续阶段拿到的是上一阶段的 payload。
video_name = Path(source).name
else:
video_name = Path(source).read_text(encoding="utf-8").strip() if source else ""
trace.append((tag, video_name))
run_root = Path(request.output_dir).parent.parent
if llm_flag_state is not None and tag == "translate":
llm_flag_state.append((run_root / KEEP_MODEL_FLAG).exists())
if flag_trace is not None:
flag_trace.append((tag, (run_root / KEEP_MODEL_FLAG).exists()))
if fail_stage == (tag, video_name):
return InvokeResponse(status="failed", error="模拟阶段失败")
output_dir = Path(request.output_dir)
output_dir.mkdir(parents=True, exist_ok=True)
output = output_dir / "payload.srt"
output.write_text(video_name, encoding="utf-8")
return InvokeResponse(status="completed", outputs={"file_uri": str(output)})
for node_type in ("echo", "llm-translate"):
registry.register(registry.get_node(node_type), handler)
yield
# ---------------------------------------------------------------------------
# 扫描与旁挂字幕判定
# ---------------------------------------------------------------------------
@@ -412,3 +486,265 @@ def test_worker_start_stop_idempotent(tmp_path: Path) -> None:
# 验证结果
assert first is second
assert worker._thread is None
def test_worker_processes_recovered_zombie_job(tmp_path: Path, monkeypatch) -> None:
"""僵尸任务(COMPLETED 但明细仍 PENDING)被恢复后能真正处理完剩余视频。
曾出现「任务已完成、视频仍未处理」的僵尸状态(完成标记先于视频收尾写出),
而引擎只拾取 QUEUED:不恢复就永远不会再处理那个视频。
"""
# 数据:已发布工作流 + 一个视频;创建任务后伪造成僵尸状态。
folder = tmp_path / "videos"
_make_video(folder / "movie.mp4")
db = _published_db(tmp_path)
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
registry.register_all()
job_id = create_job(db, str(folder), "wf")
db.update_batch_job(job_id, status="COMPLETED", progress=1.0,
updated_at="2026-09-01T00:00:00+00:00")
# 测试过程:重启恢复把僵尸任务放回队列,引擎拾起后处理剩余视频。
db.recover_interrupted_batch_jobs("2026-09-01T01:00:00+00:00")
worker = BatchWorker(db, interval_seconds=999)
worker._process_job(db.get_batch_job(job_id))
# 验证结果:视频真的处理完、产物放到视频旁、任务保持完成。
item = db.list_batch_videos(job_id)[0]
assert item["status"] == "COMPLETED"
assert db.get_batch_job(job_id)["status"] == "COMPLETED"
assert list(folder.glob("movie.*")), "应在视频旁放置最终产物"
# ---------------------------------------------------------------------------
# 引擎:分块流水线(阶段化执行)
# ---------------------------------------------------------------------------
def test_worker_runs_grouped_stage_pipeline(tmp_path: Path, monkeypatch) -> None:
"""分块流水线:组内按节点顺序跑完全部视频,而不是每个视频跑完整链路。"""
# 数据:3 个视频 + 三节点链路,分组大小 2(前两个一组、第三个一组)。
folder = tmp_path / "videos"
for name in ("a.mp4", "b.mp4", "c.mp4"):
_make_video(folder / name)
db = _staged_db(tmp_path)
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
monkeypatch.setattr("wov_app.batch.BATCH_STAGE_GROUP_SIZE", 2)
trace: list[tuple[str, str]] = []
# 测试过程:用记录调用顺序的假节点驱动引擎跑一轮。
with _recording_nodes(trace):
job_id = create_job(db, str(folder), "wf")
BatchWorker(db, interval_seconds=999)._process_job(db.get_batch_job(job_id))
# 验证结果:组1 三个阶段各跑 a/b,再轮到组2 的 c。
assert trace == [
("prep", "a.mp4"), ("prep", "b.mp4"),
("translate", "a.mp4"), ("translate", "b.mp4"),
("post", "a.mp4"), ("post", "b.mp4"),
("prep", "c.mp4"), ("translate", "c.mp4"), ("post", "c.mp4"),
]
# 三个视频都完成且产物按约定名落到视频旁。
assert db.get_batch_job(job_id)["status"] == "COMPLETED"
assert sorted(p.name for p in folder.glob("*.srt")) == ["a.CN.srt", "b.CN.srt", "c.CN.srt"]
def test_worker_releases_local_llm_once_per_group(tmp_path: Path, monkeypatch) -> None:
"""LLM 阶段结束由引擎统一释放显存:每组一次,而不是每个视频一次。"""
# 数据:3 个视频 + 分组 2,记录释放调用与 LLM 调用时的常驻信号状态。
folder = tmp_path / "videos"
for name in ("a.mp4", "b.mp4", "c.mp4"):
_make_video(folder / name)
db = _staged_db(tmp_path)
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
monkeypatch.setattr("wov_app.batch.BATCH_STAGE_GROUP_SIZE", 2)
releases: list[str | None] = []
monkeypatch.setattr(
"wov_app.batch.release_local_model",
lambda model=None: releases.append(model),
)
trace: list[tuple[str, str]] = []
flag_state: list[bool] = []
# 测试过程
with _recording_nodes(trace, llm_flag_state=flag_state):
job_id = create_job(db, str(folder), "wf")
BatchWorker(db, interval_seconds=999)._process_job(db.get_batch_job(job_id))
# 验证结果:两组各释放一次;每次 LLM 调用都在“保持常驻”信号下执行;信号已清理。
assert releases == [None, None]
assert flag_state == [True, True, True]
assert not list((tmp_path / "storage").rglob(KEEP_MODEL_FLAG))
def test_worker_defers_video_failed_in_earlier_stage(tmp_path: Path, monkeypatch) -> None:
"""上一阶段失败的视频不在后续阶段重跑(避免 LLM 已常驻时重跑 ASR 抢显存)。"""
# 数据:2 个视频(同一组)+ 三节点链路,prep 阶段让 a 失败。
folder = tmp_path / "videos"
for name in ("a.mp4", "b.mp4"):
_make_video(folder / name)
db = _staged_db(tmp_path)
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
monkeypatch.setattr("wov_app.batch.BATCH_STAGE_GROUP_SIZE", 2)
monkeypatch.setattr("wov_app.batch.release_local_model", lambda model=None: None)
trace: list[tuple[str, str]] = []
# 测试过程
with _recording_nodes(trace, fail_stage=("prep", "a.mp4")):
job_id = create_job(db, str(folder), "wf")
BatchWorker(db, interval_seconds=999)._process_job(db.get_batch_job(job_id))
# 验证结果:a 只在 prep 出现一次并记为 FAILED;b 三阶段跑完并落地产物。
assert [entry for entry in trace if entry[1] == "a.mp4"] == [("prep", "a.mp4")]
videos = {Path(v["video_path"]).name: v for v in db.list_batch_videos(job_id)}
assert videos["a.mp4"]["status"] == "FAILED"
assert videos["b.mp4"]["status"] == "COMPLETED"
assert (folder / "b.CN.srt").is_file()
assert not (folder / "a.CN.srt").exists()
assert db.get_batch_job(job_id)["failed"] == 1
# ---------------------------------------------------------------------------
# 引擎:创建期间拾起任务(明细未登记完)的自愈
# ---------------------------------------------------------------------------
def test_worker_requeues_job_without_details(tmp_path: Path, monkeypatch) -> None:
"""任务行先于明细写入:拾起到无明细的任务时保持 QUEUED,不按空任务收尾。"""
# 数据:只有任务行、还没写任何视频明细的批量任务。
db = _published_db(tmp_path)
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
db.create_batch_job({
"id": "batch-registering", "folder_path": str(tmp_path), "workflow_id": "wf",
"recursive": 1, "status": "QUEUED", "progress": 0, "total": 0, "done": 0,
"failed": 0, "current_video": None, "error": None,
"created_at": "2026-09-01T00:00:00+00:00", "updated_at": "2026-09-01T00:00:00+00:00",
})
# 测试过程
worker = BatchWorker(db, interval_seconds=999)
worker._process_job(db.get_batch_job("batch-registering"))
# 验证结果:任务仍在排队等待登记完成,而不是被标成 COMPLETED。
assert db.get_batch_job("batch-registering")["status"] == "QUEUED"
assert db.next_queued_batch_job() is not None
def test_worker_requeues_job_when_video_registered_mid_pass(tmp_path: Path, monkeypatch) -> None:
"""明细在引擎处理中途才登记进来:本轮结束后置回 QUEUED,下一轮续跑完成。"""
# 数据:1 个视频 + 单节点工作流;处理首个阶段时登记第二个视频(模拟创建中拾起)。
folder = tmp_path / "videos"
_make_video(folder / "first.mp4")
db = _published_db(tmp_path)
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
registry.register_all()
job_id = create_job(db, str(folder), "wf")
worker = BatchWorker(db, interval_seconds=999)
original_stage = worker._run_stage
injected = {"done": False}
def stage_with_late_video(*args, **kwargs):
# 模拟 create_job 仍在写明细:引擎快照之后新视频才出现在数据库里。
if not injected["done"]:
injected["done"] = True
late_video = _make_video(folder / "second.mp4")
db.create_batch_video({
"id": "bv_late", "job_id": job_id, "video_path": str(late_video),
"work_dir": str(tmp_path / "storage" / "batch" / job_id / "bv_late"),
"run_id": None, "status": "PENDING", "error": None,
"created_at": "2026-09-01T00:00:01+00:00", "updated_at": "2026-09-01T00:00:01+00:00",
})
return original_stage(*args, **kwargs)
monkeypatch.setattr(worker, "_run_stage", stage_with_late_video)
# 测试过程:第一轮只看到 first.mp4。
worker._process_job(db.get_batch_job(job_id))
# 验证结果:job 被置回 QUEUED 等待下一轮,新视频还没被处理。
assert db.get_batch_job(job_id)["status"] == "QUEUED"
statuses = {Path(v["video_path"]).name: v["status"] for v in db.list_batch_videos(job_id)}
assert statuses == {"first.mp4": "COMPLETED", "second.mp4": "PENDING"}
# 测试过程:下一轮引擎拾起后处理剩余视频并收尾。
second = BatchWorker(db, interval_seconds=999)
second._process_job(db.get_batch_job(job_id))
# 验证结果:两个视频都完成、任务完成、产物都在视频旁(真实 echo 节点产物为 echo.txt)。
statuses = {Path(v["video_path"]).name: v["status"] for v in db.list_batch_videos(job_id)}
assert statuses == {"first.mp4": "COMPLETED", "second.mp4": "COMPLETED"}
assert db.get_batch_job(job_id)["status"] == "COMPLETED"
# 真实 echo 节点的最终产物保留原扩展名(非 .srt/.ass),按视频主名放置。
assert list(folder.glob("first.*")) and list(folder.glob("second.*"))
def test_worker_removes_empty_job_workspace_after_completion(tmp_path: Path, monkeypatch) -> None:
"""任务全部完成后删掉任务级工作空间目录(每视频工作空间已各自清理)。"""
# 数据:一个视频的批量任务。
folder = tmp_path / "videos"
_make_video(folder / "movie.mp4")
db = _published_db(tmp_path)
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
registry.register_all()
job_id = create_job(db, str(folder), "wf")
# 测试过程
BatchWorker(db, interval_seconds=999)._process_job(db.get_batch_job(job_id))
# 验证结果:任务完成,任务级目录(收尾后只剩空壳)被删除。
assert db.get_batch_job(job_id)["status"] == "COMPLETED"
assert not (tmp_path / "storage" / "batch" / job_id).exists()
def test_worker_keeps_job_workspace_when_video_failed(tmp_path: Path, monkeypatch) -> None:
"""有失败视频时保留任务工作空间(失败视频的中间产物供断点重试)。"""
# 数据:三节点链路,prep 阶段让视频失败。
folder = tmp_path / "videos"
_make_video(folder / "a.mp4")
db = _staged_db(tmp_path)
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
monkeypatch.setattr("wov_app.batch.BATCH_STAGE_GROUP_SIZE", 2)
monkeypatch.setattr("wov_app.batch.release_local_model", lambda model=None: None)
# 测试过程
with _recording_nodes([], fail_stage=("prep", "a.mp4")):
job_id = create_job(db, str(folder), "wf")
BatchWorker(db, interval_seconds=999)._process_job(db.get_batch_job(job_id))
# 验证结果:视频失败、任务工作空间仍在(可重试)。
assert db.get_batch_job(job_id)["failed"] == 1
assert (tmp_path / "storage" / "batch" / job_id).exists()
def test_worker_clears_stale_keep_model_flag_before_stage(tmp_path: Path, monkeypatch) -> None:
"""强杀残留的 keep_model.flag 不会带到后续阶段:非 LLM 节点不应看到它。"""
# 数据:两节点链路 + 已存在的 run(工作空间里残留强杀时的 keep_model.flag)。
folder = tmp_path / "videos"
_make_video(folder / "a.mp4")
db = _staged_db(tmp_path)
work_root = tmp_path / "storage" / "batch"
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", work_root)
monkeypatch.setattr("wov_app.batch.BATCH_STAGE_GROUP_SIZE", 8)
monkeypatch.setattr("wov_app.batch.release_local_model", lambda model=None: None)
job_id = create_job(db, str(folder), "wf")
video = db.list_batch_videos(job_id)[0]
run_id = "run_stale_flag"
db.update_batch_video(video["id"], run_id=run_id, updated_at="2026-09-01T00:00:00+00:00")
db.create_run({
"id": run_id, "workflow_id": "wf", "workflow_version": 1, "status": "QUEUED",
"current_node_id": None, "progress": 0.0, "error": None,
"input_uri": str(folder / "a.mp4"), "param_overrides": None, "source": "batch",
"created_at": "2026-09-01T00:00:00+00:00", "updated_at": "2026-09-01T00:00:00+00:00",
})
run_dir = Path(video["work_dir"]) / "runs" / run_id
run_dir.mkdir(parents=True, exist_ok=True)
(run_dir / KEEP_MODEL_FLAG).write_text("", encoding="utf-8")
flag_trace: list[tuple[str, bool]] = []
# 测试过程
with _recording_nodes([], flag_trace=flag_trace):
BatchWorker(db, interval_seconds=999)._process_job(db.get_batch_job(job_id))
# 验证结果:prep(非 LLM)看不到残留标志;translate(LLM 阶段)才写入;
# post(非 LLM)不再看到它。
assert flag_trace == [("prep", False), ("translate", True), ("post", False)]