cat-shark
dcdc5e8604
批量引擎改为「分块流水线」：视频按 WOV_BATCH_STAGE_GROUP_SIZE（默认 8）分组， 组内按 DAG 拓扑序跑完全部视频（全部 extract → 全部 ASR → 全部翻译 → 全部 ASS） 再进入下一组，本地模型每组只加载一次、卸载一次，而不是每个视频来回加载卸载； 产物仍按组增量落到视频旁。调度器新增 execute_run(run_id, stop_after=节点)： 该节点完成后任务保持 RUNNING 不收尾，下一次调用从产物表跳过已完成节点继续， 用于实现阶段边界。 - nodes/llm.py：翻译节点结束释放本机 Ollama 显存（node 参数 unload_after > LLM_UNLOAD_AFTER > 本机 loopback 端点默认卸载，云端端点不卸载；卸载失败只告警）， 新增 keep_model.flag 语义（阶段内保持常驻）与 release_local_model()； 新增节点内暂停（按批 20 行检查 paused.flag，抛 PauseRequested，调度器保持 PAUSED）。 - src/wov_app/batch.py：分组阶段执行与阶段末统一释放显存；失败视频只在它失败 节点的那个阶段重试（避免 LLM 已常驻时重跑 ASR 抢显存）；任务没有明细时保持 QUEUED 等登记完成、仍有未完成视频时置回 QUEUED 自愈（原先留 RUNNING 会卡死： 引擎只拾取 QUEUED，任务停在“运行中但没人推进”）；无失败视频时删除任务级空目录； 每个阶段开始前清理 paused.flag / keep_model.flag，避免强杀残留影响后续阶段。 - src/wov_app/config.py：新增 WOV_BATCH_STAGE_GROUP_SIZE（设为 1 即旧的每视频全链路）。 - 任务列表与批量页分工：GET /api/runs 默认排除 source=batch（一个批量任务会产生 N 条单视频 run，会把 20 条窗口占满；且任务管理页的暂停/重试/删除对批量 run 语义不成立），需要排查时用 include_batch=1；作为补偿批量页详情新增阶段列 （阶段 i/N · 中文标签，由该视频 run 的 current_node_id 在 DAG 拓扑序中的位置 推导，节点类型映射中文标签）。阶段只有节点边界粒度，句级进度不落库、只在日志。 - 顺带纳入此前未提交的批量僵尸状态恢复：recover_interrupted_batch_jobs 除 RUNNING 外也把「COMPLETED 但仍含未结束视频」的任务置回 QUEUED；fix_zombie_batch_jobs.py 改为按条件扫描并支持 --apply 预览；批量页明细只列本批真正处理过的视频。 测试新增/更新：分块流水线调用顺序（组内按节点跑完再下一组）、每组只释放一次模型、 阶段内保持常驻标志、翻译按批暂停、失败视频不跨阶段推进、任务无明细/中途登记视频时 置回 QUEUED、任务工作空间与残留信号清理、任务列表默认过滤批量 run、详情阶段字段、 前端阶段列渲染；全量 507 passed（唯一失败为既有素材缺失的 integration 用例）。
374 lines
14 KiB
Python
374 lines
14 KiB
Python
"""src/wov_app/routers/batch.py 的模块级测试（数据 → 测试过程 → 验证结果）。
|
||
|
||
被测模块：`src/wov_app/routers/batch.py`（批量处理 API：目录浏览、任务创建
|
||
与列表、暂停/继续、删除、产物下载），可独立调用。用例通过真实 TestClient，
|
||
文件夹为临时目录下的真实视频文件。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
from fastapi.testclient import TestClient
|
||
|
||
from wov_app import registry
|
||
from wov_app.db import Database
|
||
from wov_app.main import app as fastapi_app
|
||
from wov_sdk.models import WorkflowDefinition
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _isolate_registry():
|
||
"""保存/恢复注册表（模块自带隔离）。"""
|
||
snapshot = dict(registry._registry)
|
||
yield
|
||
registry._registry.clear()
|
||
registry._registry.update(snapshot)
|
||
|
||
|
||
@pytest.fixture()
|
||
def client(tmp_path: Path, monkeypatch) -> TestClient:
|
||
"""构造隔离数据库与存储的真实 TestClient。"""
|
||
monkeypatch.setattr("wov_app.config.DB_PATH", tmp_path / "wov.db")
|
||
monkeypatch.setattr("wov_app.config.STORAGE_DIR", tmp_path / "storage")
|
||
monkeypatch.setattr("wov_app.main.DB_PATH", tmp_path / "wov.db")
|
||
monkeypatch.setattr("wov_app.main.STORAGE_DIR", tmp_path / "storage")
|
||
# 批量工作空间也指向临时目录（模块常量在导入时绑定）。
|
||
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
|
||
monkeypatch.setenv("WOV_AUTO_SEED", "0")
|
||
monkeypatch.setenv("WOV_SCHEDULER_ENABLED", "0")
|
||
monkeypatch.setenv("WOV_CLEANUP_ENABLED", "0")
|
||
monkeypatch.setenv("WOV_BATCH_ENABLED", "0")
|
||
with TestClient(fastapi_app) as test_client:
|
||
yield test_client
|
||
|
||
|
||
def _make_video(path: Path) -> Path:
|
||
"""创建真实可读的视频文件。"""
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_bytes(b"\x00\x00\x00\x18ftypmp42" + b"\x00" * 32)
|
||
return path
|
||
|
||
|
||
def _publish_workflow(client: TestClient, workflow_id: str = "wf") -> None:
|
||
"""创建并发布单 echo 节点工作流。"""
|
||
definition = WorkflowDefinition.from_dict({
|
||
"name": "批量流程", "version": 1,
|
||
"nodes": [{"id": "step", "node_type": "echo", "inputs": {"file_uri": "input.video_uri"}}],
|
||
"edges": [], "entry_inputs": {"video_uri": "file"},
|
||
"final_outputs": {"result": "step.file_uri"},
|
||
}).to_dict()
|
||
client.post("/api/admin/workflows", json={
|
||
"id": workflow_id, "name": "批量流程", "description": "", "definition": definition,
|
||
})
|
||
client.post(f"/api/admin/workflows/{workflow_id}/publish")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 目录浏览（本地后端提供，浏览器拿不到绝对路径）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_list_roots_returns_browsable_roots(client: TestClient) -> None:
|
||
"""根目录列表返回可浏览位置（POSIX 返回 / 与家目录）。"""
|
||
# 数据：无（真实文件系统）。
|
||
# 测试过程
|
||
response = client.get("/api/batch/roots")
|
||
|
||
# 验证结果：非空且每项含 path/name。
|
||
assert response.status_code == 200
|
||
roots = response.json()
|
||
assert roots, "至少应返回一个可浏览根目录"
|
||
assert all({"path", "name"} <= set(item) for item in roots)
|
||
|
||
|
||
def test_list_dirs_returns_only_directories(client: TestClient, tmp_path: Path) -> None:
|
||
"""列目录只返回直接子目录，且过滤隐藏目录。"""
|
||
# 数据：两个子目录 + 一个隐藏目录 + 一个文件。
|
||
folder = tmp_path / "root"
|
||
(folder / "sub1").mkdir(parents=True)
|
||
(folder / "sub2").mkdir()
|
||
(folder / ".hidden").mkdir()
|
||
(folder / "file.txt").write_text("x", encoding="utf-8")
|
||
|
||
# 测试过程
|
||
body = client.get("/api/batch/dirs", params={"path": str(folder)}).json()
|
||
|
||
# 验证结果
|
||
names = {item["name"] for item in body["dirs"]}
|
||
assert names == {"sub1", "sub2"}
|
||
|
||
|
||
def test_list_dirs_returns_empty_for_missing_path(client: TestClient, tmp_path: Path) -> None:
|
||
"""目录不存在时返回空列表而非 500（前端树保持可用）。"""
|
||
# 数据：不存在的路径。
|
||
# 测试过程
|
||
response = client.get("/api/batch/dirs", params={"path": str(tmp_path / "nope")})
|
||
|
||
# 验证结果
|
||
assert response.status_code == 200
|
||
assert response.json()["dirs"] == []
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 创建与查询批量任务
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_create_job_returns_job_with_videos(client: TestClient, tmp_path: Path) -> None:
|
||
"""创建批量任务：返回任务 ID、总数与明细列表（含跳过项）。"""
|
||
# 数据：两个视频，其中一个已有旁挂字幕。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "a.mp4")
|
||
_make_video(folder / "b.mp4")
|
||
(folder / "b.CN.srt").write_text("1\n", encoding="utf-8")
|
||
_publish_workflow(client)
|
||
|
||
# 测试过程
|
||
response = client.post("/api/batch/jobs", json={
|
||
"folder": str(folder), "workflow_id": "wf", "recursive": False,
|
||
})
|
||
|
||
# 验证结果
|
||
assert response.status_code == 200, response.text
|
||
body = response.json()
|
||
assert body["total"] == 1
|
||
assert len(body["videos"]) == 2
|
||
|
||
|
||
def test_create_job_rejects_missing_folder(client: TestClient, tmp_path: Path) -> None:
|
||
"""文件夹不存在返回 422。"""
|
||
# 数据：未发布的目录路径。
|
||
_publish_workflow(client)
|
||
|
||
# 测试过程
|
||
response = client.post("/api/batch/jobs", json={
|
||
"folder": str(tmp_path / "nope"), "workflow_id": "wf", "recursive": False,
|
||
})
|
||
|
||
# 验证结果
|
||
assert response.status_code == 422
|
||
|
||
|
||
def test_create_job_rejects_unpublished_workflow(client: TestClient, tmp_path: Path) -> None:
|
||
"""工作流未发布返回 422。"""
|
||
# 数据：有视频但工作流未发布。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "a.mp4")
|
||
|
||
# 测试过程
|
||
response = client.post("/api/batch/jobs", json={
|
||
"folder": str(folder), "workflow_id": "wf", "recursive": False,
|
||
})
|
||
|
||
# 验证结果
|
||
assert response.status_code == 422
|
||
|
||
|
||
def test_create_job_rejects_folder_without_videos(client: TestClient, tmp_path: Path) -> None:
|
||
"""文件夹内无视频返回 422。"""
|
||
# 数据：空文件夹 + 已发布工作流。
|
||
folder = tmp_path / "videos"
|
||
folder.mkdir()
|
||
_publish_workflow(client)
|
||
|
||
# 测试过程
|
||
response = client.post("/api/batch/jobs", json={
|
||
"folder": str(folder), "workflow_id": "wf", "recursive": False,
|
||
})
|
||
|
||
# 验证结果
|
||
assert response.status_code == 422
|
||
|
||
|
||
def test_list_and_get_job(client: TestClient, tmp_path: Path) -> None:
|
||
"""任务列表与详情可读，详情含视频明细。"""
|
||
# 数据：一条批量任务。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "a.mp4")
|
||
_publish_workflow(client)
|
||
job_id = client.post("/api/batch/jobs", json={
|
||
"folder": str(folder), "workflow_id": "wf", "recursive": False,
|
||
}).json()["id"]
|
||
|
||
# 测试过程
|
||
listed = client.get("/api/batch/jobs").json()
|
||
detail = client.get(f"/api/batch/jobs/{job_id}")
|
||
|
||
# 验证结果
|
||
assert any(item["id"] == job_id for item in listed)
|
||
assert detail.status_code == 200
|
||
assert detail.json()["videos"][0]["video_path"].endswith("a.mp4")
|
||
|
||
|
||
def test_get_missing_job_returns_404(client: TestClient) -> None:
|
||
"""不存在的批量任务返回 404。"""
|
||
# 数据：未创建。
|
||
# 测试过程与验证结果
|
||
assert client.get("/api/batch/jobs/nope").status_code == 404
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 暂停 / 继续 / 删除
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_pause_and_resume_job(client: TestClient, tmp_path: Path) -> None:
|
||
"""暂停置 PAUSED、继续置 QUEUED。"""
|
||
# 数据：一条待处理批量任务。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "a.mp4")
|
||
_publish_workflow(client)
|
||
job_id = client.post("/api/batch/jobs", json={
|
||
"folder": str(folder), "workflow_id": "wf", "recursive": False,
|
||
}).json()["id"]
|
||
|
||
# 测试过程
|
||
paused = client.post(f"/api/batch/jobs/{job_id}/pause")
|
||
after_pause = client.get(f"/api/batch/jobs/{job_id}").json()["status"]
|
||
resumed = client.post(f"/api/batch/jobs/{job_id}/resume")
|
||
after_resume = client.get(f"/api/batch/jobs/{job_id}").json()["status"]
|
||
|
||
# 验证结果
|
||
assert paused.status_code == 200 and resumed.status_code == 200
|
||
assert after_pause == "PAUSED"
|
||
assert after_resume == "QUEUED"
|
||
|
||
|
||
def test_delete_job_removes_records_and_keeps_media(client: TestClient, tmp_path: Path) -> None:
|
||
"""删除任务清理记录与私有工作空间，保留用户视频。"""
|
||
# 数据：一条批量任务。
|
||
folder = tmp_path / "videos"
|
||
video = _make_video(folder / "a.mp4")
|
||
_publish_workflow(client)
|
||
job_id = client.post("/api/batch/jobs", json={
|
||
"folder": str(folder), "workflow_id": "wf", "recursive": False,
|
||
}).json()["id"]
|
||
|
||
# 测试过程
|
||
response = client.delete(f"/api/batch/jobs/{job_id}")
|
||
|
||
# 验证结果：任务消失但视频保留。
|
||
assert response.status_code == 200
|
||
assert client.get(f"/api/batch/jobs/{job_id}").status_code == 404
|
||
assert video.is_file()
|
||
|
||
|
||
def test_delete_missing_job_returns_404(client: TestClient) -> None:
|
||
"""删除不存在的任务返回 404。"""
|
||
# 数据：未创建。
|
||
# 测试过程与验证结果
|
||
assert client.delete("/api/batch/jobs/nope").status_code == 404
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 产物下载
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_download_sidecar_product(client: TestClient, tmp_path: Path) -> None:
|
||
"""下载接口返回视频旁的产物文件内容。"""
|
||
# 数据：一条批量任务 + 视频旁的成品字幕。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "movie.mp4")
|
||
_publish_workflow(client)
|
||
body = client.post("/api/batch/jobs", json={
|
||
"folder": str(folder), "workflow_id": "wf", "recursive": False,
|
||
}).json()
|
||
job_id = body["id"]
|
||
video_id = body["videos"][0]["id"]
|
||
(folder / "movie.CN.srt").write_text("1\n00:00:01,000 --> 00:00:02,000\n你好\n", encoding="utf-8")
|
||
|
||
# 测试过程
|
||
response = client.get(
|
||
f"/api/batch/jobs/{job_id}/videos/{video_id}/download",
|
||
params={"alias": "movie.CN.srt"},
|
||
)
|
||
|
||
# 验证结果
|
||
assert response.status_code == 200
|
||
assert "你好" in response.content.decode("utf-8")
|
||
|
||
|
||
def test_download_missing_product_returns_404(client: TestClient, tmp_path: Path) -> None:
|
||
"""产物不存在时返回 404。"""
|
||
# 数据：一条批量任务（无产物）。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "movie.mp4")
|
||
_publish_workflow(client)
|
||
body = client.post("/api/batch/jobs", json={
|
||
"folder": str(folder), "workflow_id": "wf", "recursive": False,
|
||
}).json()
|
||
|
||
# 测试过程
|
||
response = client.get(
|
||
f"/api/batch/jobs/{body['id']}/videos/{body['videos'][0]['id']}/download",
|
||
params={"alias": "movie.CN.srt"},
|
||
)
|
||
|
||
# 验证结果
|
||
assert response.status_code == 404
|
||
|
||
|
||
def test_job_detail_reports_video_stage(client: TestClient, tmp_path: Path) -> None:
|
||
"""详情给处理中的视频补阶段信息：第几阶段/共几阶段 + 中文标签。"""
|
||
# 数据：三节点工作流（prep → translate → post），视频 run 停在第二阶段。
|
||
definition = WorkflowDefinition.from_dict({
|
||
"name": "分阶段流程", "version": 1,
|
||
"nodes": [
|
||
{"id": "prep", "node_type": "echo", "inputs": {"file_uri": "input.video_uri"}},
|
||
{"id": "translate", "node_type": "llm-translate", "inputs": {"srt_uri": "prep.file_uri"}},
|
||
{"id": "post", "node_type": "echo", "inputs": {"file_uri": "translate.file_uri"}},
|
||
],
|
||
"edges": [{"from": "prep", "to": "translate"}, {"from": "translate", "to": "post"}],
|
||
"entry_inputs": {"video_uri": "file"},
|
||
"final_outputs": {"result": "post.file_uri"},
|
||
}).to_dict()
|
||
client.post("/api/admin/workflows", json={
|
||
"id": "wf-staged", "name": "分阶段流程", "description": "", "definition": definition,
|
||
})
|
||
client.post("/api/admin/workflows/wf-staged/publish")
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "movie.mp4")
|
||
job_id = client.post("/api/batch/jobs", json={
|
||
"folder": str(folder), "workflow_id": "wf-staged", "recursive": True,
|
||
}).json()["id"]
|
||
db = Database(tmp_path / "wov.db")
|
||
video = [v for v in db.list_batch_videos(job_id) if v["status"] != "SKIPPED"][0]
|
||
db.update_batch_video(video["id"], status="RUNNING", run_id="run_stage", updated_at="2026-09-01T00:00:00+00:00")
|
||
db.create_run({
|
||
"id": "run_stage", "workflow_id": "wf-staged", "workflow_version": 1,
|
||
"status": "RUNNING", "current_node_id": "translate", "progress": 0.3333,
|
||
"error": None, "input_uri": str(folder / "movie.mp4"), "param_overrides": None,
|
||
"source": "batch", "created_at": "2026-09-01T00:00:00+00:00",
|
||
"updated_at": "2026-09-01T00:00:00+00:00",
|
||
})
|
||
|
||
# 测试过程
|
||
body = client.get(f"/api/batch/jobs/{job_id}").json()
|
||
|
||
# 验证结果：阶段序号/总数与节点类型对应的中文标签。
|
||
item = [v for v in body["videos"] if v["status"] != "SKIPPED"][0]
|
||
assert item["stage_label"] == "翻译"
|
||
assert (item["stage_index"], item["stage_total"]) == (2, 3)
|
||
|
||
|
||
def test_job_detail_omits_stage_for_unstarted_video(client: TestClient, tmp_path: Path) -> None:
|
||
"""还没开始处理的视频没有阶段信息（前端显示占位符）。"""
|
||
# 数据：一个 PENDING 视频（无 run）。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "movie.mp4")
|
||
_publish_workflow(client)
|
||
job_id = client.post("/api/batch/jobs", json={
|
||
"folder": str(folder), "workflow_id": "wf", "recursive": True,
|
||
}).json()["id"]
|
||
|
||
# 测试过程
|
||
body = client.get(f"/api/batch/jobs/{job_id}").json()
|
||
|
||
# 验证结果：阶段字段为空。
|
||
item = [v for v in body["videos"] if v["status"] != "SKIPPED"][0]
|
||
assert item["stage_label"] is None
|
||
assert item["stage_index"] is None and item["stage_total"] is None
|