cat-shark
8a715a8064
按"测试规则"重写 tests/：一个模块一个目录、用例按数据→过程→验证三段书写、 不保留全局 conftest.py、测试过程只调用真实生产代码。 结构（73 个文件、30 个模块目录、477 用例）： - tests/nodes/ 15 个模块目录（srt/whisper/ass/ffmpeg/frame_extract/vlm/ subtitle_ocr/llm/llm_filter/subtitle_cleanup/subtitle_correction/ proper_nouns/adaptive_pool/vad_profiler/echo）； - tests/app/ 11 个模块目录（db/scheduler/batch/maintenance/registry/seed/ storage/config/logging/main/routers 三组 API）； - tests/sdk/test_models、tests/web/test_crop、tests/shared（公共设施）。 测试数据随模块目录入库（tests/**/data/），删除根级 testdata/；.gitignore 的 data/ 改为 /data/，否则会连带忽略 tests/**/data/ 导致测试数据无法入库。 顺带发现并修复三个真实缺陷： - nodes/srt.py：相邻条目缺少空行时把下一条时间轴吞进正文（静默错位）， 改为正文行遇时间戳行即报错； - src/wov_app/scheduler.py：_file_size 只捕获 OSError，含 \x00 的产物 URI 抛 ValueError 导致任务误判失败，改为同时捕获； - nodes/subtitle_correction.py：生产代码依赖测试包解析 SRT， 改用生产模块 nodes/srt.py。 真实模型/服务集成测试按外部状态跳过：新增 tests/shared/gpu_memory.py （运行时探测显存、CUDA OOM 转跳过）与 tests/shared/llm_service.py （无 Key / 余额 / 限流转跳过）。全量 477 passed。
415 lines
16 KiB
Python
415 lines
16 KiB
Python
"""src/wov_app/batch.py 的模块级测试（数据 → 测试过程 → 验证结果）。
|
||
|
||
被测模块：`src/wov_app/batch.py`（文件夹批量处理：扫描定位、旁挂字幕跳过、
|
||
引擎执行与产物放置），可独立调用。用例在临时目录构造真实视频/字幕文件与
|
||
真实 SQLite 记录，使用真实 echo 节点跑通执行链路。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from wov_app import registry
|
||
from wov_app.batch import (
|
||
MARKER_NAME,
|
||
SUBTITLE_EXTENSIONS,
|
||
VIDEO_EXTENSIONS,
|
||
BatchWorker,
|
||
_sidecar_product_name,
|
||
create_job,
|
||
list_sidecar_subtitles,
|
||
load_marker,
|
||
remove_job_workspace,
|
||
scan_videos,
|
||
)
|
||
from wov_app.db import Database
|
||
from wov_sdk.models import WorkflowDefinition
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _isolate_registry():
|
||
"""用例前清空注册表、用例后恢复快照：保证用例看到的是干净基线，
|
||
不受其他模块（如 main 生命周期 register_all）的注册结果影响。"""
|
||
snapshot = dict(registry._registry)
|
||
registry._registry.clear()
|
||
yield
|
||
registry._registry.clear()
|
||
registry._registry.update(snapshot)
|
||
|
||
|
||
def _make_video(path: Path) -> Path:
|
||
"""创建真实可读的视频文件（内容不重要，但必须是真实文件）。"""
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_bytes(b"\x00\x00\x00\x18ftypmp42" + b"\x00" * 64)
|
||
return path
|
||
|
||
|
||
def _echo_definition(final_output: str = "step.file_uri") -> WorkflowDefinition:
|
||
"""单 echo 节点的真实工作流定义。"""
|
||
return WorkflowDefinition.from_dict({
|
||
"name": "批量流程",
|
||
"version": 1,
|
||
"nodes": [{"id": "step", "node_type": "echo", "inputs": {"file_uri": "input.video_uri"}}],
|
||
"edges": [],
|
||
"entry_inputs": {"video_uri": "file"},
|
||
"final_outputs": {"result": final_output},
|
||
})
|
||
|
||
|
||
def _published_db(tmp_path: Path, workflow_id: str = "wf") -> Database:
|
||
"""建好已发布工作流（含版本）的临时库。"""
|
||
db = Database(tmp_path / "wov.db")
|
||
db.upsert_workflow({
|
||
"id": workflow_id, "name": "批量流程", "description": "", "published": 1,
|
||
"latest_version": 1,
|
||
})
|
||
db.create_workflow_version(workflow_id, 1, _echo_definition().to_dict())
|
||
return db
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 扫描与旁挂字幕判定
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_scan_videos_recursive_and_flat(tmp_path: Path) -> None:
|
||
"""递归扫描包含子目录视频；非递归只扫顶层；结果按路径排序。"""
|
||
# 数据：顶层 2 个视频 + 子目录 1 个视频 + 1 个非视频文件。
|
||
_make_video(tmp_path / "b.mp4")
|
||
_make_video(tmp_path / "a.mkv")
|
||
_make_video(tmp_path / "sub" / "c.mp4")
|
||
(tmp_path / "note.txt").write_text("x", encoding="utf-8")
|
||
|
||
# 测试过程
|
||
recursive = [p.name for p in scan_videos(tmp_path, recursive=True)]
|
||
flat = [p.name for p in scan_videos(tmp_path, recursive=False)]
|
||
|
||
# 验证结果
|
||
assert recursive == ["a.mkv", "b.mp4", "c.mp4"]
|
||
assert flat == ["a.mkv", "b.mp4"]
|
||
|
||
|
||
def test_video_extensions_are_lowercase_dotted() -> None:
|
||
"""视频扩展名集合为小写带点形式（与 suffix.lower() 比较一致）。"""
|
||
# 数据：模块常量。
|
||
# 测试过程与验证结果
|
||
assert all(ext.startswith(".") and ext.islower() for ext in VIDEO_EXTENSIONS)
|
||
assert ".mp4" in VIDEO_EXTENSIONS and ".mkv" in VIDEO_EXTENSIONS
|
||
|
||
|
||
def test_list_sidecar_subtitles_matches_by_stem(tmp_path: Path) -> None:
|
||
"""视频旁含视频主名的字幕文件被识别（含 CN/dual_eye 等约定命名）。"""
|
||
# 数据：视频 + 三种约定命名的字幕 + 一个无关文件。
|
||
video = _make_video(tmp_path / "movie.mp4")
|
||
expected = [
|
||
tmp_path / "movie.CN.srt",
|
||
tmp_path / "movie.CN_dual_eye.ass",
|
||
tmp_path / "movie.srt",
|
||
]
|
||
for path in expected:
|
||
path.write_text("1\n", encoding="utf-8")
|
||
(tmp_path / "other.srt").write_text("1\n", encoding="utf-8")
|
||
|
||
# 测试过程
|
||
found = list_sidecar_subtitles(video)
|
||
|
||
# 验证结果：三个匹配、无关文件不在结果里。
|
||
assert set(found) == set(expected)
|
||
assert (tmp_path / "other.srt") not in found
|
||
|
||
|
||
def test_list_sidecar_subtitles_ignores_non_subtitle_files(tmp_path: Path) -> None:
|
||
"""同名但非字幕扩展名的文件不算旁挂字幕。"""
|
||
# 数据：视频 + 同名字幕 + 同名文本。
|
||
video = _make_video(tmp_path / "movie.mp4")
|
||
srt = tmp_path / "movie.srt"
|
||
srt.write_text("1\n", encoding="utf-8")
|
||
(tmp_path / "movie.txt").write_text("x", encoding="utf-8")
|
||
|
||
# 测试过程
|
||
found = list_sidecar_subtitles(video)
|
||
|
||
# 验证结果
|
||
assert found == [srt]
|
||
|
||
|
||
def test_list_sidecar_subtitles_short_stem_requires_dot_prefix(tmp_path: Path) -> None:
|
||
"""视频主名只有一个字符时只接受"主名."前缀，避免 a.mp4 误配 apple.srt。"""
|
||
# 数据：a.mp4 + apple.srt（不应命中）+ a.srt（应命中）。
|
||
video = _make_video(tmp_path / "a.mp4")
|
||
(tmp_path / "apple.srt").write_text("1\n", encoding="utf-8")
|
||
good = tmp_path / "a.srt"
|
||
good.write_text("1\n", encoding="utf-8")
|
||
|
||
# 测试过程
|
||
found = list_sidecar_subtitles(video)
|
||
|
||
# 验证结果
|
||
assert found == [good]
|
||
|
||
|
||
def test_subtitle_extensions_cover_common_formats() -> None:
|
||
"""字幕扩展名覆盖 srt/ass/ssa/vtt。"""
|
||
# 数据：模块常量。
|
||
# 测试过程与验证结果
|
||
assert SUBTITLE_EXTENSIONS == {".srt", ".ass", ".ssa", ".vtt"}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 产物命名映射
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_sidecar_product_name_maps_srt_and_ass() -> None:
|
||
"""最终产物映射为媒体库约定名（.srt → CN.srt，.ass → CN_dual_eye.ass）。"""
|
||
# 数据：视频与两类最终产物。
|
||
video = Path("/videos/movie.mp4")
|
||
|
||
# 测试过程与验证结果
|
||
assert _sidecar_product_name(video, Path("/tmp/x.zh-CN.20260101.srt")) == "movie.CN.srt"
|
||
assert _sidecar_product_name(video, Path("/tmp/x.ass")) == "movie.CN_dual_eye.ass"
|
||
|
||
|
||
def test_sidecar_product_name_keeps_other_extensions() -> None:
|
||
"""其他扩展名产物保留原文件名（不误改语义）。"""
|
||
# 数据：vtt 产物。
|
||
# 测试过程与验证结果
|
||
assert _sidecar_product_name(Path("/v/movie.mp4"), Path("/tmp/movie.vtt")) == "movie.vtt"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 完成标记与工作空间清理
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_load_marker_reads_valid_json(tmp_path: Path) -> None:
|
||
"""旧版完成标记（batch.done.json）可读回字典。"""
|
||
# 数据：真实标记文件。
|
||
work = tmp_path / "work"
|
||
work.mkdir()
|
||
payload = {"finals": {"cn_srt_uri": "/videos/movie.CN.srt"}}
|
||
(work / MARKER_NAME).write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
|
||
|
||
# 测试过程
|
||
marker = load_marker(work)
|
||
|
||
# 验证结果
|
||
assert marker == payload
|
||
|
||
|
||
def test_load_marker_returns_none_for_corrupt_or_missing(tmp_path: Path) -> None:
|
||
"""标记缺失或内容损坏时返回 None（走旁挂字幕判定，不报错）。"""
|
||
# 数据：不存在标记 + 损坏标记。
|
||
work = tmp_path / "work"
|
||
work.mkdir()
|
||
assert load_marker(work) is None
|
||
(work / MARKER_NAME).write_text("{broken", encoding="utf-8")
|
||
|
||
# 测试过程与验证结果
|
||
assert load_marker(work) is None
|
||
|
||
|
||
def test_remove_job_workspace_only_touches_private_dir(tmp_path: Path, monkeypatch) -> None:
|
||
"""删除任务只清理应用私有工作空间，不触碰用户视频目录。"""
|
||
# 数据：私有工作空间 + 用户媒体目录。
|
||
private = tmp_path / "storage" / "batch" / "job-1"
|
||
private.mkdir(parents=True)
|
||
(private / "temp.wav").write_bytes(b"x")
|
||
media = tmp_path / "media"
|
||
_make_video(media / "movie.mp4")
|
||
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
|
||
|
||
# 测试过程
|
||
remove_job_workspace("job-1")
|
||
|
||
# 验证结果：私有空间被删，用户目录完整。
|
||
assert not private.exists()
|
||
assert (media / "movie.mp4").is_file()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 创建批量任务：一次性定位
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_create_job_registers_pending_and_skipped(tmp_path: Path) -> None:
|
||
"""创建任务一次性定位视频：无字幕记 PENDING，已有字幕记 SKIPPED。"""
|
||
# 数据：3 个视频，其中一个已有旁挂字幕。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "a.mp4")
|
||
_make_video(folder / "b.mp4")
|
||
_make_video(folder / "c.mp4")
|
||
(folder / "b.CN.srt").write_text("1\n", encoding="utf-8")
|
||
db = _published_db(tmp_path)
|
||
|
||
# 测试过程
|
||
job_id = create_job(db, str(folder), "wf", recursive=False)
|
||
videos = db.list_batch_videos(job_id)
|
||
|
||
# 验证结果：状态分布正确，总数只算待处理。
|
||
statuses = {Path(v["video_path"]).name: v["status"] for v in videos}
|
||
assert statuses == {"a.mp4": "PENDING", "b.mp4": "SKIPPED", "c.mp4": "PENDING"}
|
||
job = db.get_batch_job(job_id)
|
||
assert job["total"] == 2
|
||
assert job["status"] == "QUEUED"
|
||
|
||
|
||
def test_create_job_completes_immediately_when_all_skipped(tmp_path: Path) -> None:
|
||
"""全部视频都已有字幕时任务直接完成，不排队不触发流水线。"""
|
||
# 数据：两个视频都带字幕。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "a.mp4")
|
||
_make_video(folder / "b.mp4")
|
||
(folder / "a.srt").write_text("1\n", encoding="utf-8")
|
||
(folder / "b.srt").write_text("1\n", encoding="utf-8")
|
||
db = _published_db(tmp_path)
|
||
|
||
# 测试过程
|
||
job_id = create_job(db, str(folder), "wf")
|
||
|
||
# 验证结果
|
||
job = db.get_batch_job(job_id)
|
||
assert job["status"] == "COMPLETED"
|
||
assert job["total"] == 0
|
||
assert db.next_queued_batch_job() is None
|
||
|
||
|
||
def test_create_job_uses_private_work_dir(tmp_path: Path, monkeypatch) -> None:
|
||
"""明细的工作空间位于应用私有目录（与用户媒体库隔离）。"""
|
||
# 数据：一个视频。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "a.mp4")
|
||
db = _published_db(tmp_path)
|
||
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
|
||
|
||
# 测试过程
|
||
job_id = create_job(db, str(folder), "wf")
|
||
video = db.list_batch_videos(job_id)[0]
|
||
|
||
# 验证结果：work_dir 在私有 storage/batch 下，且不在视频目录内。
|
||
work_dir = Path(video["work_dir"])
|
||
assert (tmp_path / "storage" / "batch") in work_dir.parents
|
||
assert folder not in work_dir.parents
|
||
|
||
|
||
@pytest.mark.parametrize(
|
||
("folder_setup", "workflow_id", "message"),
|
||
[
|
||
("missing", "wf", "folder not found"),
|
||
("empty", "wf", "no videos found"),
|
||
("ok", "unknown", "published workflow not found"),
|
||
],
|
||
)
|
||
def test_create_job_rejects_invalid_input(tmp_path: Path, folder_setup: str, workflow_id: str, message: str) -> None:
|
||
"""校验失败时抛 ValueError（路由层转 422）：目录缺失/无视频/工作流未发布。"""
|
||
# 数据：按参数准备目录与工作流。
|
||
folder = tmp_path / "videos"
|
||
if folder_setup == "empty":
|
||
folder.mkdir()
|
||
elif folder_setup == "ok":
|
||
_make_video(folder / "a.mp4")
|
||
db = _published_db(tmp_path)
|
||
|
||
# 测试过程与验证结果
|
||
with pytest.raises(ValueError, match=message):
|
||
create_job(db, str(folder), workflow_id)
|
||
|
||
|
||
def test_create_job_rejects_workflow_without_version(tmp_path: Path) -> None:
|
||
"""已发布但无版本记录的工作流被拒绝（无法执行）。"""
|
||
# 数据：有工作流记录但无版本。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "a.mp4")
|
||
db = Database(tmp_path / "wov.db")
|
||
db.upsert_workflow({"id": "wf", "name": "无版本", "description": "", "published": 1})
|
||
|
||
# 测试过程与验证结果
|
||
with pytest.raises(ValueError, match="no version"):
|
||
create_job(db, str(folder), "wf")
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 引擎：执行、产物放置、暂停
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_worker_processes_pending_video_end_to_end(tmp_path: Path, monkeypatch) -> None:
|
||
"""引擎处理待处理视频：跑通流水线、产物放到视频旁、明细与任务标记完成。"""
|
||
# 数据：一个视频 + echo 单节点工作流（产物为复制后的输入文件）。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "movie.mp4")
|
||
db = _published_db(tmp_path)
|
||
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
|
||
registry.register_all()
|
||
job_id = create_job(db, str(folder), "wf")
|
||
|
||
# 测试过程：直接驱动一轮处理（避免后台线程时序不确定）。
|
||
worker = BatchWorker(db, interval_seconds=999)
|
||
worker._process_job(db.get_batch_job(job_id))
|
||
|
||
# 验证结果：明细与任务完成，产物按约定名放到视频旁，工作空间被清理。
|
||
item = db.list_batch_videos(job_id)[0]
|
||
assert item["status"] == "COMPLETED"
|
||
assert db.get_batch_job(job_id)["status"] == "COMPLETED"
|
||
# echo 节点产物是 .txt，按约定保留原文件名放置在视频旁。
|
||
assert list(folder.glob("movie.*")), "应在视频旁放置最终产物"
|
||
|
||
|
||
def test_worker_skips_video_with_sidecar_subtitle(tmp_path: Path, monkeypatch) -> None:
|
||
"""已有旁挂字幕的视频不触发流水线（SKIPPED 不产生 run）。"""
|
||
# 数据：一个已带字幕的视频。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "movie.mp4")
|
||
(folder / "movie.CN.srt").write_text("1\n", encoding="utf-8")
|
||
_make_video(folder / "other.mp4")
|
||
db = _published_db(tmp_path)
|
||
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
|
||
registry.register_all()
|
||
|
||
# 测试过程
|
||
job_id = create_job(db, str(folder), "wf")
|
||
|
||
# 验证结果：SKIPPED 视频没有 run_id。
|
||
videos = {Path(v["video_path"]).name: v for v in db.list_batch_videos(job_id)}
|
||
assert videos["movie.mp4"]["status"] == "SKIPPED"
|
||
assert videos["movie.mp4"]["run_id"] is None
|
||
assert videos["other.mp4"]["status"] == "PENDING"
|
||
|
||
|
||
def test_worker_pause_sets_job_paused(tmp_path: Path, monkeypatch) -> None:
|
||
"""暂停批量任务：任务状态置 PAUSED，待处理视频不被推进。"""
|
||
# 数据：一个待处理视频的任务。
|
||
folder = tmp_path / "videos"
|
||
_make_video(folder / "a.mp4")
|
||
db = _published_db(tmp_path)
|
||
monkeypatch.setattr("wov_app.batch.BATCH_WORK_ROOT", tmp_path / "storage" / "batch")
|
||
job_id = create_job(db, str(folder), "wf")
|
||
|
||
# 测试过程
|
||
db.update_batch_job(job_id, status="PAUSED", updated_at="2026-09-01T01:00:00+00:00")
|
||
|
||
# 验证结果：不会被 next_queued_batch_job 拾起（等待显式 resume）。
|
||
assert db.next_queued_batch_job() is None
|
||
assert db.get_batch_job(job_id)["status"] == "PAUSED"
|
||
|
||
|
||
def test_worker_start_stop_idempotent(tmp_path: Path) -> None:
|
||
"""引擎 start 重复调用不产生多余线程；stop 正常结束。"""
|
||
# 数据：空库。
|
||
db = Database(tmp_path / "wov.db")
|
||
worker = BatchWorker(db, interval_seconds=999)
|
||
|
||
# 测试过程
|
||
worker.start()
|
||
first = worker._thread
|
||
worker.start()
|
||
second = worker._thread
|
||
worker.stop()
|
||
|
||
# 验证结果
|
||
assert first is second
|
||
assert worker._thread is None
|