为视频生成 VR 双眼字幕的单体实现:FastAPI 后端、调度器与全部节点 (提音/转写/翻译/ASS/抽帧/OCR/LLM 过滤)在单进程内运行。 - 节点协议(wov_sdk 数据模型)与分布式版保持一致,预留回退桥梁 - 工作流即数据:DAG 存于 workflows/*.json,模型/链路改动只改数据 - 调度器:拓扑顺序执行、断点续跑(产物重建)、任务暂停/继续 - 抽帧按帧间隔(select 按帧号精确取帧),VLM OCR 与 LLM 过滤使用 自适应线程池弹性并发,并打印数据处理速度进度日志 - 100% 行覆盖率(pytest --cov-fail-under=100)
166 lines
6.3 KiB
Python
166 lines
6.3 KiB
Python
"""孤儿数据清理器测试。
|
|
|
|
覆盖 COMPLETED 无文件任务的删除、各类保留分支(有文件/失败/宽限期内)、
|
|
无任务记录的残留目录清理、清理线程启停以及防御性分支。
|
|
"""
|
|
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from wov_app.db import Database
|
|
from wov_app.maintenance import OrphanCleaner
|
|
|
|
|
|
def _db(tmp_path) -> Database:
|
|
"""在临时目录创建独立数据库。"""
|
|
return Database(tmp_path / "wov.db")
|
|
|
|
|
|
def _make_run(db, run_id, status="COMPLETED", updated="2020-01-01T00:00:00+00:00", input_uri=None):
|
|
"""创建指定状态与更新时间的工作流任务记录。"""
|
|
db.upsert_workflow({"id": "flow", "name": "F", "published": 1, "latest_version": 1})
|
|
db.create_run(
|
|
{
|
|
"id": run_id,
|
|
"workflow_id": "flow",
|
|
"workflow_version": 1,
|
|
"status": status,
|
|
"progress": 1,
|
|
"input_uri": input_uri,
|
|
"created_at": updated,
|
|
"updated_at": updated,
|
|
}
|
|
)
|
|
|
|
|
|
def _now_iso() -> str:
|
|
"""返回当前 UTC 时间的 ISO 字符串。"""
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
|
|
def test_cleaner_removes_completed_orphan_run(tmp_path) -> None:
|
|
"""验证 COMPLETED 且无任何产物文件、超过宽限期的任务被整体清理。"""
|
|
db = _db(tmp_path)
|
|
upload_dir = tmp_path / "storage" / "uploads" / "run_orphan"
|
|
upload_dir.mkdir(parents=True)
|
|
upload_file = upload_dir / "in.mp4"
|
|
upload_file.write_bytes(b"x")
|
|
_make_run(db, "run_orphan", input_uri=str(upload_file))
|
|
db.create_artifact(
|
|
{
|
|
"run_id": "run_orphan",
|
|
"node_id": "asr",
|
|
"name": "asr.srt_uri",
|
|
"uri": str(tmp_path / "storage" / "runs" / "run_orphan" / "out.srt"),
|
|
"mime_type": "application/x-subrip",
|
|
"size": 1,
|
|
}
|
|
)
|
|
cleaner = OrphanCleaner(db, tmp_path / "storage", grace_seconds=3600)
|
|
assert cleaner.clean_once() == 1
|
|
assert db.get_run("run_orphan") is None
|
|
assert db.list_artifacts("run_orphan") == []
|
|
assert not upload_dir.exists()
|
|
|
|
|
|
def test_cleaner_keeps_completed_run_with_files(tmp_path) -> None:
|
|
"""验证仍有产物文件的 COMPLETED 任务不会被清理。"""
|
|
db = _db(tmp_path)
|
|
steps = tmp_path / "storage" / "runs" / "run_keep"
|
|
(steps / "asr").mkdir(parents=True)
|
|
(steps / "asr" / "out.srt").write_text("1\n00:00:00,000 --> 00:00:01,000\nok\n", encoding="utf-8")
|
|
_make_run(db, "run_keep")
|
|
cleaner = OrphanCleaner(db, tmp_path / "storage", grace_seconds=3600)
|
|
assert cleaner.clean_once() == 0
|
|
assert db.get_run("run_keep") is not None
|
|
|
|
|
|
def test_cleaner_keeps_failed_and_recent_runs(tmp_path) -> None:
|
|
"""验证 FAILED 任务与宽限期内的任务都不会被自动清理。"""
|
|
db = _db(tmp_path)
|
|
_make_run(db, "run_failed", status="FAILED")
|
|
_make_run(db, "run_recent", updated=_now_iso())
|
|
cleaner = OrphanCleaner(db, tmp_path / "storage", grace_seconds=3600)
|
|
assert cleaner.clean_once() == 0
|
|
assert db.get_run("run_failed") is not None
|
|
assert db.get_run("run_recent") is not None
|
|
|
|
|
|
def test_cleaner_removes_dangling_dirs_only(tmp_path) -> None:
|
|
"""验证无任务记录的残留目录被删除,已有任务的上传目录被保留。"""
|
|
db = _db(tmp_path)
|
|
ghost_upload = tmp_path / "storage" / "uploads" / "ghost"
|
|
ghost_upload.mkdir(parents=True)
|
|
ghost_steps = tmp_path / "storage" / "runs" / "ghost"
|
|
ghost_steps.mkdir(parents=True)
|
|
keep_upload = tmp_path / "storage" / "uploads" / "run_keep"
|
|
keep_upload.mkdir(parents=True)
|
|
(keep_upload / "in.mp4").write_bytes(b"x")
|
|
# run_keep 存在产物文件,不属于孤儿任务。
|
|
keep_steps = tmp_path / "storage" / "runs" / "run_keep" / "asr"
|
|
keep_steps.mkdir(parents=True)
|
|
(keep_steps / "out.srt").write_text("ok", encoding="utf-8")
|
|
_make_run(db, "run_keep")
|
|
cleaner = OrphanCleaner(db, tmp_path / "storage", grace_seconds=3600)
|
|
assert cleaner.clean_once() == 2
|
|
assert not ghost_upload.exists()
|
|
assert not ghost_steps.exists()
|
|
assert keep_upload.exists()
|
|
|
|
|
|
def test_cleaner_removes_run_with_empty_steps_dir(tmp_path) -> None:
|
|
"""验证步骤目录存在但为空(无文件)时仍视为孤儿清理。"""
|
|
db = _db(tmp_path)
|
|
steps = tmp_path / "storage" / "runs" / "run_empty"
|
|
(steps / "asr").mkdir(parents=True)
|
|
_make_run(db, "run_empty", input_uri="")
|
|
cleaner = OrphanCleaner(db, tmp_path / "storage", grace_seconds=3600)
|
|
assert cleaner.clean_once() == 1
|
|
assert db.get_run("run_empty") is None
|
|
assert not steps.exists()
|
|
|
|
|
|
def test_cleaner_default_config_and_defensive_branches(tmp_path, monkeypatch) -> None:
|
|
"""验证默认配置构造、缺失/非法时间与缺失任务记录的防御分支。"""
|
|
db = _db(tmp_path)
|
|
# 默认配置(interval/grace 走 config 默认值)。
|
|
cleaner = OrphanCleaner(db, tmp_path / "storage")
|
|
assert cleaner.interval_seconds > 0
|
|
assert cleaner.grace_seconds > 0
|
|
# 无更新时间 / 非法时间均保守视为未过期。
|
|
assert cleaner._expired(None) is False
|
|
assert cleaner._expired("not-a-date") is False
|
|
# 不存在的根目录直接返回 0。
|
|
assert cleaner._clean_dangling(tmp_path / "missing", set()) == 0
|
|
# list_run_ids 返回的 ID 在读取详情前已不存在时跳过。
|
|
monkeypatch.setattr(db, "list_run_ids", lambda: ["ghost"])
|
|
monkeypatch.setattr(db, "get_run", lambda run_id: None)
|
|
assert cleaner.clean_once() == 0
|
|
|
|
|
|
def test_cleaner_start_stop_loop(tmp_path) -> None:
|
|
"""验证清理线程可启动、周期执行并正常停止。"""
|
|
import time
|
|
|
|
db = _db(tmp_path)
|
|
cleaner = OrphanCleaner(db, tmp_path / "storage", interval_seconds=0.05, grace_seconds=3600)
|
|
cleaner.start()
|
|
try:
|
|
cleaner.start()
|
|
time.sleep(0.2)
|
|
finally:
|
|
cleaner.stop()
|
|
assert cleaner._thread is None
|
|
|
|
|
|
def test_lifespan_starts_cleaner(monkeypatch) -> None:
|
|
"""验证启用清理器时应用生命周期会启动清理线程并随退出停止。"""
|
|
from fastapi.testclient import TestClient
|
|
|
|
from wov_app.main import app
|
|
|
|
monkeypatch.setenv("WOV_CLEANUP_ENABLED", "1")
|
|
with TestClient(app) as client:
|
|
assert client.get("/health").status_code == 200
|
|
assert app.state.cleaner._thread is not None
|