fix: 保持产物收尾幂等并保护批量成品完整性
This commit is contained in:
+26
-10
@@ -682,13 +682,16 @@ def test_batch_worker_pending_leftover_never_marks_completed(tmp_path, monkeypat
|
||||
monkeypatch.setattr("wov_app.batch.WorkflowScheduler", _CrashScheduler)
|
||||
# 放开假调度器的崩溃：第二次调用时不再删 run，让真实调度器跑通。
|
||||
class _RecoveringScheduler:
|
||||
"""第二轮：不再崩溃，把 run 置 COMPLETED（由引擎收尾放产物）。"""
|
||||
"""第二轮：真实执行工作流并产生成品，由引擎收尾放置。"""
|
||||
|
||||
def __init__(self, db: Database, work_dir: Path) -> None:
|
||||
self.db = db
|
||||
|
||||
def execute_run(self, run_id: str) -> None:
|
||||
self.db.update_run(run_id, status="COMPLETED", updated_at=_now_iso())
|
||||
from wov_app.scheduler import WorkflowScheduler
|
||||
|
||||
video = self.db.list_batch_videos(job["id"])[0]
|
||||
WorkflowScheduler(self.db, Path(video["work_dir"])).execute_run(run_id)
|
||||
|
||||
monkeypatch.setattr("wov_app.batch.WorkflowScheduler", _RecoveringScheduler)
|
||||
worker._process_job(job)
|
||||
@@ -701,7 +704,7 @@ def test_batch_worker_already_completed_runs_finalized(tmp_path) -> None:
|
||||
"""run 已完成但视频未标记（收尾前中断）：直接放置产物、清理并标记完成。
|
||||
|
||||
覆盖 _place_products 各分支：产物齐全（复制）、产物记录存在但文件丢失
|
||||
（跳过）、无产物记录（跳过）——完成后均清理工作空间与 run 记录。
|
||||
（报错）、无产物记录（报错）——只有成品齐全才清理工作空间与 run 记录。
|
||||
"""
|
||||
db = _db(tmp_path)
|
||||
_seed_echo_workflow(db)
|
||||
@@ -714,6 +717,7 @@ def test_batch_worker_already_completed_runs_finalized(tmp_path) -> None:
|
||||
"""创建 COMPLETED run；可选产物文件与产物记录。"""
|
||||
_create_run(db, run_id, folder, status="COMPLETED", name=name)
|
||||
work_dir = Path(videos[name]["work_dir"])
|
||||
work_dir.mkdir(parents=True)
|
||||
if with_artifact:
|
||||
path = work_dir / "runs" / run_id / "steps" / "step" / f"{name}.result.txt"
|
||||
db.create_artifact(
|
||||
@@ -739,20 +743,24 @@ def test_batch_worker_already_completed_runs_finalized(tmp_path) -> None:
|
||||
|
||||
BatchWorker(db, interval_seconds=0.05)._process_job(db.get_batch_job(job["id"]))
|
||||
videos = {Path(v["video_path"]).stem: v for v in db.list_batch_videos(job["id"])}
|
||||
assert all(videos[name]["status"] == "COMPLETED" for name in ("a", "b", "c"))
|
||||
assert videos["a"]["status"] == "COMPLETED"
|
||||
assert all(videos[name]["status"] == "FAILED" for name in ("b", "c"))
|
||||
# a 的产物复制到视频旁；b/c 无产物可复制。
|
||||
product = _find_product(folder, "a", "result")
|
||||
assert product.read_text(encoding="utf-8") == "产物 a"
|
||||
assert not list(folder.glob("b.result.*")) and not list(folder.glob("c.result.*"))
|
||||
# 三个视频的工作空间与 run 记录都被清理。
|
||||
for name in ("a", "b", "c"):
|
||||
assert not Path(videos[name]["work_dir"]).exists()
|
||||
assert db.get_run(f"run_done_{name}") is None
|
||||
assert db.list_runs() == []
|
||||
# a 清理完成；b/c 必须保留工作空间、关联 run 与错误供修复后重试。
|
||||
assert not Path(videos["a"]["work_dir"]).exists()
|
||||
assert db.get_run("run_done_a") is None
|
||||
for name in ("b", "c"):
|
||||
assert Path(videos[name]["work_dir"]).exists()
|
||||
assert db.get_run(f"run_done_{name}") is not None
|
||||
assert videos[name]["run_id"] == f"run_done_{name}"
|
||||
assert "result" in videos[name]["error"]
|
||||
|
||||
|
||||
def test_batch_worker_place_products_branches(tmp_path) -> None:
|
||||
"""_place_products：无产物记录/文件丢失跳过；.srt/.ass 按库内约定命名覆盖；
|
||||
"""_place_products：无产物记录/文件丢失报错；.srt/.ass 按库内约定命名覆盖；
|
||||
其他扩展名保留原文件名复制。"""
|
||||
db = _db(tmp_path)
|
||||
_seed_echo_workflow(db)
|
||||
@@ -805,6 +813,14 @@ def test_batch_worker_place_products_branches(tmp_path) -> None:
|
||||
{"run_id": run_id, "node_id": "step", "name": "alias_txt", "uri": str(src_txt), "mime_type": "text/plain", "size": src_txt.stat().st_size}
|
||||
)
|
||||
|
||||
# 所有必需成品预检通过前不开始覆盖，缺失原因中包含具体别名。
|
||||
with pytest.raises(ValueError, match="alias_none"):
|
||||
worker._place_products(run_id, video, WorkflowDefinition.from_dict(definition))
|
||||
del definition["final_outputs"]["alias_none"]
|
||||
with pytest.raises(ValueError, match="alias_lost"):
|
||||
worker._place_products(run_id, video, WorkflowDefinition.from_dict(definition))
|
||||
assert target_srt.read_text(encoding="utf-8") == "旧字幕内容"
|
||||
del definition["final_outputs"]["alias_lost"]
|
||||
placed = worker._place_products(run_id, video, WorkflowDefinition.from_dict(definition))
|
||||
assert placed == ["a.CN.srt", "a.CN_dual_eye.ass", "a.other.20260902120000.txt"]
|
||||
# srt/ass 被改名为标准名且覆盖旧文件；txt 保留原名。
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
"""产物收尾回归：真实 SQLite、字幕文件和 API 下载验证暂停及故障恢复。"""
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from wov_app.db import Database
|
||||
from wov_app.main import app
|
||||
from wov_app.scheduler import WorkflowScheduler
|
||||
from wov_sdk.models import WorkflowDefinition
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def subtitle(tmp_path):
|
||||
"""复用真实字幕资产，不在测试中生成模型输出或占位媒体。"""
|
||||
source = Path(__file__).resolve().parent.parent / "testdata/ocr_srt_run_ac7f480a3ccb.srt"
|
||||
if not source.is_file():
|
||||
pytest.skip("缺少真实字幕资产")
|
||||
target = tmp_path / "original.srt"
|
||||
shutil.copy2(source, target)
|
||||
return target
|
||||
|
||||
|
||||
def _seed(db, source, finals):
|
||||
"""登记已完成节点的真实产物，模拟节点执行结束后的断点。"""
|
||||
definition = {"name": "finalization", "version": 1,
|
||||
"nodes": [{"id": "step", "node_type": "echo"}],
|
||||
"final_outputs": finals}
|
||||
db.upsert_workflow({"id": "finalization", "name": "收尾回归"})
|
||||
db.create_workflow_version("finalization", 1, definition)
|
||||
db.create_run({"id": "run_finalization", "workflow_id": "finalization",
|
||||
"workflow_version": 1, "status": "QUEUED", "input_uri": str(source),
|
||||
"created_at": "2026-01-01T00:00:00+00:00", "updated_at": "2026-01-01T00:00:00+00:00"})
|
||||
db.create_artifact({"run_id": "run_finalization", "node_id": "step",
|
||||
"name": "step.file_uri", "uri": str(source)})
|
||||
return WorkflowDefinition.from_dict(definition)
|
||||
|
||||
|
||||
def test_pause_during_final_registration_keeps_downloads(subtitle, tmp_path, monkeypatch):
|
||||
"""在第一个最终别名登记后暂停，再继续，两别名均可下载且 URI 不漂移。"""
|
||||
db = Database(tmp_path / "test.db")
|
||||
_seed(db, subtitle, {"result": "step.file_uri", "copy": "step.file_uri"})
|
||||
original = subtitle.read_bytes()
|
||||
create = db.create_artifact
|
||||
|
||||
def pause_once(artifact):
|
||||
create(artifact)
|
||||
if artifact["name"] == "result":
|
||||
db.pause_run("run_finalization", "2026-01-01T00:00:00+00:00")
|
||||
|
||||
scheduler = WorkflowScheduler(db, tmp_path / "storage")
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(db, "create_artifact", pause_once)
|
||||
scheduler.execute_run("run_finalization")
|
||||
first_uri = db.get_artifact("run_finalization", "result")["uri"]
|
||||
assert db.get_run("run_finalization")["status"] == "PAUSED"
|
||||
db.resume_run("run_finalization", "2026-01-01T00:00:00+00:00")
|
||||
scheduler.execute_run("run_finalization")
|
||||
assert db.get_run("run_finalization")["status"] == "COMPLETED"
|
||||
assert db.get_artifact("run_finalization", "result")["uri"] == first_uri
|
||||
with TestClient(app) as client:
|
||||
monkeypatch.setattr(app.state, "db", db)
|
||||
for alias in ("result", "copy", "step.file_uri"):
|
||||
response = client.get(f"/api/runs/run_finalization/artifacts/{alias}")
|
||||
assert response.status_code == 200
|
||||
assert response.content == original
|
||||
|
||||
|
||||
def test_legacy_renamed_final_survives_resume(subtitle, tmp_path):
|
||||
"""兼容旧版已改名但最终记录有效的断点，恢复不能覆盖成不存在的旧路径。"""
|
||||
db = Database(tmp_path / "test.db")
|
||||
_seed(db, subtitle, {"result": "step.file_uri"})
|
||||
renamed = subtitle.with_name("legacy.srt")
|
||||
subtitle.rename(renamed)
|
||||
db.create_artifact({"run_id": "run_finalization", "node_id": "step",
|
||||
"name": "result", "uri": str(renamed)})
|
||||
WorkflowScheduler(db, tmp_path / "storage").execute_run("run_finalization")
|
||||
assert db.get_run("run_finalization")["status"] == "COMPLETED"
|
||||
assert Path(db.get_artifact("run_finalization", "result")["uri"]).read_bytes() == renamed.read_bytes()
|
||||
|
||||
|
||||
def test_failed_final_copy_preserves_source_and_can_retry(subtitle, tmp_path, monkeypatch):
|
||||
"""复制中途写入失败不登记残缺成品，源字幕可用，恢复后可成功收尾。"""
|
||||
db = Database(tmp_path / "test.db")
|
||||
_seed(db, subtitle, {"result": "step.file_uri"})
|
||||
expected = subtitle.read_bytes()
|
||||
scheduler = WorkflowScheduler(db, tmp_path / "storage")
|
||||
|
||||
def fail_copy(source, destination, **kwargs):
|
||||
# 文件 I/O 边界模拟磁盘写满：真实写出截断内容后报错。
|
||||
Path(destination).write_bytes(Path(source).read_bytes()[:64])
|
||||
raise OSError("disk full")
|
||||
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(shutil, "copy2", fail_copy)
|
||||
scheduler.execute_run("run_finalization")
|
||||
assert db.get_run("run_finalization")["status"] == "FAILED"
|
||||
assert db.get_artifact("run_finalization", "result") is None
|
||||
assert subtitle.read_bytes() == expected
|
||||
assert not list((tmp_path / "storage").rglob("*.tmp"))
|
||||
db.update_run("run_finalization", status="QUEUED", updated_at="2026-01-01T00:00:00+00:00")
|
||||
scheduler.execute_run("run_finalization")
|
||||
assert db.get_run("run_finalization")["status"] == "COMPLETED"
|
||||
assert Path(db.get_artifact("run_finalization", "result")["uri"]).read_bytes() == expected
|
||||
|
||||
|
||||
def test_batch_copy_failure_preserves_old_product_and_workspace(subtitle, tmp_path, monkeypatch):
|
||||
"""批量成品写到一半失败：旧字幕不变、run/工作空间保留，重试可收尾。"""
|
||||
from wov_app.batch import BatchWorker, create_job
|
||||
|
||||
video_asset = Path(__file__).resolve().parent.parent / "testdata/subtitle_10s.mp4"
|
||||
if not video_asset.is_file():
|
||||
pytest.skip("缺少真实视频资产")
|
||||
library = tmp_path / "library"
|
||||
library.mkdir()
|
||||
video = library / "movie.mp4"
|
||||
shutil.copy2(video_asset, video)
|
||||
db = Database(tmp_path / "test.db")
|
||||
_seed(db, subtitle, {"result": "step.file_uri"})
|
||||
db.upsert_workflow({"id": "finalization", "name": "收尾回归", "published": 1, "latest_version": 1})
|
||||
job_id = create_job(db, str(library), "finalization")
|
||||
item = db.list_batch_videos(job_id)[0]
|
||||
work_dir = Path(item["work_dir"])
|
||||
work_dir.mkdir(parents=True)
|
||||
source = work_dir / "result.srt"
|
||||
shutil.copy2(subtitle, source)
|
||||
now = "2026-01-01T00:00:00+00:00"
|
||||
db.update_batch_video(item["id"], run_id="run_finalization", updated_at=now)
|
||||
db.update_run("run_finalization", status="COMPLETED", updated_at=now)
|
||||
db.create_artifact({"run_id": "run_finalization", "node_id": "step",
|
||||
"name": "result", "uri": str(source)})
|
||||
target = library / "movie.CN.srt"
|
||||
shutil.copy2(subtitle, target)
|
||||
original = target.read_bytes()
|
||||
worker = BatchWorker(db)
|
||||
|
||||
def fail_copy(source, destination, **kwargs):
|
||||
Path(destination).write_bytes(Path(source).read_bytes()[:64])
|
||||
raise OSError("disk full")
|
||||
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setattr(shutil, "copy2", fail_copy)
|
||||
worker._process_job(db.get_batch_job(job_id))
|
||||
assert db.get_batch_video(item["id"])["status"] == "FAILED"
|
||||
assert target.read_bytes() == original
|
||||
assert db.get_run("run_finalization") is not None
|
||||
assert source.read_bytes() == original
|
||||
assert not list(library.glob("*.tmp"))
|
||||
worker._process_job(db.get_batch_job(job_id))
|
||||
assert db.get_batch_video(item["id"])["status"] == "COMPLETED"
|
||||
assert db.get_run("run_finalization") is None
|
||||
assert not work_dir.exists()
|
||||
assert target.read_bytes() == original
|
||||
assert video.read_bytes() == video_asset.read_bytes()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing_ref", [False, True])
|
||||
def test_missing_final_fails_run(subtitle, tmp_path, missing_ref):
|
||||
"""必需最终输出无引用或文件丢失都应 FAILED，不能成功登记失效链接。"""
|
||||
db = Database(tmp_path / "test.db")
|
||||
_seed(db, subtitle, {"result": "step.missing" if missing_ref else "step.file_uri"})
|
||||
if not missing_ref:
|
||||
subtitle.unlink()
|
||||
WorkflowScheduler(db, tmp_path / "storage").execute_run("run_finalization")
|
||||
run = db.get_run("run_finalization")
|
||||
assert run["status"] == "FAILED"
|
||||
assert "result" in run["error"]
|
||||
@@ -429,9 +429,9 @@ def test_final_artifact_renamed_with_language_tag(tmp_path) -> None:
|
||||
assert filename.startswith("movie01.zh-CN.")
|
||||
assert filename.endswith(".txt")
|
||||
assert Path(final["uri"]).is_file()
|
||||
# 原始未重命名文件不应残留。
|
||||
# 节点原始 URI 必须保持有效；成品副本与原始文本一致，保证断点可恢复。
|
||||
step_artifacts = [item for item in artifacts if item["name"] == "step.file_uri"]
|
||||
assert not Path(step_artifacts[0]["uri"]).exists()
|
||||
assert Path(step_artifacts[0]["uri"]).read_bytes() == Path(final["uri"]).read_bytes()
|
||||
|
||||
|
||||
def test_final_artifact_renamed_fallback_base_and_tag(tmp_path) -> None:
|
||||
|
||||
Reference in New Issue
Block a user