feat: VRSub 单体应用（WOV 单机版）初始提交
为视频生成 VR 双眼字幕的单体实现：FastAPI 后端、调度器与全部节点 （提音/转写/翻译/ASS/抽帧/OCR/LLM 过滤）在单进程内运行。 - 节点协议（wov_sdk 数据模型）与分布式版保持一致，预留回退桥梁 - 工作流即数据：DAG 存于 workflows/*.json，模型/链路改动只改数据 - 调度器：拓扑顺序执行、断点续跑（产物重建）、任务暂停/继续 - 抽帧按帧间隔（select 按帧号精确取帧），VLM OCR 与 LLM 过滤使用 自适应线程池弹性并发，并打印数据处理速度进度日志 - 100% 行覆盖率（pytest --cov-fail-under=100）
This commit is contained in:
@@ -0,0 +1,5 @@
|
||||
"""WOV 单体应用包。
|
||||
|
||||
包含 FastAPI 应用入口、SQLite 数据访问、进程内节点注册表、工作流调度器
|
||||
以及管理端/用户端路由。所有节点在同一进程内直接调用，无子进程边界。
|
||||
"""
|
||||
@@ -0,0 +1,28 @@
|
||||
"""应用配置中心。
|
||||
|
||||
集中读取环境变量并推导路径常量，避免业务代码散落魔法值。路径统一使用
|
||||
pathlib，Windows 与 Linux 开发环境均可用。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# WOV 单体根目录：本文件位于 src/wov_app/config.py，向上三级即仓库根。
|
||||
WORKSPACE_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
# 数据目录、SQLite 文件与产物存储目录均可通过环境变量覆盖，便于测试隔离。
|
||||
DATA_DIR = Path(os.getenv("WOV_DATA_DIR", str(WORKSPACE_ROOT / "data")))
|
||||
DB_PATH = Path(os.getenv("WOV_DB_PATH", str(DATA_DIR / "wov.db")))
|
||||
STORAGE_DIR = Path(os.getenv("WOV_STORAGE_DIR", str(DATA_DIR / "storage")))
|
||||
|
||||
# 调度器轮询排队任务的间隔（秒）。
|
||||
SCHEDULER_INTERVAL_SECONDS = float(os.getenv("WOV_SCHEDULER_INTERVAL_SECONDS", "1.0"))
|
||||
|
||||
# 孤儿数据清理器配置：定时扫描并清理无对应文件/记录的死数据。
|
||||
CLEANUP_ENABLED = os.getenv("WOV_CLEANUP_ENABLED", "1") == "1"
|
||||
# 清理扫描周期（秒），默认每小时一次。
|
||||
CLEANUP_INTERVAL_SECONDS = float(os.getenv("WOV_CLEANUP_INTERVAL_SECONDS", "3600"))
|
||||
# 宽限期（秒）：任务最后更新距今超过该时长且满足孤儿条件才清理。
|
||||
CLEANUP_GRACE_SECONDS = float(os.getenv("WOV_CLEANUP_GRACE_SECONDS", "3600"))
|
||||
@@ -0,0 +1,398 @@
|
||||
"""SQLite 数据访问层。
|
||||
|
||||
所有持久化逻辑集中在本模块，业务代码只依赖 Database 提供的方法。后续切换
|
||||
PostgreSQL 时只需替换本层实现，不修改调度器与路由的业务逻辑。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
|
||||
|
||||
class Database:
|
||||
"""SQLite 数据库封装：负责建表以及工作流/任务/产物的 CRUD。"""
|
||||
|
||||
def __init__(self, path: Path) -> None:
|
||||
"""打开数据库并确保父目录存在、表结构已初始化。"""
|
||||
self.path = path
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._init_schema()
|
||||
|
||||
@contextmanager
|
||||
def _connect(self) -> Iterator[sqlite3.Connection]:
|
||||
"""提供带事务提交的数据库连接上下文。"""
|
||||
conn = sqlite3.connect(self.path)
|
||||
# 按列名读取结果，返回 dict 更直观。
|
||||
conn.row_factory = sqlite3.Row
|
||||
# 开启外键约束，保证子表记录引用有效。
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
"""创建全部业务表；已存在的表保持不变。"""
|
||||
with self._connect() as conn:
|
||||
conn.executescript(
|
||||
"""
|
||||
-- 工作流表：只保存概要信息，完整定义存版本表。
|
||||
CREATE TABLE IF NOT EXISTS workflows (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
published INTEGER NOT NULL DEFAULT 0,
|
||||
latest_version INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- 工作流版本表：每个版本保存一份 DAG 定义 JSON。
|
||||
CREATE TABLE IF NOT EXISTS workflow_versions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
workflow_id TEXT NOT NULL,
|
||||
version INTEGER NOT NULL,
|
||||
definition_json TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(workflow_id, version),
|
||||
FOREIGN KEY(workflow_id) REFERENCES workflows(id)
|
||||
);
|
||||
|
||||
-- 工作流运行表：记录任务从排队到完成/失败的状态机。
|
||||
CREATE TABLE IF NOT EXISTS workflow_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
workflow_id TEXT NOT NULL,
|
||||
workflow_version INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
current_node_id TEXT,
|
||||
progress REAL NOT NULL DEFAULT 0,
|
||||
error TEXT,
|
||||
input_uri TEXT,
|
||||
param_overrides TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
FOREIGN KEY(workflow_id) REFERENCES workflows(id)
|
||||
);
|
||||
|
||||
-- 产物表：记录每个任务各节点的输出 URI，按名称唯一。
|
||||
CREATE TABLE IF NOT EXISTS artifacts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id TEXT NOT NULL,
|
||||
node_id TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
uri TEXT NOT NULL,
|
||||
mime_type TEXT NOT NULL DEFAULT '',
|
||||
size INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(run_id, name),
|
||||
FOREIGN KEY(run_id) REFERENCES workflow_runs(id)
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
# 旧库迁移：workflow_runs 补充 param_overrides 列（前端框选覆盖）。
|
||||
columns = [
|
||||
row["name"]
|
||||
for row in conn.execute("PRAGMA table_info(workflow_runs)").fetchall()
|
||||
]
|
||||
if "param_overrides" not in columns:
|
||||
conn.execute("ALTER TABLE workflow_runs ADD COLUMN param_overrides TEXT")
|
||||
|
||||
def upsert_workflow(self, workflow: dict[str, Any]) -> None:
|
||||
"""插入或更新工作流概要信息。"""
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO workflows (id, name, description, published, latest_version)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
description = excluded.description,
|
||||
published = excluded.published,
|
||||
latest_version = excluded.latest_version
|
||||
""",
|
||||
(
|
||||
workflow["id"],
|
||||
workflow["name"],
|
||||
workflow.get("description", ""),
|
||||
int(workflow.get("published", 0)),
|
||||
int(workflow.get("latest_version", 0)),
|
||||
),
|
||||
)
|
||||
|
||||
def get_workflow(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
"""按 ID 读取工作流概要。"""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute("SELECT * FROM workflows WHERE id = ?", (workflow_id,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def list_workflows(self) -> list[dict[str, Any]]:
|
||||
"""按创建时间倒序返回全部工作流。"""
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM workflows ORDER BY created_at DESC").fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def delete_workflow(self, workflow_id: str) -> None:
|
||||
"""级联删除工作流相关的产物、任务、版本和概要记录。"""
|
||||
with self._connect() as conn:
|
||||
# 外键没有级联删除配置，手动按依赖顺序清理。
|
||||
conn.execute("DELETE FROM artifacts WHERE run_id IN (SELECT id FROM workflow_runs WHERE workflow_id = ?)", (workflow_id,))
|
||||
conn.execute("DELETE FROM workflow_runs WHERE workflow_id = ?", (workflow_id,))
|
||||
conn.execute("DELETE FROM workflow_versions WHERE workflow_id = ?", (workflow_id,))
|
||||
conn.execute("DELETE FROM workflows WHERE id = ?", (workflow_id,))
|
||||
|
||||
def create_workflow_version(self, workflow_id: str, version: int, definition: dict[str, Any]) -> None:
|
||||
"""为工作流新增一个版本，definition 以 JSON 保存。"""
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO workflow_versions (workflow_id, version, definition_json)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
(workflow_id, version, json.dumps(definition, ensure_ascii=False)),
|
||||
)
|
||||
|
||||
def get_latest_workflow_version(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
"""返回工作流最新版本，并把 definition_json 反序列化为 definition。"""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM workflow_versions
|
||||
WHERE workflow_id = ?
|
||||
ORDER BY version DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(workflow_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
result = dict(row)
|
||||
# 对外统一暴露 definition 字典，隐藏 JSON 存储细节。
|
||||
result["definition"] = json.loads(result.pop("definition_json"))
|
||||
return result
|
||||
|
||||
def get_workflow_version(self, workflow_id: str, version: int) -> dict[str, Any] | None:
|
||||
"""按版本号读取指定工作流版本。"""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM workflow_versions
|
||||
WHERE workflow_id = ? AND version = ?
|
||||
""",
|
||||
(workflow_id, version),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
result = dict(row)
|
||||
result["definition"] = json.loads(result.pop("definition_json"))
|
||||
return result
|
||||
|
||||
def list_workflow_versions(self, workflow_id: str) -> list[dict[str, Any]]:
|
||||
"""按版本倒序返回工作流全部版本。"""
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM workflow_versions
|
||||
WHERE workflow_id = ?
|
||||
ORDER BY version DESC
|
||||
""",
|
||||
(workflow_id,),
|
||||
).fetchall()
|
||||
versions = []
|
||||
for row in rows:
|
||||
item = dict(row)
|
||||
item["definition"] = json.loads(item.pop("definition_json"))
|
||||
versions.append(item)
|
||||
return versions
|
||||
|
||||
def create_run(self, run: dict[str, Any]) -> None:
|
||||
"""创建一条排队中的工作流运行记录。"""
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO workflow_runs (
|
||||
id, workflow_id, workflow_version, status, current_node_id,
|
||||
progress, error, input_uri, param_overrides, created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
run["id"],
|
||||
run["workflow_id"],
|
||||
run["workflow_version"],
|
||||
run["status"],
|
||||
run.get("current_node_id"),
|
||||
float(run.get("progress", 0)),
|
||||
run.get("error"),
|
||||
run.get("input_uri"),
|
||||
json.dumps(run["param_overrides"], ensure_ascii=False)
|
||||
if run.get("param_overrides")
|
||||
else None,
|
||||
run["created_at"],
|
||||
run["updated_at"],
|
||||
),
|
||||
)
|
||||
|
||||
def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
"""按 ID 读取任务运行记录。"""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute("SELECT * FROM workflow_runs WHERE id = ?", (run_id,)).fetchone()
|
||||
return self._parse_overrides(row) if row else None
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _parse_overrides(row) -> dict:
|
||||
"""把查询行中的 param_overrides JSON 字符串解析为字典。"""
|
||||
result = dict(row)
|
||||
raw = result.get("param_overrides")
|
||||
result["param_overrides"] = json.loads(raw) if raw else None
|
||||
return result
|
||||
|
||||
def list_runs(self, limit: int = 20) -> list[dict[str, Any]]:
|
||||
"""按创建时间倒序返回最近的运行记录。"""
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM workflow_runs ORDER BY created_at DESC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [self._parse_overrides(row) for row in rows]
|
||||
|
||||
def list_run_ids(self) -> list[str]:
|
||||
"""返回全部任务 ID，供孤儿数据清理对照磁盘目录使用。"""
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute("SELECT id FROM workflow_runs").fetchall()
|
||||
return [row["id"] for row in rows]
|
||||
|
||||
def update_run(self, run_id: str, **fields: Any) -> None:
|
||||
"""更新运行状态字段，同时刷新 updated_at；未知字段会被忽略。"""
|
||||
# 只允许更新状态机相关字段，防止任意列被改写。
|
||||
allowed = {
|
||||
"status",
|
||||
"current_node_id",
|
||||
"progress",
|
||||
"error",
|
||||
}
|
||||
updates = {key: value for key, value in fields.items() if key in allowed}
|
||||
if not updates:
|
||||
return
|
||||
updates["updated_at"] = fields.get("updated_at")
|
||||
# 动态拼接 SET 子句，键来自白名单，不存在 SQL 注入风险。
|
||||
assignments = ", ".join(f"{key} = ?" for key in updates)
|
||||
values = list(updates.values()) + [run_id]
|
||||
with self._connect() as conn:
|
||||
conn.execute(f"UPDATE workflow_runs SET {assignments} WHERE id = ?", values)
|
||||
|
||||
def reset_run(self, run_id: str, updated_at: str) -> None:
|
||||
"""把失败任务重置为 QUEUED，并清空进度与旧产物，供重试使用。"""
|
||||
with self._connect() as conn:
|
||||
# 清空错误和进度，恢复到首次排队时的状态。
|
||||
conn.execute(
|
||||
"""
|
||||
UPDATE workflow_runs
|
||||
SET status = 'QUEUED', current_node_id = NULL, progress = 0,
|
||||
error = NULL, updated_at = ?
|
||||
WHERE id = ?
|
||||
""",
|
||||
(updated_at, run_id),
|
||||
)
|
||||
# 删除旧产物，避免重试后残留过期下载链接。
|
||||
conn.execute("DELETE FROM artifacts WHERE run_id = ?", (run_id,))
|
||||
|
||||
def next_queued_run(self) -> dict[str, Any] | None:
|
||||
"""按创建时间返回最早一条可执行任务（排队或已暂停待续跑）。"""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM workflow_runs
|
||||
WHERE status IN ('QUEUED', 'PAUSED')
|
||||
ORDER BY created_at ASC
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
return self._parse_overrides(row) if row else None
|
||||
|
||||
def pause_run(self, run_id: str, updated_at: str) -> None:
|
||||
"""暂停任务：置为 PAUSED；调度器会在节点边界检查并停止推进。"""
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE workflow_runs SET status = 'PAUSED', updated_at = ? WHERE id = ?",
|
||||
(updated_at, run_id),
|
||||
)
|
||||
|
||||
def resume_run(self, run_id: str, updated_at: str) -> None:
|
||||
"""继续任务：PAUSED 恢复为 QUEUED，等待调度器从断点续跑。"""
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"UPDATE workflow_runs SET status = 'QUEUED', updated_at = ? WHERE id = ?",
|
||||
(updated_at, run_id),
|
||||
)
|
||||
|
||||
def restore_run_outputs(self, run_id: str) -> dict[str, dict[str, str]]:
|
||||
"""从已登记的产物重建各节点输出，供暂停后断点续跑使用。
|
||||
|
||||
返回 {节点ID: {输出名: URI}}；已完成节点的产物可直接作为后续节点的输入。
|
||||
"""
|
||||
outputs: dict[str, dict[str, str]] = {}
|
||||
for artifact in self.list_artifacts(run_id):
|
||||
name = artifact["name"]
|
||||
# 产物名形如 "节点ID.输出名"（如 a.data_uri），还原为 {输出名: URI}。
|
||||
prefix = artifact["node_id"] + "."
|
||||
if name.startswith(prefix):
|
||||
name = name[len(prefix):]
|
||||
outputs.setdefault(artifact["node_id"], {})[name] = artifact["uri"]
|
||||
return outputs
|
||||
def create_artifact(self, artifact: dict[str, Any]) -> None:
|
||||
"""记录任务产物；同 run 与 name 冲突时覆盖。"""
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO artifacts (
|
||||
run_id, node_id, name, uri, mime_type, size
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
artifact["run_id"],
|
||||
artifact["node_id"],
|
||||
artifact["name"],
|
||||
artifact["uri"],
|
||||
artifact.get("mime_type", ""),
|
||||
int(artifact.get("size", 0)),
|
||||
),
|
||||
)
|
||||
|
||||
def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
"""按创建时间返回任务的全部产物。"""
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM artifacts WHERE run_id = ? ORDER BY created_at",
|
||||
(run_id,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def get_artifact(self, run_id: str, name: str) -> dict[str, Any] | None:
|
||||
"""按任务与产物名读取单个产物记录。"""
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM artifacts WHERE run_id = ? AND name = ?",
|
||||
(run_id, name),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def delete_run_artifacts(self, run_id: str) -> None:
|
||||
"""删除任务的全部产物记录。"""
|
||||
with self._connect() as conn:
|
||||
conn.execute("DELETE FROM artifacts WHERE run_id = ?", (run_id,))
|
||||
|
||||
def delete_run(self, run_id: str) -> None:
|
||||
"""删除任务记录本身及其产物记录。
|
||||
|
||||
产物表外键引用任务表，必须先删产物再删任务，否则违反外键约束。
|
||||
"""
|
||||
with self._connect() as conn:
|
||||
conn.execute("DELETE FROM artifacts WHERE run_id = ?", (run_id,))
|
||||
conn.execute("DELETE FROM workflow_runs WHERE id = ?", (run_id,))
|
||||
@@ -0,0 +1,31 @@
|
||||
"""轻量日志配置。
|
||||
|
||||
单体版所有节点在 API 主进程内运行，这里把节点运行日志直接输出到主进程
|
||||
控制台（uvicorn 的 stderr），便于观察各节点的执行过程与耗时。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
# 应用日志统一前缀，便于与其他库日志区分。
|
||||
_APP_LOGGER_NAME = "vrsub"
|
||||
|
||||
|
||||
def _ensure_console_handler(logger: logging.Logger) -> None:
|
||||
"""为日志器附加控制台输出；已配置过则跳过，避免重复打印。"""
|
||||
if any(isinstance(handler, logging.StreamHandler) for handler in logger.handlers):
|
||||
return
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(logging.INFO)
|
||||
# 不向 uvicorn 根日志传播，防止消息重复输出。
|
||||
logger.propagate = False
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""获取并确保输出到主进程控制台的日志器。"""
|
||||
logger = logging.getLogger(f"{_APP_LOGGER_NAME}.{name}")
|
||||
_ensure_console_handler(logger)
|
||||
return logger
|
||||
@@ -0,0 +1,82 @@
|
||||
"""FastAPI 应用入口。
|
||||
|
||||
负责组装数据库、进程内节点注册表、调度器与静态前端，并在应用生命周期内
|
||||
管理后台线程的启动与清理。单体版所有节点在同一进程内直接调用。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# 先加载 .env（含 LLM API Key 等本地配置），再导入读取环境变量的 config。
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
import os # noqa: E402
|
||||
from contextlib import asynccontextmanager # noqa: E402
|
||||
from pathlib import Path # noqa: E402
|
||||
|
||||
from fastapi import FastAPI # noqa: E402
|
||||
from fastapi.middleware.cors import CORSMiddleware # noqa: E402
|
||||
from fastapi.staticfiles import StaticFiles # noqa: E402
|
||||
|
||||
from wov_app import registry
|
||||
from wov_app.config import DB_PATH, STORAGE_DIR, WORKSPACE_ROOT
|
||||
from wov_app.db import Database
|
||||
from wov_app.maintenance import OrphanCleaner
|
||||
from wov_app.routers import apps, workflows
|
||||
from wov_app.scheduler import WorkflowScheduler
|
||||
from wov_app.seed import seed_default_workflows
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""应用生命周期：启动时初始化数据、注册节点和后台服务，退出时回收资源。"""
|
||||
# 确保数据与存储目录存在，避免首次启动写文件失败。
|
||||
db = Database(DB_PATH)
|
||||
STORAGE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
# 静态注册全部内置节点到进程内注册表（内存态，不落库）。
|
||||
registry.register_all()
|
||||
# 默认创建演示工作流，可关闭便于测试。
|
||||
if os.getenv("WOV_AUTO_SEED", "1") == "1":
|
||||
seed_default_workflows(db)
|
||||
|
||||
scheduler = WorkflowScheduler(db, STORAGE_DIR)
|
||||
# 调度器默认开启，处理排队中的任务；测试可关闭后手动执行。
|
||||
if os.getenv("WOV_SCHEDULER_ENABLED", "1") == "1":
|
||||
scheduler.start()
|
||||
cleaner = OrphanCleaner(db, STORAGE_DIR)
|
||||
# 孤儿数据清理默认开启，定时清除死数据；测试可关闭。
|
||||
if os.getenv("WOV_CLEANUP_ENABLED", "1") == "1":
|
||||
cleaner.start()
|
||||
# 共享对象挂到 app.state，路由通过 Depends 延迟获取。
|
||||
app.state.db = db
|
||||
app.state.scheduler = scheduler
|
||||
app.state.cleaner = cleaner
|
||||
yield
|
||||
# 退出时先停调度器与清理器，避免残留后台线程。
|
||||
cleaner.stop()
|
||||
scheduler.stop()
|
||||
|
||||
|
||||
app = FastAPI(title="VRSub API（单体版）", version="0.1.0", lifespan=lifespan)
|
||||
|
||||
# MVP 阶段不做鉴权，允许跨域便于本地调试与静态页面访问。
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(workflows.router)
|
||||
app.include_router(apps.router)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict:
|
||||
"""进程存活探针，供部署环境与前端检测后端可用性。"""
|
||||
return {"status": "ok", "service": "wov-api", "mode": "monolith"}
|
||||
|
||||
|
||||
# 静态前端目录位于仓库根下的 web，由 FastAPI 直接挂载。
|
||||
FRONTEND_DIR = WORKSPACE_ROOT / "web"
|
||||
app.mount("/", StaticFiles(directory=FRONTEND_DIR, html=True), name="frontend")
|
||||
@@ -0,0 +1,129 @@
|
||||
"""孤儿数据清理器。
|
||||
|
||||
定时扫描存储目录与数据库，清理不再有意义的死数据：
|
||||
|
||||
1. 磁盘上存在但没有对应任务记录的上传/步骤目录（删除任务中断等残留）。
|
||||
2. 状态为 COMPLETED 但产物文件已全部丢失、且超过宽限期的任务记录
|
||||
（这类任务在任务页会显示"完成"但下载全部 404，属于孤儿数据）。
|
||||
|
||||
出于安全考虑，以下数据**不会**被自动清理：
|
||||
|
||||
- FAILED 任务（用户可能重试，且失败任务本就可能没有文件）。
|
||||
- 状态非终态（QUEUED/RUNNING）的任务。
|
||||
- 最近宽限期内的任务，避免误删刚完成的运行。
|
||||
|
||||
手动删除任务仍走删除接口，本模块只做保守的孤儿兜底。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from wov_app.config import CLEANUP_GRACE_SECONDS, CLEANUP_INTERVAL_SECONDS
|
||||
from wov_app.db import Database
|
||||
|
||||
|
||||
class OrphanCleaner:
|
||||
"""后台孤儿清理器：周期扫描并清理孤儿数据，只保留明确的死数据。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db: Database,
|
||||
storage_dir: Path,
|
||||
interval_seconds: float | None = None,
|
||||
grace_seconds: float | None = None,
|
||||
) -> None:
|
||||
"""保存依赖并初始化轮询线程控制字段。"""
|
||||
self.db = db
|
||||
self.storage_dir = storage_dir
|
||||
self.interval_seconds = interval_seconds or CLEANUP_INTERVAL_SECONDS
|
||||
self.grace_seconds = grace_seconds or CLEANUP_GRACE_SECONDS
|
||||
self._thread: threading.Thread | None = None
|
||||
self._stopping = False
|
||||
|
||||
def start(self) -> None:
|
||||
"""启动清理线程；重复调用无副作用。"""
|
||||
if self._thread is not None:
|
||||
return
|
||||
self._stopping = False
|
||||
self._thread = threading.Thread(
|
||||
target=self._loop,
|
||||
name="wov-orphan-cleaner",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
"""请求停止并等待清理线程退出。"""
|
||||
self._stopping = True
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=5)
|
||||
self._thread = None
|
||||
|
||||
def _loop(self) -> None:
|
||||
"""周期循环：每隔一个间隔执行一次清理。"""
|
||||
while not self._stopping:
|
||||
time.sleep(self.interval_seconds)
|
||||
self.clean_once()
|
||||
|
||||
def clean_once(self) -> int:
|
||||
"""执行一次孤儿清理，返回清理的数据条目数。"""
|
||||
run_ids = set(self.db.list_run_ids())
|
||||
removed = 0
|
||||
# 1) 无对应任务记录的上传/步骤目录视为残留，直接删除。
|
||||
removed += self._clean_dangling(self.storage_dir / "uploads", run_ids)
|
||||
removed += self._clean_dangling(self.storage_dir / "runs", run_ids)
|
||||
# 2) COMPLETED 且产物文件全失、超过宽限期的任务记录删除。
|
||||
for run_id in run_ids:
|
||||
run = self.db.get_run(run_id)
|
||||
if run is None:
|
||||
continue
|
||||
if run["status"] != "COMPLETED":
|
||||
continue
|
||||
if not self._expired(run.get("updated_at")):
|
||||
continue
|
||||
if self._has_files(self.storage_dir / "runs" / run_id):
|
||||
continue
|
||||
removed += self._remove_run(run_id, run)
|
||||
return removed
|
||||
|
||||
def _clean_dangling(self, root: Path, run_ids: set[str]) -> int:
|
||||
"""删除 root 下没有对应任务记录的残留子目录，返回删除数。"""
|
||||
if not root.is_dir():
|
||||
return 0
|
||||
removed = 0
|
||||
for child in root.iterdir():
|
||||
if child.is_dir() and child.name not in run_ids:
|
||||
shutil.rmtree(child, ignore_errors=True)
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
def _expired(self, updated_at: str | None) -> bool:
|
||||
"""判断任务最后更新时间是否已超过宽限期；无法解析时保守视为未过期。"""
|
||||
if not updated_at:
|
||||
return False
|
||||
try:
|
||||
updated = datetime.fromisoformat(updated_at)
|
||||
return (datetime.now(timezone.utc) - updated).total_seconds() > self.grace_seconds
|
||||
except ValueError:
|
||||
# 时间格式损坏时保守保留，避免误删。
|
||||
return False
|
||||
|
||||
def _has_files(self, run_dir: Path) -> bool:
|
||||
"""判断任务目录下是否仍存在产物文件。"""
|
||||
if not run_dir.is_dir():
|
||||
return False
|
||||
return any(path.is_file() for path in run_dir.rglob("*"))
|
||||
|
||||
def _remove_run(self, run_id: str, run: dict) -> int:
|
||||
"""删除孤儿任务：数据库记录（含产物）、上传目录与步骤目录。"""
|
||||
self.db.delete_run(run_id)
|
||||
input_uri = run.get("input_uri")
|
||||
if input_uri:
|
||||
shutil.rmtree(Path(input_uri).parent, ignore_errors=True)
|
||||
shutil.rmtree(self.storage_dir / "runs" / run_id, ignore_errors=True)
|
||||
return 1
|
||||
@@ -0,0 +1,110 @@
|
||||
"""进程内节点注册表。
|
||||
|
||||
单体版不再启动子进程：节点清单与 invoke 处理器在启动时静态注册到本模块，
|
||||
调度器通过 invoke(node_id, request) 在同一个进程内直接调用处理器。
|
||||
协议数据模型（NodeManifest / InvokeRequest / InvokeResponse）保持不变，
|
||||
为将来回退分布式保留兼容桥梁。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from wov_app.logging import get_logger
|
||||
from wov_sdk.models import InvokeRequest, InvokeResponse, NodeManifest
|
||||
|
||||
# 节点运行日志：输出到主进程控制台。
|
||||
logger = get_logger("node")
|
||||
|
||||
# 节点调用处理器签名：接收调用请求，返回调用结果。
|
||||
NodeHandler = Callable[[InvokeRequest], InvokeResponse]
|
||||
|
||||
|
||||
@dataclass
|
||||
class NodeEntry:
|
||||
"""注册表条目：节点清单与其进程内处理器。"""
|
||||
|
||||
manifest: NodeManifest
|
||||
handler: NodeHandler
|
||||
|
||||
|
||||
# 进程内注册表：key 为节点 ID（即工作流中的 node_type），value 为注册条目。
|
||||
_registry: dict[str, NodeEntry] = {}
|
||||
|
||||
|
||||
def _load_manifest(name: str) -> NodeManifest:
|
||||
"""从 manifests/ 目录加载节点清单文件并校验。"""
|
||||
path = Path(__file__).resolve().parent.parent.parent / "manifests" / f"{name}.json"
|
||||
return NodeManifest.load(str(path))
|
||||
|
||||
|
||||
def register(manifest: NodeManifest, handler: NodeHandler) -> None:
|
||||
"""注册单个节点；manifest 校验失败时抛出 ValueError。"""
|
||||
manifest.validate()
|
||||
_registry[manifest.id] = NodeEntry(manifest=manifest, handler=handler)
|
||||
|
||||
|
||||
def register_all() -> None:
|
||||
"""注册全部内置节点，启动时调用一次；重复调用按 ID 覆盖，幂等。"""
|
||||
from nodes import ass, echo, ffmpeg, frame_extract, llm, llm_filter, subtitle_ocr, vlm, whisper
|
||||
register(_load_manifest("echo"), echo.invoke)
|
||||
register(_load_manifest("ffmpeg"), ffmpeg.invoke)
|
||||
register(_load_manifest("whisper"), whisper.invoke)
|
||||
register(_load_manifest("llm"), llm.invoke)
|
||||
register(_load_manifest("vlm"), vlm.invoke)
|
||||
register(_load_manifest("frame-extract"), frame_extract.invoke)
|
||||
register(_load_manifest("subtitle-ocr"), subtitle_ocr.invoke)
|
||||
register(_load_manifest("llm-filter"), llm_filter.invoke)
|
||||
register(_load_manifest("ass"), ass.invoke)
|
||||
|
||||
|
||||
def list_nodes() -> list[NodeManifest]:
|
||||
"""按节点 ID 顺序返回全部已注册节点清单。"""
|
||||
return [entry.manifest for _, entry in sorted(_registry.items())]
|
||||
|
||||
|
||||
def get_node(node_id: str) -> NodeManifest | None:
|
||||
"""按节点 ID 返回清单；未注册时返回 None。"""
|
||||
entry = _registry.get(node_id)
|
||||
return entry.manifest if entry else None
|
||||
|
||||
|
||||
def invoke(node_id: str, request: InvokeRequest) -> InvokeResponse:
|
||||
"""调用指定节点的进程内处理器；节点未注册时抛出 ValueError。
|
||||
|
||||
统一在这里记录节点的开始/结束/耗时/产物日志，所有节点自动获得
|
||||
主进程可见的运行日志，无需在各节点实现内重复埋点。
|
||||
"""
|
||||
entry = _registry.get(node_id)
|
||||
if entry is None:
|
||||
raise ValueError(f"node not registered: {node_id}")
|
||||
logger.info(
|
||||
"节点 %s 开始 run=%s inputs=%s params=%s",
|
||||
node_id,
|
||||
request.run_id,
|
||||
request.inputs,
|
||||
request.params,
|
||||
)
|
||||
start = time.perf_counter()
|
||||
response = entry.handler(request)
|
||||
elapsed = time.perf_counter() - start
|
||||
if response.status == "completed":
|
||||
logger.info(
|
||||
"节点 %s 完成 run=%s 耗时=%.2fs outputs=%s",
|
||||
node_id,
|
||||
request.run_id,
|
||||
elapsed,
|
||||
response.outputs,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"节点 %s 失败 run=%s 耗时=%.2fs error=%s",
|
||||
node_id,
|
||||
request.run_id,
|
||||
elapsed,
|
||||
response.error,
|
||||
)
|
||||
return response
|
||||
@@ -0,0 +1,5 @@
|
||||
"""WOV 单体 API 路由包。
|
||||
|
||||
按职责拆分为用户应用与工作流管理两组路由，统一由 wov_app.main 挂载。
|
||||
节点注册/实例管理路由已随单体化移除。
|
||||
"""
|
||||
Executable
+213
@@ -0,0 +1,213 @@
|
||||
"""用户端应用路由。
|
||||
|
||||
面向普通用户暴露“应用中心”能力：列出已发布工作流、上传输入创建任务、
|
||||
查询进度、重试失败任务以及下载产物。用户只看到输入 -> 进度 -> 结果。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from wov_app.db import Database
|
||||
|
||||
router = APIRouter(tags=["apps"])
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
"""返回当前 UTC 时间的 ISO 格式字符串。"""
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _get_db() -> Database:
|
||||
"""从 FastAPI 应用状态中延迟获取数据库实例。"""
|
||||
from wov_app.main import app
|
||||
|
||||
return app.state.db
|
||||
|
||||
|
||||
@router.get("/api/apps")
|
||||
def list_apps(db: Database = Depends(_get_db)) -> list[dict]:
|
||||
"""返回全部已发布工作流及其最新版本定义。"""
|
||||
apps = []
|
||||
for workflow in db.list_workflows():
|
||||
# 草稿工作流不对用户端可见。
|
||||
if not workflow["published"]:
|
||||
continue
|
||||
latest = db.get_latest_workflow_version(workflow["id"])
|
||||
apps.append(
|
||||
{
|
||||
"id": workflow["id"],
|
||||
"name": workflow["name"],
|
||||
"description": workflow["description"],
|
||||
"version": workflow["latest_version"],
|
||||
"definition": latest["definition"] if latest else None,
|
||||
}
|
||||
)
|
||||
return apps
|
||||
|
||||
|
||||
@router.post("/api/apps/{workflow_id}/runs")
|
||||
async def create_run(
|
||||
workflow_id: str,
|
||||
file: UploadFile = File(...),
|
||||
params: str = Form(default=""),
|
||||
db: Database = Depends(_get_db),
|
||||
) -> dict:
|
||||
"""接收用户上传文件，创建排队中的工作流任务。"""
|
||||
workflow = db.get_workflow(workflow_id)
|
||||
# 只允许对已发布且存在版本的工作流发起任务。
|
||||
if workflow is None or not workflow["published"]:
|
||||
raise HTTPException(status_code=404, detail="published workflow not found")
|
||||
|
||||
latest = db.get_latest_workflow_version(workflow_id)
|
||||
if latest is None:
|
||||
raise HTTPException(status_code=422, detail="workflow has no version")
|
||||
|
||||
run_id = f"run_{uuid.uuid4().hex[:12]}"
|
||||
# 使用安全文件名，避免路径穿越。
|
||||
filename = Path(file.filename or "upload.bin").name
|
||||
from wov_app.config import STORAGE_DIR
|
||||
|
||||
# 上传文件按 run 隔离存放，调度器通过 input_uri 引用。
|
||||
input_dir = STORAGE_DIR / "uploads" / run_id
|
||||
input_dir.mkdir(parents=True, exist_ok=True)
|
||||
input_uri = input_dir / filename
|
||||
content = await file.read()
|
||||
input_uri.write_bytes(content)
|
||||
|
||||
# 可选参数覆盖（如前端框选的 crop）：{节点ID: {参数: 值}}，随任务持久化。
|
||||
param_overrides = None
|
||||
if params.strip():
|
||||
try:
|
||||
parsed = json.loads(params)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise HTTPException(status_code=422, detail="params must be valid JSON") from exc
|
||||
if not isinstance(parsed, dict):
|
||||
raise HTTPException(status_code=422, detail="params must be a JSON object")
|
||||
param_overrides = parsed
|
||||
|
||||
now = _now_iso()
|
||||
db.create_run(
|
||||
{
|
||||
"id": run_id,
|
||||
"workflow_id": workflow_id,
|
||||
"workflow_version": latest["version"],
|
||||
"status": "QUEUED",
|
||||
"progress": 0,
|
||||
"input_uri": str(input_uri),
|
||||
"param_overrides": param_overrides,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"id": run_id,
|
||||
"status": "QUEUED",
|
||||
"progress": 0,
|
||||
"artifacts": [],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/runs")
|
||||
def list_runs(db: Database = Depends(_get_db)) -> list[dict]:
|
||||
"""返回最近的运行记录，供任务管理页展示。"""
|
||||
return db.list_runs()
|
||||
|
||||
|
||||
@router.get("/api/runs/{run_id}")
|
||||
def get_run(run_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
"""返回任务详情，并附带当前产物列表。"""
|
||||
run = db.get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail="run not found")
|
||||
run["artifacts"] = db.list_artifacts(run_id)
|
||||
return run
|
||||
|
||||
|
||||
@router.post("/api/runs/{run_id}/retry")
|
||||
def retry_run(run_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
"""重置失败任务为排队状态，清空旧产物后重新执行。"""
|
||||
run = db.get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail="run not found")
|
||||
if run["status"] != "FAILED":
|
||||
raise HTTPException(status_code=422, detail="only failed runs can be retried")
|
||||
# reset_run 会清空进度、错误和旧产物，确保从头开始。
|
||||
db.reset_run(run_id, _now_iso())
|
||||
return {"id": run_id, "status": "QUEUED"}
|
||||
|
||||
|
||||
@router.post("/api/runs/{run_id}/pause")
|
||||
def pause_run(run_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
"""暂停任务：排队中或运行中的任务可暂停，运行中的任务在节点边界停下。"""
|
||||
run = db.get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail="run not found")
|
||||
if run["status"] not in ("QUEUED", "RUNNING"):
|
||||
raise HTTPException(status_code=422, detail="only queued or running runs can be paused")
|
||||
db.pause_run(run_id, _now_iso())
|
||||
return {"id": run_id, "status": "PAUSED"}
|
||||
|
||||
|
||||
@router.post("/api/runs/{run_id}/resume")
|
||||
def resume_run(run_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
"""继续任务：暂停的任务恢复排队，由调度器从断点继续执行。"""
|
||||
run = db.get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail="run not found")
|
||||
if run["status"] != "PAUSED":
|
||||
raise HTTPException(status_code=422, detail="only paused runs can be resumed")
|
||||
db.resume_run(run_id, _now_iso())
|
||||
return {"id": run_id, "status": "QUEUED"}
|
||||
|
||||
@router.delete("/api/runs/{run_id}")
|
||||
def delete_run(run_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
"""删除任务：清理产物记录、上传文件与步骤产物目录。"""
|
||||
run = db.get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail="run not found")
|
||||
from wov_app.config import STORAGE_DIR
|
||||
|
||||
# 先删数据库记录（含产物表），再清理磁盘上的上传与中间产物。
|
||||
db.delete_run(run_id)
|
||||
input_uri = run.get("input_uri")
|
||||
if input_uri:
|
||||
# 上传文件位于 <storage>/uploads/<run_id>/，整目录一并删除。
|
||||
shutil.rmtree(Path(input_uri).parent, ignore_errors=True)
|
||||
# 步骤产物位于 <storage>/runs/<run_id>/，整目录一并删除。
|
||||
shutil.rmtree(STORAGE_DIR / "runs" / run_id, ignore_errors=True)
|
||||
return {"deleted": run_id}
|
||||
|
||||
@router.get("/api/runs/{run_id}/artifacts")
|
||||
def list_run_artifacts(run_id: str, db: Database = Depends(_get_db)) -> list[dict]:
|
||||
"""返回任务全部产物记录。"""
|
||||
if db.get_run(run_id) is None:
|
||||
raise HTTPException(status_code=404, detail="run not found")
|
||||
return db.list_artifacts(run_id)
|
||||
|
||||
|
||||
@router.get("/api/runs/{run_id}/artifacts/{artifact_name}")
|
||||
def download_artifact(
|
||||
run_id: str,
|
||||
artifact_name: str,
|
||||
db: Database = Depends(_get_db),
|
||||
) -> FileResponse:
|
||||
"""按任务与产物名下载文件，文件缺失时返回 404。"""
|
||||
artifact = db.get_artifact(run_id, artifact_name)
|
||||
if artifact is None:
|
||||
raise HTTPException(status_code=404, detail="artifact not found")
|
||||
path = Path(artifact["uri"])
|
||||
if not path.is_file():
|
||||
raise HTTPException(status_code=404, detail="artifact file missing")
|
||||
return FileResponse(
|
||||
path,
|
||||
media_type=artifact["mime_type"],
|
||||
filename=path.name,
|
||||
)
|
||||
Executable
+140
@@ -0,0 +1,140 @@
|
||||
"""工作流管理路由。
|
||||
|
||||
提供工作流的创建、查询、校验、发布和删除能力。工作流以版本化 DAG 数据保存，
|
||||
不写死在业务代码中。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from wov_app.db import Database
|
||||
from wov_app.schemas import WorkflowCreate
|
||||
from wov_sdk.models import WorkflowDefinition
|
||||
|
||||
router = APIRouter(prefix="/api/admin/workflows", tags=["workflows"])
|
||||
|
||||
|
||||
def _get_db() -> Database:
|
||||
"""从应用状态延迟获取数据库实例。"""
|
||||
from wov_app.main import app
|
||||
|
||||
return app.state.db
|
||||
|
||||
|
||||
def _slugify(value: str) -> str:
|
||||
"""把工作流名称转换为小写连字符 ID；无有效字符时生成随机 ID。"""
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
|
||||
return slug or uuid.uuid4().hex[:8]
|
||||
|
||||
|
||||
def _validate_definition(raw: dict) -> WorkflowDefinition:
|
||||
"""解析并校验 DAG 定义，非法时转换为 422 HTTP 异常。"""
|
||||
try:
|
||||
definition = WorkflowDefinition.from_dict(raw)
|
||||
definition.validate()
|
||||
return definition
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_workflows(db: Database = Depends(_get_db)) -> list[dict]:
|
||||
"""返回全部工作流概要。"""
|
||||
return db.list_workflows()
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_workflow(
|
||||
payload: WorkflowCreate,
|
||||
db: Database = Depends(_get_db),
|
||||
) -> dict:
|
||||
"""创建新工作流或为已有工作流追加一个版本。"""
|
||||
definition = _validate_definition(payload.definition)
|
||||
# 未显式指定 ID 时由名称生成；已有工作流则版本号递增。
|
||||
workflow_id = payload.id or _slugify(payload.name)
|
||||
existing = db.get_workflow(workflow_id)
|
||||
version = (existing or {}).get("latest_version", 0) + 1
|
||||
# 每次创建都保存新版本，发布操作只切换 published 标记。
|
||||
db.upsert_workflow(
|
||||
{
|
||||
"id": workflow_id,
|
||||
"name": payload.name,
|
||||
"description": payload.description,
|
||||
"published": 0,
|
||||
"latest_version": version,
|
||||
}
|
||||
)
|
||||
db.create_workflow_version(workflow_id, version, definition.to_dict())
|
||||
return {
|
||||
"id": workflow_id,
|
||||
"name": payload.name,
|
||||
"description": payload.description,
|
||||
"published": False,
|
||||
"latest_version": version,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{workflow_id}")
|
||||
def get_workflow(workflow_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
"""返回工作流概要及最新版本定义。"""
|
||||
workflow = db.get_workflow(workflow_id)
|
||||
if workflow is None:
|
||||
raise HTTPException(status_code=404, detail="workflow not found")
|
||||
latest = db.get_latest_workflow_version(workflow_id)
|
||||
workflow["latest_version_data"] = latest
|
||||
return workflow
|
||||
|
||||
|
||||
@router.delete("/{workflow_id}")
|
||||
def delete_workflow(workflow_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
"""删除工作流及其版本、任务和产物记录。"""
|
||||
if db.get_workflow(workflow_id) is None:
|
||||
raise HTTPException(status_code=404, detail="workflow not found")
|
||||
db.delete_workflow(workflow_id)
|
||||
return {"deleted": workflow_id}
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/validate")
|
||||
def validate_workflow(
|
||||
workflow_id: str,
|
||||
definition: dict,
|
||||
db: Database = Depends(_get_db),
|
||||
) -> dict:
|
||||
"""在不保存的情况下校验一份 DAG 定义。"""
|
||||
if db.get_workflow(workflow_id) is None:
|
||||
raise HTTPException(status_code=404, detail="workflow not found")
|
||||
parsed = _validate_definition(definition)
|
||||
return {"valid": True, "node_ids": [node.id for node in parsed.nodes]}
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/publish")
|
||||
def publish_workflow(workflow_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
"""把工作流标记为已发布，使其出现在用户应用中心。"""
|
||||
workflow = db.get_workflow(workflow_id)
|
||||
if workflow is None:
|
||||
raise HTTPException(status_code=404, detail="workflow not found")
|
||||
if workflow["latest_version"] == 0:
|
||||
raise HTTPException(status_code=422, detail="workflow has no version")
|
||||
# 发布只是状态切换，不修改已保存的版本数据。
|
||||
db.upsert_workflow(
|
||||
{
|
||||
"id": workflow_id,
|
||||
"name": workflow["name"],
|
||||
"description": workflow["description"],
|
||||
"published": 1,
|
||||
"latest_version": workflow["latest_version"],
|
||||
}
|
||||
)
|
||||
return {"published": workflow_id}
|
||||
|
||||
|
||||
@router.get("/{workflow_id}/versions")
|
||||
def list_versions(workflow_id: str, db: Database = Depends(_get_db)) -> list[dict]:
|
||||
"""返回工作流全部版本定义。"""
|
||||
if db.get_workflow(workflow_id) is None:
|
||||
raise HTTPException(status_code=404, detail="workflow not found")
|
||||
return db.list_workflow_versions(workflow_id)
|
||||
Executable
+308
@@ -0,0 +1,308 @@
|
||||
"""工作流调度器。
|
||||
|
||||
轮询 SQLite 中的排队任务，按工作流 DAG 的拓扑顺序依次调用进程内节点
|
||||
处理器，并把节点产物登记为任务产物。单体版使用单线程顺序执行，节点在
|
||||
同一进程内直接调用，不再经过子进程与 HTTP 协议。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from wov_sdk.models import InvokeRequest, WorkflowDefinition
|
||||
|
||||
from wov_app import registry
|
||||
from wov_app.config import SCHEDULER_INTERVAL_SECONDS
|
||||
from wov_app.db import Database
|
||||
from wov_app.logging import get_logger
|
||||
|
||||
# 调度器运行日志：节点进度、暂停/续跑等状态变化。
|
||||
logger = get_logger("scheduler")
|
||||
|
||||
def _now_iso() -> str:
|
||||
"""返回当前 UTC 时间的 ISO 格式字符串。"""
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def topological_sort(definition: WorkflowDefinition) -> list[str]:
|
||||
"""对工作流 DAG 做拓扑排序，返回可执行的节点 ID 顺序。"""
|
||||
nodes = {node.id: node for node in definition.nodes}
|
||||
# 统计每个节点的入度，并记录依赖关系。
|
||||
indegree = {node_id: 0 for node_id in nodes}
|
||||
dependents: dict[str, list[str]] = {node_id: [] for node_id in nodes}
|
||||
|
||||
for edge in definition.edges:
|
||||
# 边引用了不存在的节点时直接报错。
|
||||
if edge.from_node not in nodes or edge.to_node not in nodes:
|
||||
raise ValueError(f"unknown edge: {edge.from_node} -> {edge.to_node}")
|
||||
indegree[edge.to_node] += 1
|
||||
dependents[edge.from_node].append(edge.to_node)
|
||||
|
||||
# Kahn 算法：从入度为 0 的节点开始逐层取出。
|
||||
queue = [node_id for node_id, degree in indegree.items() if degree == 0]
|
||||
ordered: list[str] = []
|
||||
while queue:
|
||||
current = queue.pop(0)
|
||||
ordered.append(current)
|
||||
for dependent in dependents[current]:
|
||||
indegree[dependent] -= 1
|
||||
if indegree[dependent] == 0:
|
||||
queue.append(dependent)
|
||||
|
||||
# 排序结果数量不足说明存在环，无法确定执行顺序。
|
||||
if len(ordered) != len(nodes):
|
||||
raise ValueError("workflow contains a cycle")
|
||||
return ordered
|
||||
|
||||
|
||||
class WorkflowScheduler:
|
||||
"""后台任务调度器：单线程轮询并执行排队中的工作流运行。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db: Database,
|
||||
storage_dir: Path,
|
||||
interval_seconds: float | None = None,
|
||||
) -> None:
|
||||
"""保存依赖并初始化轮询线程控制字段。"""
|
||||
self.db = db
|
||||
self.storage_dir = storage_dir
|
||||
self.interval_seconds = interval_seconds or SCHEDULER_INTERVAL_SECONDS
|
||||
self._thread: threading.Thread | None = None
|
||||
self._stopping = False
|
||||
|
||||
def start(self) -> None:
|
||||
"""启动调度线程；重复调用无副作用。"""
|
||||
if self._thread is not None:
|
||||
return
|
||||
self._stopping = False
|
||||
self._thread = threading.Thread(
|
||||
target=self._loop,
|
||||
name="wov-workflow-scheduler",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
"""请求停止并等待轮询线程退出。"""
|
||||
self._stopping = True
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=5)
|
||||
self._thread = None
|
||||
|
||||
def _loop(self) -> None:
|
||||
"""轮询循环：有排队任务就立即执行，否则休眠一个间隔。"""
|
||||
while not self._stopping:
|
||||
run = self.db.next_queued_run()
|
||||
if run is not None:
|
||||
self.execute_run(run["id"])
|
||||
else:
|
||||
time.sleep(self.interval_seconds)
|
||||
|
||||
def _resolve_ref(
|
||||
self,
|
||||
ref: str,
|
||||
run_input_uri: str | None,
|
||||
outputs_by_node: dict[str, dict[str, str]],
|
||||
) -> str | None:
|
||||
"""解析输入引用：input.xxx 取任务入口，node.key 取前序节点产物。"""
|
||||
# 入口引用以 input. 为前缀。
|
||||
if ref.startswith("input."):
|
||||
return run_input_uri
|
||||
# 其余引用必须形如 "节点ID.输出名"。
|
||||
node_id, separator, key = ref.partition(".")
|
||||
if not separator:
|
||||
return None
|
||||
return outputs_by_node.get(node_id, {}).get(key)
|
||||
|
||||
def execute_run(self, run_id: str) -> None:
|
||||
"""执行单个任务：加载 DAG、按拓扑顺序调用节点并登记产物。"""
|
||||
run = self.db.get_run(run_id)
|
||||
# 任务不存在或不在可执行状态（排队/暂停）时直接返回，避免重复执行。
|
||||
if run is None or run["status"] not in ("QUEUED", "PAUSED"):
|
||||
return
|
||||
|
||||
# 工作流或版本记录丢失时把任务标记为失败。
|
||||
workflow = self.db.get_workflow(run["workflow_id"])
|
||||
if workflow is None:
|
||||
self.db.update_run(run_id, status="FAILED", error="workflow not found", updated_at=_now_iso())
|
||||
return
|
||||
|
||||
version = self.db.get_workflow_version(run["workflow_id"], run["workflow_version"])
|
||||
if version is None:
|
||||
self.db.update_run(run_id, status="FAILED", error="workflow version not found", updated_at=_now_iso())
|
||||
return
|
||||
|
||||
# 解析并校验 DAG，随后计算拓扑执行顺序。
|
||||
definition = WorkflowDefinition.from_dict(version["definition"])
|
||||
definition.validate()
|
||||
ordered = topological_sort(definition)
|
||||
# 从已登记产物重建已完成节点的输出，支持暂停后断点续跑。
|
||||
outputs_by_node = self.db.restore_run_outputs(run_id)
|
||||
run_started = time.monotonic()
|
||||
self.db.update_run(run_id, status="RUNNING", progress=0, updated_at=_now_iso())
|
||||
try:
|
||||
for index, node_id in enumerate(ordered):
|
||||
# 暂停检查：用户暂停后调度器在节点边界停下，保持 PAUSED 等待续跑。
|
||||
current = self.db.get_run(run_id)
|
||||
if current is None or current["status"] == "PAUSED":
|
||||
logger.info("任务 %s 已暂停，停止在节点 %s 之前", run_id, node_id)
|
||||
return
|
||||
# 断点续跑：跳过已产出结果的节点（其产物已作为输入可用）。
|
||||
if node_id in outputs_by_node:
|
||||
continue
|
||||
# 当前节点进度 = 已完成节点数 / 总节点数。
|
||||
node_spec = next(item for item in definition.nodes if item.id == node_id)
|
||||
self.db.update_run(
|
||||
run_id,
|
||||
current_node_id=node_id,
|
||||
progress=index / len(ordered),
|
||||
updated_at=_now_iso(),
|
||||
)
|
||||
# 解析节点声明的每个输入引用，缺任一输入即失败。
|
||||
invoke_inputs: dict[str, str] = {}
|
||||
for input_name, ref in node_spec.inputs.items():
|
||||
value = self._resolve_ref(ref, run.get("input_uri"), outputs_by_node)
|
||||
if value is None:
|
||||
raise ValueError(f"missing input {input_name} for node {node_id}")
|
||||
invoke_inputs[input_name] = value
|
||||
|
||||
# 前端框选的 crop 等参数覆盖：按节点 ID 合并进节点参数。
|
||||
node_params = dict(node_spec.params)
|
||||
overrides = run.get("param_overrides") or {}
|
||||
node_params.update(overrides.get(node_id, {}))
|
||||
|
||||
# 每个任务的每个节点使用独立产物目录，避免并发冲突。
|
||||
node_started = time.monotonic()
|
||||
output_dir = (
|
||||
self.storage_dir
|
||||
/ "runs"
|
||||
/ run_id
|
||||
/ "steps"
|
||||
/ node_id
|
||||
)
|
||||
response = registry.invoke(
|
||||
node_spec.node_type,
|
||||
InvokeRequest(
|
||||
run_id=run_id,
|
||||
node_instance_id="",
|
||||
inputs=invoke_inputs,
|
||||
params=node_params,
|
||||
output_dir=str(output_dir),
|
||||
),
|
||||
)
|
||||
# 节点返回非 completed 即视为步骤失败。
|
||||
if response.status != "completed":
|
||||
raise RuntimeError(response.error or f"node {node_id} failed")
|
||||
|
||||
# 记录节点输出，供后续节点引用和最终产物映射使用。
|
||||
outputs_by_node[node_id] = {
|
||||
str(key): str(value) for key, value in response.outputs.items()
|
||||
}
|
||||
for key, uri in outputs_by_node[node_id].items():
|
||||
# 产物名带节点前缀，例如 asr.srt_uri，避免跨节点重名。
|
||||
artifact = {
|
||||
"run_id": run_id,
|
||||
"node_id": node_id,
|
||||
"name": f"{node_id}.{key}",
|
||||
"uri": uri,
|
||||
"mime_type": self._mime_type(uri),
|
||||
"size": self._file_size(uri),
|
||||
}
|
||||
self.db.create_artifact(artifact)
|
||||
# 进度日志：节点序号/总数、耗时与任务累计运行时间（数据速度可观测）。
|
||||
logger.info(
|
||||
"任务 %s 进度 %d/%d 节点: %s 耗时 %.1fs, 运行累计 %.1fs",
|
||||
run_id, index + 1, len(ordered), node_id,
|
||||
time.monotonic() - node_started,
|
||||
time.monotonic() - run_started,
|
||||
)
|
||||
# 处理 final_outputs，为用户端提供简洁的下载别名。
|
||||
for alias, ref in definition.final_outputs.items():
|
||||
resolved = self._resolve_ref(ref, run.get("input_uri"), outputs_by_node)
|
||||
if resolved is not None:
|
||||
# 最终产物按 上传文件名.标识.时间戳 重命名，区分语言与版本。
|
||||
resolved = self._final_artifact_uri(resolved, run, definition, alias, ref)
|
||||
self.db.create_artifact(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"node_id": ref.partition(".")[0],
|
||||
"name": alias,
|
||||
"uri": resolved,
|
||||
"mime_type": self._mime_type(resolved),
|
||||
"size": self._file_size(resolved),
|
||||
}
|
||||
)
|
||||
|
||||
# 全部节点成功后标记完成；期间被暂停则保持 PAUSED，等待续跑补做收尾。
|
||||
if self.db.get_run(run_id)["status"] == "PAUSED":
|
||||
logger.info("任务 %s 节点全部完成但已暂停，保持 PAUSED", run_id)
|
||||
return
|
||||
self.db.update_run(
|
||||
run_id,
|
||||
status="COMPLETED",
|
||||
current_node_id=None,
|
||||
progress=1.0,
|
||||
updated_at=_now_iso(),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# 任一步骤异常都结束任务并记录错误，等待用户重试。
|
||||
self.db.update_run(
|
||||
run_id,
|
||||
status="FAILED",
|
||||
error=str(exc),
|
||||
updated_at=_now_iso(),
|
||||
)
|
||||
|
||||
def _final_artifact_uri(
|
||||
self,
|
||||
resolved: str,
|
||||
run: dict,
|
||||
definition: WorkflowDefinition,
|
||||
alias: str,
|
||||
ref: str,
|
||||
) -> str:
|
||||
"""把最终产物重命名为 上传文件名.标识.时间戳 并返回新 URI。
|
||||
|
||||
标识优先取产出节点的 target_language 参数（如 zh-CN），否则回退为
|
||||
产物别名；时间戳取当前时刻，用于区分同一上传文件的多次运行版本。
|
||||
重命名在原地进行（同目录），不复制文件。
|
||||
"""
|
||||
source = Path(resolved)
|
||||
# 续跑等场景下源文件可能已被上次收尾重命名过：不再重命名，原样返回。
|
||||
if not source.is_file():
|
||||
return resolved
|
||||
# 基础名来自上传文件名；无上传文件时退回通用名称 subtitle。
|
||||
base = Path(run["input_uri"]).stem if run.get("input_uri") else "subtitle"
|
||||
# 通过最终输出引用定位产出节点，取其语言参数作为标识。
|
||||
node_id = ref.partition(".")[0]
|
||||
node = next((item for item in definition.nodes if item.id == node_id), None)
|
||||
tag = (node.params.get("target_language") if node else None) or alias
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
|
||||
new_path = source.with_name(f"{base}.{tag}.{timestamp}{source.suffix}")
|
||||
source.rename(new_path)
|
||||
return str(new_path)
|
||||
|
||||
@staticmethod
|
||||
def _mime_type(uri: str) -> str:
|
||||
"""按扩展名推断产物 MIME 类型，未知类型使用通用二进制类型。"""
|
||||
path = Path(uri)
|
||||
suffix = path.suffix.lower()
|
||||
return {
|
||||
".srt": "application/x-subrip",
|
||||
".ass": "text/plain",
|
||||
".wav": "audio/wav",
|
||||
".mp4": "video/mp4",
|
||||
".txt": "text/plain",
|
||||
}.get(suffix, "application/octet-stream")
|
||||
|
||||
@staticmethod
|
||||
def _file_size(uri: str) -> int:
|
||||
"""读取产物文件大小；文件缺失时按 0 处理。"""
|
||||
try:
|
||||
return Path(uri).stat().st_size
|
||||
except OSError:
|
||||
return 0
|
||||
@@ -0,0 +1,22 @@
|
||||
"""FastAPI 请求/响应 schema。
|
||||
|
||||
使用 Pydantic 模型校验管理 API 的 JSON 请求体。节点管理功能已移除，仅保留
|
||||
工作流相关请求模型。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class WorkflowCreate(BaseModel):
|
||||
"""创建工作流或新增版本的请求体。"""
|
||||
|
||||
# 缺省时由后端根据名称生成 slug ID。
|
||||
id: str | None = None
|
||||
name: str = Field(min_length=1)
|
||||
description: str = ""
|
||||
# DAG 原始字典，后端会解析并校验为 WorkflowDefinition。
|
||||
definition: dict[str, Any]
|
||||
@@ -0,0 +1,48 @@
|
||||
"""种子数据模块。
|
||||
|
||||
从 workflows/*.json 数据文件加载默认工作流并写入数据库（幂等）。
|
||||
工作流定义是**数据**（JSON）：切换模型、调整链路只改数据文件，不涉及代码，
|
||||
满足"工作流即数据"与"切换模型不改代码"的设计约束。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from wov_sdk.models import WorkflowDefinition
|
||||
|
||||
from wov_app.config import WORKSPACE_ROOT
|
||||
from wov_app.db import Database
|
||||
|
||||
|
||||
def seed_default_workflows(db: Database, workflows_dir: Path | None = None) -> int:
|
||||
"""从数据目录加载默认工作流，已存在的工作流跳过，返回创建数量。
|
||||
|
||||
每个 JSON 文件结构：
|
||||
{"id", "name", "description", "version", "definition"}，
|
||||
definition 为 WorkflowDefinition 的标准 DAG 字典。
|
||||
"""
|
||||
workflows_dir = workflows_dir or (WORKSPACE_ROOT / "workflows")
|
||||
created = 0
|
||||
for path in sorted(workflows_dir.glob("*.json")):
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
workflow_id = str(payload["id"])
|
||||
# 已存在的工作流不覆盖，避免启动时反复改写用户数据。
|
||||
if db.get_workflow(workflow_id) is not None:
|
||||
continue
|
||||
definition = WorkflowDefinition.from_dict(payload["definition"])
|
||||
definition.validate()
|
||||
version = int(payload.get("version", 1))
|
||||
db.upsert_workflow(
|
||||
{
|
||||
"id": workflow_id,
|
||||
"name": str(payload["name"]),
|
||||
"description": str(payload.get("description", "")),
|
||||
"published": 1,
|
||||
"latest_version": version,
|
||||
}
|
||||
)
|
||||
db.create_workflow_version(workflow_id, version, definition.to_dict())
|
||||
created += 1
|
||||
return created
|
||||
@@ -0,0 +1,28 @@
|
||||
"""WOV SDK 公共导出入口。
|
||||
|
||||
单体版中调度器、节点与 API 统一从 wov_sdk 导入协议模型，而无需关心具体
|
||||
模块路径。协议数据模型保持与分布式版一致，为将来回退保留兼容桥梁。
|
||||
"""
|
||||
|
||||
from wov_sdk.models import (
|
||||
HealthResponse,
|
||||
InvokeRequest,
|
||||
InvokeResponse,
|
||||
NodeManifest,
|
||||
ProgressEvent,
|
||||
WorkflowDefinition,
|
||||
WorkflowEdge,
|
||||
WorkflowNode,
|
||||
)
|
||||
|
||||
# 对外稳定的公共 API 清单；新增模型时必须同步追加到这里。
|
||||
__all__ = [
|
||||
"HealthResponse",
|
||||
"InvokeRequest",
|
||||
"InvokeResponse",
|
||||
"NodeManifest",
|
||||
"ProgressEvent",
|
||||
"WorkflowDefinition",
|
||||
"WorkflowEdge",
|
||||
"WorkflowNode",
|
||||
]
|
||||
Executable
+321
@@ -0,0 +1,321 @@
|
||||
"""WOV 节点协议核心数据模型。
|
||||
|
||||
本模块定义节点 Manifest、调用请求/响应、健康检查、进度事件以及工作流 DAG 的
|
||||
通用数据结构。单体版中调度器、节点与 API 共用这些类，字段语义必须长期保持
|
||||
稳定，新增能力时只能向后兼容地扩展字段。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _require_non_empty(value: str, name: str) -> None:
|
||||
"""校验必填字符串字段，空字符串或纯空白字符串都会被拒绝。"""
|
||||
if not value or not value.strip():
|
||||
raise ValueError(f"{name} must not be empty")
|
||||
|
||||
|
||||
@dataclass
|
||||
class NodeManifest:
|
||||
"""节点注册清单：描述节点能力、输入输出与资源参数。
|
||||
|
||||
该清单由 manifests/ 目录下的 JSON 文件提供，单体启动时注册到进程内
|
||||
节点注册表，调度器据此把 node_type 解析到对应的 invoke 处理器。
|
||||
"""
|
||||
|
||||
# 节点稳定唯一 ID，例如 faster-whisper；注册后不可随意更改。
|
||||
id: str
|
||||
# 展示名称，仅用于管理后台等界面。
|
||||
name: str
|
||||
# 节点版本号，与节点仓库 Git tag 保持一致。
|
||||
version: str
|
||||
# 能力标识，工作流通过 node_type 引用能力，而不是直接绑定具体仓库。
|
||||
capability: str
|
||||
# 启动命令；单体版不再启动子进程，字段保留仅为协议兼容。
|
||||
command: list[str]
|
||||
# 节点代码所在目录；单体版保留仅为协议兼容。
|
||||
repo_dir: str = "."
|
||||
# 节点环境变量；单体版保留仅为协议兼容。
|
||||
env: dict[str, str] = field(default_factory=dict)
|
||||
# 输入字段 schema，当前主要用于文档展示，后续可用于运行时校验。
|
||||
input_schema: dict[str, Any] = field(default_factory=dict)
|
||||
# 输出字段 schema，用于描述节点产物的名称与类型。
|
||||
output_schema: dict[str, Any] = field(default_factory=dict)
|
||||
# 单进程内无并发槽位概念，字段保留仅为协议兼容。
|
||||
max_concurrency: int = 1
|
||||
# 单体内模型常驻不回收，字段保留仅为协议兼容。
|
||||
idle_ttl_seconds: int = 300
|
||||
# 无进程启动等待，字段保留仅为协议兼容。
|
||||
health_timeout_seconds: int = 10
|
||||
# 单体内模型常驻不回收，字段保留仅为协议兼容。
|
||||
keep_warm: bool = False
|
||||
|
||||
def validate(self) -> None:
|
||||
"""校验 manifest 必填字段与数值边界，非法配置抛出 ValueError。"""
|
||||
_require_non_empty(self.id, "id")
|
||||
_require_non_empty(self.name, "name")
|
||||
_require_non_empty(self.version, "version")
|
||||
_require_non_empty(self.capability, "capability")
|
||||
_require_non_empty(self.repo_dir, "repo_dir")
|
||||
# 命令不能为空，否则节点进程无法启动。
|
||||
if not self.command:
|
||||
raise ValueError("command must not be empty")
|
||||
if self.max_concurrency < 1:
|
||||
raise ValueError("max_concurrency must be >= 1")
|
||||
if self.idle_ttl_seconds < 0:
|
||||
raise ValueError("idle_ttl_seconds must be >= 0")
|
||||
if self.health_timeout_seconds < 1:
|
||||
raise ValueError("health_timeout_seconds must be >= 1")
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转换为可 JSON 序列化的普通字典。"""
|
||||
return {
|
||||
"id": self.id,
|
||||
"name": self.name,
|
||||
"version": self.version,
|
||||
"capability": self.capability,
|
||||
"command": self.command,
|
||||
"repo_dir": self.repo_dir,
|
||||
"env": self.env,
|
||||
"input_schema": self.input_schema,
|
||||
"output_schema": self.output_schema,
|
||||
"max_concurrency": self.max_concurrency,
|
||||
"idle_ttl_seconds": self.idle_ttl_seconds,
|
||||
"health_timeout_seconds": self.health_timeout_seconds,
|
||||
"keep_warm": self.keep_warm,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "NodeManifest":
|
||||
"""从注册 API 或 JSON 文件解析出的字典恢复 manifest。"""
|
||||
return cls(
|
||||
id=str(data["id"]),
|
||||
name=str(data["name"]),
|
||||
version=str(data["version"]),
|
||||
capability=str(data["capability"]),
|
||||
# command 可能缺省，解析时提供空列表兜底。
|
||||
command=[str(item) for item in data.get("command", [])],
|
||||
repo_dir=str(data.get("repo_dir", ".")),
|
||||
env={str(k): str(v) for k, v in data.get("env", {}).items()},
|
||||
input_schema=dict(data.get("input_schema", {})),
|
||||
output_schema=dict(data.get("output_schema", {})),
|
||||
# 数值字段缺省时使用与 dataclass 一致的默认值。
|
||||
max_concurrency=int(data.get("max_concurrency", 1)),
|
||||
idle_ttl_seconds=int(data.get("idle_ttl_seconds", 300)),
|
||||
health_timeout_seconds=int(data.get("health_timeout_seconds", 10)),
|
||||
keep_warm=bool(data.get("keep_warm", False)),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: str) -> "NodeManifest":
|
||||
"""从磁盘上的 node.manifest.json 加载并校验 manifest。"""
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
manifest = cls.from_dict(json.load(f))
|
||||
manifest.validate()
|
||||
return manifest
|
||||
|
||||
|
||||
@dataclass
|
||||
class InvokeRequest:
|
||||
"""节点调用请求：由调度器或管理后台发送给节点 HTTP 服务。"""
|
||||
|
||||
# 工作流运行 ID，用于追踪一次完整执行。
|
||||
run_id: str
|
||||
# 实际承载本次调用的节点实例 ID，由 NodeManager 回填。
|
||||
node_instance_id: str
|
||||
# 输入产物映射，key 为输入名，value 为产物 URI 或直接文本。
|
||||
inputs: dict[str, Any] = field(default_factory=dict)
|
||||
# 节点运行参数，例如采样率、语言、模型路径等。
|
||||
params: dict[str, Any] = field(default_factory=dict)
|
||||
# 节点产物输出目录。
|
||||
output_dir: str = "."
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转换为可 JSON 序列化的普通字典。"""
|
||||
return {
|
||||
"run_id": self.run_id,
|
||||
"node_instance_id": self.node_instance_id,
|
||||
"inputs": self.inputs,
|
||||
"params": self.params,
|
||||
"output_dir": self.output_dir,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "InvokeRequest":
|
||||
"""从 HTTP 请求 JSON 解析调用请求。"""
|
||||
return cls(
|
||||
run_id=str(data["run_id"]),
|
||||
node_instance_id=str(data["node_instance_id"]),
|
||||
inputs=dict(data.get("inputs", {})),
|
||||
params=dict(data.get("params", {})),
|
||||
output_dir=str(data.get("output_dir", ".")),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class InvokeResponse:
|
||||
"""节点调用响应：completed 表示成功，failed 表示执行失败。"""
|
||||
|
||||
# 执行状态，固定为 completed / failed。
|
||||
status: str
|
||||
# 输出产物映射，key 为输出名，value 为产物 URI。
|
||||
outputs: dict[str, Any] = field(default_factory=dict)
|
||||
# 失败原因，仅在 failed 时有意义。
|
||||
error: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转换为可 JSON 序列化的普通字典。"""
|
||||
return {
|
||||
"status": self.status,
|
||||
"outputs": self.outputs,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "InvokeResponse":
|
||||
"""从 HTTP 响应 JSON 解析调用结果。"""
|
||||
return cls(
|
||||
# 缺省按失败处理，避免未知状态被误判为成功。
|
||||
status=str(data.get("status", "failed")),
|
||||
outputs=dict(data.get("outputs", {})),
|
||||
error=data.get("error"),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HealthResponse:
|
||||
"""节点健康检查响应：节点进程就绪后返回 ok 与自身标识。"""
|
||||
|
||||
status: str
|
||||
node_id: str
|
||||
version: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转换为可 JSON 序列化的普通字典。"""
|
||||
return {
|
||||
"status": self.status,
|
||||
"node_id": self.node_id,
|
||||
"version": self.version,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProgressEvent:
|
||||
"""进度事件：预留用于节点向调度器上报执行进度。"""
|
||||
|
||||
run_id: str
|
||||
node_id: str
|
||||
progress: float
|
||||
message: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转换为可 JSON 序列化的普通字典。"""
|
||||
return {
|
||||
"run_id": self.run_id,
|
||||
"node_id": self.node_id,
|
||||
"progress": self.progress,
|
||||
"message": self.message,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowNode:
|
||||
"""工作流中的一个节点：声明节点类型、参数和输入引用。"""
|
||||
|
||||
# 节点在 DAG 内的唯一 ID，例如 extract、asr。
|
||||
id: str
|
||||
# 引用的节点能力，例如 ffmpeg-extract、faster-whisper。
|
||||
node_type: str
|
||||
# 传递给节点 invoke 的 params。
|
||||
params: dict[str, Any] = field(default_factory=dict)
|
||||
# 输入引用，value 形如 "前序节点ID.输出名" 或 "input.入口字段"。
|
||||
inputs: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转换为可 JSON 序列化的普通字典。"""
|
||||
return {
|
||||
"id": self.id,
|
||||
"node_type": self.node_type,
|
||||
"params": self.params,
|
||||
"inputs": self.inputs,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "WorkflowNode":
|
||||
"""从工作流定义 JSON 解析节点。"""
|
||||
return cls(
|
||||
id=str(data["id"]),
|
||||
node_type=str(data["node_type"]),
|
||||
params=dict(data.get("params", {})),
|
||||
inputs={str(k): str(v) for k, v in data.get("inputs", {}).items()},
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowEdge:
|
||||
"""工作流有向边：from_node 的输出流向 to_node 的输入。"""
|
||||
|
||||
from_node: str
|
||||
to_node: str
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转换为 JSON 时使用 from/to 短字段名。"""
|
||||
return {"from": self.from_node, "to": self.to_node}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "WorkflowEdge":
|
||||
"""从工作流定义 JSON 解析边。"""
|
||||
return cls(from_node=str(data["from"]), to_node=str(data["to"]))
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowDefinition:
|
||||
"""工作流 DAG 定义：包含节点列表、依赖边和输入输出映射。"""
|
||||
|
||||
name: str
|
||||
version: int
|
||||
nodes: list[WorkflowNode] = field(default_factory=list)
|
||||
edges: list[WorkflowEdge] = field(default_factory=list)
|
||||
# 用户上传入口与入口字段名的映射，例如 {"video_uri": "file"}。
|
||||
entry_inputs: dict[str, Any] = field(default_factory=dict)
|
||||
# 最终对外暴露的产物别名映射，例如 {"ass": "ass.ass_uri"}。
|
||||
final_outputs: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def validate(self) -> None:
|
||||
"""校验 DAG 基本约束：名称、版本、节点 ID 唯一、边引用有效。"""
|
||||
_require_non_empty(self.name, "name")
|
||||
if self.version < 1:
|
||||
raise ValueError("version must be >= 1")
|
||||
# 节点 ID 集合用于检查重复和边引用。
|
||||
node_ids = {node.id for node in self.nodes}
|
||||
if len(node_ids) != len(self.nodes):
|
||||
raise ValueError("workflow node ids must be unique")
|
||||
for edge in self.edges:
|
||||
if edge.from_node not in node_ids or edge.to_node not in node_ids:
|
||||
raise ValueError(f"edge references unknown node: {edge}")
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""转换为可 JSON 序列化的普通字典。"""
|
||||
return {
|
||||
"name": self.name,
|
||||
"version": self.version,
|
||||
"nodes": [node.to_dict() for node in self.nodes],
|
||||
"edges": [edge.to_dict() for edge in self.edges],
|
||||
"entry_inputs": self.entry_inputs,
|
||||
"final_outputs": self.final_outputs,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "WorkflowDefinition":
|
||||
"""从工作流定义 JSON 解析 DAG。"""
|
||||
return cls(
|
||||
name=str(data["name"]),
|
||||
version=int(data.get("version", 1)),
|
||||
nodes=[WorkflowNode.from_dict(item) for item in data.get("nodes", [])],
|
||||
edges=[WorkflowEdge.from_dict(item) for item in data.get("edges", [])],
|
||||
entry_inputs=dict(data.get("entry_inputs", {})),
|
||||
final_outputs=dict(data.get("final_outputs", {})),
|
||||
)
|
||||
Reference in New Issue
Block a user