feat: 完成工作流调度与上传下载 API
This commit is contained in:
cat-shark
@@ -0,0 +1 @@
|
||||
"""WOV platform API package."""
|
||||
@@ -0,0 +1,18 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
WORKSPACE_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
SDK_SRC = WORKSPACE_ROOT / "wov-sdk" / "src"
|
||||
|
||||
if str(SDK_SRC) not in sys.path:
|
||||
sys.path.insert(0, str(SDK_SRC))
|
||||
|
||||
DATA_DIR = Path(os.getenv("WOV_DATA_DIR", str(WORKSPACE_ROOT / "wov-api" / "data")))
|
||||
DB_PATH = Path(os.getenv("WOV_DB_PATH", str(DATA_DIR / "wov.db")))
|
||||
STORAGE_DIR = Path(os.getenv("WOV_STORAGE_DIR", str(DATA_DIR / "storage")))
|
||||
|
||||
NODE_REAP_INTERVAL_SECONDS = float(os.getenv("WOV_REAP_INTERVAL_SECONDS", "3"))
|
||||
NODE_READY_TIMEOUT_SECONDS = float(os.getenv("WOV_READY_TIMEOUT_SECONDS", "12"))
|
||||
@@ -0,0 +1,382 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
|
||||
from wov_sdk.models import NodeManifest
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._init_schema()
|
||||
|
||||
@contextmanager
|
||||
def _connect(self) -> Iterator[sqlite3.Connection]:
|
||||
conn = sqlite3.connect(self.path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
try:
|
||||
yield conn
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _init_schema(self) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.executescript(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS nodes (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
capability TEXT NOT NULL,
|
||||
manifest_json TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS node_instances (
|
||||
id TEXT PRIMARY KEY,
|
||||
node_id TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
pid INTEGER,
|
||||
address TEXT,
|
||||
started_at TEXT,
|
||||
last_used_at TEXT,
|
||||
busy_since TEXT,
|
||||
error TEXT,
|
||||
FOREIGN KEY(node_id) REFERENCES nodes(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS workflows (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
published INTEGER NOT NULL DEFAULT 0,
|
||||
latest_version INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS workflow_versions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
workflow_id TEXT NOT NULL,
|
||||
version INTEGER NOT NULL,
|
||||
definition_json TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(workflow_id, version),
|
||||
FOREIGN KEY(workflow_id) REFERENCES workflows(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS workflow_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
workflow_id TEXT NOT NULL,
|
||||
workflow_version INTEGER NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
current_node_id TEXT,
|
||||
progress REAL NOT NULL DEFAULT 0,
|
||||
error TEXT,
|
||||
input_uri TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
FOREIGN KEY(workflow_id) REFERENCES workflows(id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS artifacts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
run_id TEXT NOT NULL,
|
||||
node_id TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
uri TEXT NOT NULL,
|
||||
mime_type TEXT NOT NULL DEFAULT '',
|
||||
size INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(run_id, name),
|
||||
FOREIGN KEY(run_id) REFERENCES workflow_runs(id)
|
||||
);
|
||||
"""
|
||||
)
|
||||
|
||||
def upsert_node(self, manifest: NodeManifest) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO nodes (id, name, version, capability, manifest_json)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
version = excluded.version,
|
||||
capability = excluded.capability,
|
||||
manifest_json = excluded.manifest_json
|
||||
""",
|
||||
(
|
||||
manifest.id,
|
||||
manifest.name,
|
||||
manifest.version,
|
||||
manifest.capability,
|
||||
json.dumps(manifest.to_dict(), ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
|
||||
def get_node(self, node_id: str) -> NodeManifest | None:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute("SELECT manifest_json FROM nodes WHERE id = ?", (node_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return NodeManifest.from_dict(json.loads(row["manifest_json"]))
|
||||
|
||||
def list_nodes(self) -> list[NodeManifest]:
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute("SELECT manifest_json FROM nodes ORDER BY id").fetchall()
|
||||
return [NodeManifest.from_dict(json.loads(row["manifest_json"])) for row in rows]
|
||||
|
||||
def delete_node(self, node_id: str) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute("DELETE FROM node_instances WHERE node_id = ?", (node_id,))
|
||||
conn.execute("DELETE FROM nodes WHERE id = ?", (node_id,))
|
||||
|
||||
def upsert_instance(self, instance: dict[str, Any]) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO node_instances (
|
||||
id, node_id, status, pid, address, started_at, last_used_at,
|
||||
busy_since, error
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
status = excluded.status,
|
||||
pid = excluded.pid,
|
||||
address = excluded.address,
|
||||
started_at = excluded.started_at,
|
||||
last_used_at = excluded.last_used_at,
|
||||
busy_since = excluded.busy_since,
|
||||
error = excluded.error
|
||||
""",
|
||||
(
|
||||
instance["id"],
|
||||
instance["node_id"],
|
||||
instance["status"],
|
||||
instance.get("pid"),
|
||||
instance.get("address"),
|
||||
instance.get("started_at"),
|
||||
instance.get("last_used_at"),
|
||||
instance.get("busy_since"),
|
||||
instance.get("error"),
|
||||
),
|
||||
)
|
||||
|
||||
def list_instances(self) -> list[dict[str, Any]]:
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM node_instances ORDER BY started_at DESC"
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def delete_instance(self, instance_id: str) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute("DELETE FROM node_instances WHERE id = ?", (instance_id,))
|
||||
|
||||
def upsert_workflow(self, workflow: dict[str, Any]) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO workflows (id, name, description, published, latest_version)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
name = excluded.name,
|
||||
description = excluded.description,
|
||||
published = excluded.published,
|
||||
latest_version = excluded.latest_version
|
||||
""",
|
||||
(
|
||||
workflow["id"],
|
||||
workflow["name"],
|
||||
workflow.get("description", ""),
|
||||
int(workflow.get("published", 0)),
|
||||
int(workflow.get("latest_version", 0)),
|
||||
),
|
||||
)
|
||||
|
||||
def get_workflow(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute("SELECT * FROM workflows WHERE id = ?", (workflow_id,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def list_workflows(self) -> list[dict[str, Any]]:
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute("SELECT * FROM workflows ORDER BY created_at DESC").fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def delete_workflow(self, workflow_id: str) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute("DELETE FROM artifacts WHERE run_id IN (SELECT id FROM workflow_runs WHERE workflow_id = ?)", (workflow_id,))
|
||||
conn.execute("DELETE FROM workflow_runs WHERE workflow_id = ?", (workflow_id,))
|
||||
conn.execute("DELETE FROM workflow_versions WHERE workflow_id = ?", (workflow_id,))
|
||||
conn.execute("DELETE FROM workflows WHERE id = ?", (workflow_id,))
|
||||
|
||||
def create_workflow_version(self, workflow_id: str, version: int, definition: dict[str, Any]) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO workflow_versions (workflow_id, version, definition_json)
|
||||
VALUES (?, ?, ?)
|
||||
""",
|
||||
(workflow_id, version, json.dumps(definition, ensure_ascii=False)),
|
||||
)
|
||||
|
||||
def get_latest_workflow_version(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM workflow_versions
|
||||
WHERE workflow_id = ?
|
||||
ORDER BY version DESC
|
||||
LIMIT 1
|
||||
""",
|
||||
(workflow_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
result = dict(row)
|
||||
result["definition"] = json.loads(result.pop("definition_json"))
|
||||
return result
|
||||
|
||||
def get_workflow_version(self, workflow_id: str, version: int) -> dict[str, Any] | None:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM workflow_versions
|
||||
WHERE workflow_id = ? AND version = ?
|
||||
""",
|
||||
(workflow_id, version),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
result = dict(row)
|
||||
result["definition"] = json.loads(result.pop("definition_json"))
|
||||
return result
|
||||
|
||||
def list_workflow_versions(self, workflow_id: str) -> list[dict[str, Any]]:
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"""
|
||||
SELECT * FROM workflow_versions
|
||||
WHERE workflow_id = ?
|
||||
ORDER BY version DESC
|
||||
""",
|
||||
(workflow_id,),
|
||||
).fetchall()
|
||||
versions = []
|
||||
for row in rows:
|
||||
item = dict(row)
|
||||
item["definition"] = json.loads(item.pop("definition_json"))
|
||||
versions.append(item)
|
||||
return versions
|
||||
|
||||
def create_run(self, run: dict[str, Any]) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO workflow_runs (
|
||||
id, workflow_id, workflow_version, status, current_node_id,
|
||||
progress, error, input_uri, created_at, updated_at
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
run["id"],
|
||||
run["workflow_id"],
|
||||
run["workflow_version"],
|
||||
run["status"],
|
||||
run.get("current_node_id"),
|
||||
float(run.get("progress", 0)),
|
||||
run.get("error"),
|
||||
run.get("input_uri"),
|
||||
run["created_at"],
|
||||
run["updated_at"],
|
||||
),
|
||||
)
|
||||
|
||||
def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute("SELECT * FROM workflow_runs WHERE id = ?", (run_id,)).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def list_runs(self, limit: int = 20) -> list[dict[str, Any]]:
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM workflow_runs ORDER BY created_at DESC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def update_run(self, run_id: str, **fields: Any) -> None:
|
||||
allowed = {
|
||||
"status",
|
||||
"current_node_id",
|
||||
"progress",
|
||||
"error",
|
||||
}
|
||||
updates = {key: value for key, value in fields.items() if key in allowed}
|
||||
if not updates:
|
||||
return
|
||||
updates["updated_at"] = fields.get("updated_at")
|
||||
assignments = ", ".join(f"{key} = ?" for key in updates)
|
||||
values = list(updates.values()) + [run_id]
|
||||
with self._connect() as conn:
|
||||
conn.execute(f"UPDATE workflow_runs SET {assignments} WHERE id = ?", values)
|
||||
|
||||
def next_queued_run(self) -> dict[str, Any] | None:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"""
|
||||
SELECT * FROM workflow_runs
|
||||
WHERE status = 'QUEUED'
|
||||
ORDER BY created_at ASC
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def create_artifact(self, artifact: dict[str, Any]) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO artifacts (
|
||||
run_id, node_id, name, uri, mime_type, size
|
||||
)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
artifact["run_id"],
|
||||
artifact["node_id"],
|
||||
artifact["name"],
|
||||
artifact["uri"],
|
||||
artifact.get("mime_type", ""),
|
||||
int(artifact.get("size", 0)),
|
||||
),
|
||||
)
|
||||
|
||||
def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
with self._connect() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM artifacts WHERE run_id = ? ORDER BY created_at",
|
||||
(run_id,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
def get_artifact(self, run_id: str, name: str) -> dict[str, Any] | None:
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM artifacts WHERE run_id = ? AND name = ?",
|
||||
(run_id, name),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
def delete_run_artifacts(self, run_id: str) -> None:
|
||||
with self._connect() as conn:
|
||||
conn.execute("DELETE FROM artifacts WHERE run_id = ?", (run_id,))
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from app.config import DATA_DIR, DB_PATH, STORAGE_DIR, WORKSPACE_ROOT
|
||||
from app.db import Database
|
||||
from app.node_manager import NodeManager
|
||||
from app.routers import apps, instances, nodes, workflows
|
||||
from app.scheduler import WorkflowScheduler
|
||||
from app.seed import seed_demo_workflow, seed_nodes
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
STORAGE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
db = Database(DB_PATH)
|
||||
manager = NodeManager(db)
|
||||
manager.start_reaper()
|
||||
if os.getenv("WOV_AUTO_SEED", "1") == "1":
|
||||
seed_nodes(db, WORKSPACE_ROOT)
|
||||
seed_demo_workflow(db)
|
||||
scheduler = WorkflowScheduler(db, manager, STORAGE_DIR)
|
||||
if os.getenv("WOV_SCHEDULER_ENABLED", "1") == "1":
|
||||
scheduler.start()
|
||||
app.state.db = db
|
||||
app.state.node_manager = manager
|
||||
app.state.scheduler = scheduler
|
||||
yield
|
||||
scheduler.stop()
|
||||
manager.shutdown()
|
||||
|
||||
|
||||
app = FastAPI(title="WOV API", version="0.1.0", lifespan=lifespan)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(nodes.router)
|
||||
app.include_router(instances.router)
|
||||
app.include_router(workflows.router)
|
||||
app.include_router(apps.router)
|
||||
|
||||
@app.get("/health")
|
||||
def health() -> dict:
|
||||
return {"status": "ok", "service": "wov-api"}
|
||||
|
||||
|
||||
FRONTEND_DIR = Path(__file__).resolve().parent.parent.parent / "wov-web"
|
||||
app.mount("/", StaticFiles(directory=FRONTEND_DIR, html=True), name="frontend")
|
||||
@@ -0,0 +1,298 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from wov_sdk.models import InvokeRequest, InvokeResponse, NodeManifest
|
||||
|
||||
from app.config import NODE_READY_TIMEOUT_SECONDS, NODE_REAP_INTERVAL_SECONDS, SDK_SRC, WORKSPACE_ROOT
|
||||
from app.db import Database
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _idle_seconds(last_used_at: str) -> float:
|
||||
if not last_used_at:
|
||||
return 0.0
|
||||
try:
|
||||
last = datetime.fromisoformat(last_used_at)
|
||||
return max(0.0, (datetime.now(timezone.utc) - last).total_seconds())
|
||||
except ValueError:
|
||||
return 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class NodeRuntime:
|
||||
instance_id: str
|
||||
node_id: str
|
||||
manifest: NodeManifest
|
||||
status: str = "starting"
|
||||
process: subprocess.Popen | None = None
|
||||
pid: int | None = None
|
||||
address: str | None = None
|
||||
started_at: str = field(default_factory=_now_iso)
|
||||
last_used_at: str = field(default_factory=_now_iso)
|
||||
busy_count: int = 0
|
||||
error: str | None = None
|
||||
stderr_tail: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class NodeManager:
|
||||
def __init__(self, db: Database) -> None:
|
||||
self.db = db
|
||||
self._runtimes: dict[str, NodeRuntime] = {}
|
||||
self._lock = threading.RLock()
|
||||
self._reaper_thread: threading.Thread | None = None
|
||||
self._stopping = False
|
||||
|
||||
def start_reaper(self) -> None:
|
||||
self._stopping = False
|
||||
self._reaper_thread = threading.Thread(
|
||||
target=self._reaper_loop,
|
||||
name="wov-node-reaper",
|
||||
daemon=True,
|
||||
)
|
||||
self._reaper_thread.start()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._stopping = True
|
||||
with self._lock:
|
||||
for runtime in list(self._runtimes.values()):
|
||||
self._stop_locked(runtime)
|
||||
|
||||
def _reaper_loop(self) -> None:
|
||||
while not self._stopping:
|
||||
time.sleep(NODE_REAP_INTERVAL_SECONDS)
|
||||
with self._lock:
|
||||
for runtime in list(self._runtimes.values()):
|
||||
if runtime.status != "ready" or runtime.busy_count > 0:
|
||||
continue
|
||||
if runtime.manifest.keep_warm:
|
||||
continue
|
||||
ttl = runtime.manifest.idle_ttl_seconds
|
||||
if ttl >= 0 and _idle_seconds(runtime.last_used_at) >= ttl:
|
||||
runtime.status = "stopping"
|
||||
self._stop_locked(runtime)
|
||||
|
||||
def _resolve_command(self, manifest: NodeManifest) -> list[str]:
|
||||
command = list(manifest.command)
|
||||
if command and command[0].lower() in {"python", "python3"}:
|
||||
repo_dir = WORKSPACE_ROOT / manifest.repo_dir
|
||||
windows_python = repo_dir / ".venv" / "Scripts" / "python.exe"
|
||||
unix_python = repo_dir / ".venv" / "bin" / "python"
|
||||
if windows_python.is_file():
|
||||
command[0] = str(windows_python)
|
||||
elif unix_python.is_file():
|
||||
command[0] = str(unix_python)
|
||||
else:
|
||||
command[0] = sys.executable
|
||||
return command
|
||||
|
||||
def _node_env(self, manifest: NodeManifest) -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
current_pythonpath = env.get("PYTHONPATH", "")
|
||||
env["PYTHONPATH"] = os.pathsep.join(
|
||||
item for item in [str(SDK_SRC), current_pythonpath] if item
|
||||
)
|
||||
env.update(manifest.env)
|
||||
return env
|
||||
|
||||
def _start_locked(self, manifest: NodeManifest) -> NodeRuntime:
|
||||
runtime = NodeRuntime(
|
||||
instance_id=f"ni_{uuid.uuid4().hex[:12]}",
|
||||
node_id=manifest.id,
|
||||
manifest=manifest,
|
||||
)
|
||||
repo_dir = WORKSPACE_ROOT / manifest.repo_dir
|
||||
if not repo_dir.is_dir():
|
||||
runtime.status = "error"
|
||||
runtime.error = f"repo_dir not found: {repo_dir}"
|
||||
self.db.upsert_instance(self._instance_row(runtime))
|
||||
return runtime
|
||||
|
||||
ready_event = threading.Event()
|
||||
ready_port: dict[str, int] = {}
|
||||
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
self._resolve_command(manifest),
|
||||
cwd=str(repo_dir),
|
||||
env=self._node_env(manifest),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
bufsize=1,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
runtime.status = "error"
|
||||
runtime.error = str(exc)
|
||||
self.db.upsert_instance(self._instance_row(runtime))
|
||||
return runtime
|
||||
|
||||
runtime.process = process
|
||||
runtime.pid = process.pid
|
||||
|
||||
def read_stdout() -> None:
|
||||
assert process.stdout is not None
|
||||
for line in iter(process.stdout.readline, ""):
|
||||
line = line.strip()
|
||||
if line.startswith("WOV_NODE_READY"):
|
||||
try:
|
||||
ready_port["port"] = int(line.split("port=", 1)[1])
|
||||
except (IndexError, ValueError):
|
||||
ready_port["port"] = 0
|
||||
ready_event.set()
|
||||
|
||||
def read_stderr() -> None:
|
||||
assert process.stderr is not None
|
||||
for line in iter(process.stderr.readline, ""):
|
||||
line = line.strip()
|
||||
if line:
|
||||
runtime.stderr_tail.append(line)
|
||||
runtime.stderr_tail = runtime.stderr_tail[-200:]
|
||||
|
||||
threading.Thread(target=read_stdout, daemon=True).start()
|
||||
threading.Thread(target=read_stderr, daemon=True).start()
|
||||
|
||||
ready = ready_event.wait(timeout=NODE_READY_TIMEOUT_SECONDS)
|
||||
if not ready or "port" not in ready_port or not ready_port["port"]:
|
||||
self._stop_process(process)
|
||||
runtime.status = "error"
|
||||
runtime.error = "node did not report readiness"
|
||||
self.db.upsert_instance(self._instance_row(runtime))
|
||||
return runtime
|
||||
|
||||
address = f"http://127.0.0.1:{ready_port['port']}"
|
||||
try:
|
||||
with urllib.request.urlopen(f"{address}/health", timeout=2) as response:
|
||||
if response.status != 200:
|
||||
raise RuntimeError(f"health check returned {response.status}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._stop_process(process)
|
||||
runtime.status = "error"
|
||||
runtime.error = f"health check failed: {exc}"
|
||||
self.db.upsert_instance(self._instance_row(runtime))
|
||||
return runtime
|
||||
|
||||
runtime.address = address
|
||||
runtime.status = "ready"
|
||||
runtime.started_at = _now_iso()
|
||||
runtime.last_used_at = _now_iso()
|
||||
self._runtimes[runtime.instance_id] = runtime
|
||||
self.db.upsert_instance(self._instance_row(runtime))
|
||||
return runtime
|
||||
|
||||
def _stop_process(self, process: subprocess.Popen) -> None:
|
||||
if process.poll() is not None:
|
||||
return
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=5)
|
||||
|
||||
def _stop_locked(self, runtime: NodeRuntime) -> None:
|
||||
if runtime.process is not None:
|
||||
self._stop_process(runtime.process)
|
||||
runtime.status = "stopped"
|
||||
runtime.pid = None
|
||||
runtime.address = None
|
||||
self._runtimes.pop(runtime.instance_id, None)
|
||||
self.db.upsert_instance(self._instance_row(runtime))
|
||||
|
||||
def _instance_row(self, runtime: NodeRuntime) -> dict:
|
||||
return {
|
||||
"id": runtime.instance_id,
|
||||
"node_id": runtime.node_id,
|
||||
"status": runtime.status,
|
||||
"pid": runtime.pid,
|
||||
"address": runtime.address,
|
||||
"started_at": runtime.started_at,
|
||||
"last_used_at": runtime.last_used_at,
|
||||
"busy_since": None,
|
||||
"error": runtime.error,
|
||||
}
|
||||
|
||||
def acquire(self, node_id: str) -> tuple[NodeRuntime, str]:
|
||||
manifest = self.db.get_node(node_id)
|
||||
if manifest is None:
|
||||
raise ValueError(f"node not registered: {node_id}")
|
||||
|
||||
with self._lock:
|
||||
for runtime in self._runtimes.values():
|
||||
if (
|
||||
runtime.node_id == node_id
|
||||
and runtime.status == "ready"
|
||||
and runtime.busy_count < manifest.max_concurrency
|
||||
):
|
||||
runtime.busy_count += 1
|
||||
runtime.last_used_at = _now_iso()
|
||||
return runtime, runtime.address or ""
|
||||
|
||||
runtime = self._start_locked(manifest)
|
||||
if runtime.status != "ready":
|
||||
raise RuntimeError(runtime.error or "node failed to start")
|
||||
runtime.busy_count += 1
|
||||
runtime.last_used_at = _now_iso()
|
||||
return runtime, runtime.address or ""
|
||||
|
||||
def release(self, instance_id: str) -> None:
|
||||
with self._lock:
|
||||
runtime = self._runtimes.get(instance_id)
|
||||
if runtime is None:
|
||||
return
|
||||
runtime.busy_count = max(0, runtime.busy_count - 1)
|
||||
runtime.last_used_at = _now_iso()
|
||||
self.db.upsert_instance(self._instance_row(runtime))
|
||||
|
||||
def invoke(self, node_id: str, request: InvokeRequest) -> InvokeResponse:
|
||||
runtime, address = self.acquire(node_id)
|
||||
request.node_instance_id = runtime.instance_id
|
||||
body = json.dumps(request.to_dict(), ensure_ascii=False).encode("utf-8")
|
||||
http_request = urllib.request.Request(
|
||||
f"{address}/invoke",
|
||||
data=body,
|
||||
headers={"Content-Type": "application/json; charset=utf-8"},
|
||||
method="POST",
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(http_request, timeout=300) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
return InvokeResponse.from_dict(payload)
|
||||
except urllib.error.HTTPError as exc:
|
||||
try:
|
||||
payload = json.loads(exc.read().decode("utf-8"))
|
||||
return InvokeResponse.from_dict(payload)
|
||||
except Exception: # noqa: BLE001
|
||||
return InvokeResponse(status="failed", error=f"node returned {exc.code}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return InvokeResponse(status="failed", error=str(exc))
|
||||
finally:
|
||||
self.release(request.node_instance_id)
|
||||
|
||||
def stop_instance(self, instance_id: str) -> None:
|
||||
with self._lock:
|
||||
runtime = self._runtimes.get(instance_id)
|
||||
if runtime is None:
|
||||
return
|
||||
self._stop_locked(runtime)
|
||||
|
||||
def stop_all_for_node(self, node_id: str) -> None:
|
||||
with self._lock:
|
||||
for runtime in list(self._runtimes.values()):
|
||||
if runtime.node_id == node_id:
|
||||
self._stop_locked(runtime)
|
||||
@@ -0,0 +1 @@
|
||||
"""WOV API routers."""
|
||||
@@ -0,0 +1,126 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from app.db import Database
|
||||
|
||||
router = APIRouter(tags=["apps"])
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _get_db() -> Database:
|
||||
from app.main import app
|
||||
|
||||
return app.state.db
|
||||
|
||||
|
||||
@router.get("/api/apps")
|
||||
def list_apps(db: Database = Depends(_get_db)) -> list[dict]:
|
||||
apps = []
|
||||
for workflow in db.list_workflows():
|
||||
if not workflow["published"]:
|
||||
continue
|
||||
latest = db.get_latest_workflow_version(workflow["id"])
|
||||
apps.append(
|
||||
{
|
||||
"id": workflow["id"],
|
||||
"name": workflow["name"],
|
||||
"description": workflow["description"],
|
||||
"version": workflow["latest_version"],
|
||||
"definition": latest["definition"] if latest else None,
|
||||
}
|
||||
)
|
||||
return apps
|
||||
|
||||
|
||||
@router.post("/api/apps/{workflow_id}/runs")
|
||||
async def create_run(
|
||||
workflow_id: str,
|
||||
file: UploadFile = File(...),
|
||||
db: Database = Depends(_get_db),
|
||||
) -> dict:
|
||||
workflow = db.get_workflow(workflow_id)
|
||||
if workflow is None or not workflow["published"]:
|
||||
raise HTTPException(status_code=404, detail="published workflow not found")
|
||||
|
||||
latest = db.get_latest_workflow_version(workflow_id)
|
||||
if latest is None:
|
||||
raise HTTPException(status_code=422, detail="workflow has no version")
|
||||
|
||||
run_id = f"run_{uuid.uuid4().hex[:12]}"
|
||||
filename = Path(file.filename or "upload.bin").name
|
||||
from app.config import STORAGE_DIR
|
||||
|
||||
input_dir = STORAGE_DIR / "uploads" / run_id
|
||||
input_dir.mkdir(parents=True, exist_ok=True)
|
||||
input_uri = input_dir / filename
|
||||
content = await file.read()
|
||||
input_uri.write_bytes(content)
|
||||
|
||||
now = _now_iso()
|
||||
db.create_run(
|
||||
{
|
||||
"id": run_id,
|
||||
"workflow_id": workflow_id,
|
||||
"workflow_version": latest["version"],
|
||||
"status": "QUEUED",
|
||||
"progress": 0,
|
||||
"input_uri": str(input_uri),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"id": run_id,
|
||||
"status": "QUEUED",
|
||||
"progress": 0,
|
||||
"artifacts": [],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/runs")
|
||||
def list_runs(db: Database = Depends(_get_db)) -> list[dict]:
|
||||
return db.list_runs()
|
||||
|
||||
|
||||
@router.get("/api/runs/{run_id}")
|
||||
def get_run(run_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
run = db.get_run(run_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail="run not found")
|
||||
run["artifacts"] = db.list_artifacts(run_id)
|
||||
return run
|
||||
|
||||
|
||||
@router.get("/api/runs/{run_id}/artifacts")
|
||||
def list_run_artifacts(run_id: str, db: Database = Depends(_get_db)) -> list[dict]:
|
||||
if db.get_run(run_id) is None:
|
||||
raise HTTPException(status_code=404, detail="run not found")
|
||||
return db.list_artifacts(run_id)
|
||||
|
||||
|
||||
@router.get("/api/runs/{run_id}/artifacts/{artifact_name}")
|
||||
def download_artifact(
|
||||
run_id: str,
|
||||
artifact_name: str,
|
||||
db: Database = Depends(_get_db),
|
||||
) -> FileResponse:
|
||||
artifact = db.get_artifact(run_id, artifact_name)
|
||||
if artifact is None:
|
||||
raise HTTPException(status_code=404, detail="artifact not found")
|
||||
path = Path(artifact["uri"])
|
||||
if not path.is_file():
|
||||
raise HTTPException(status_code=404, detail="artifact file missing")
|
||||
return FileResponse(
|
||||
path,
|
||||
media_type=artifact["mime_type"],
|
||||
filename=path.name,
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from app.db import Database
|
||||
from app.node_manager import NodeManager
|
||||
|
||||
router = APIRouter(prefix="/api/admin/node-instances", tags=["node-instances"])
|
||||
|
||||
|
||||
def _get_db() -> Database:
|
||||
from app.main import app
|
||||
|
||||
return app.state.db
|
||||
|
||||
|
||||
def _get_manager() -> NodeManager:
|
||||
from app.main import app
|
||||
|
||||
return app.state.node_manager
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_instances(db: Database = Depends(_get_db)) -> list[dict]:
|
||||
return db.list_instances()
|
||||
|
||||
|
||||
@router.post("/{instance_id}/stop")
|
||||
def stop_instance(
|
||||
instance_id: str,
|
||||
db: Database = Depends(_get_db),
|
||||
manager: NodeManager = Depends(_get_manager),
|
||||
) -> dict:
|
||||
manager.stop_instance(instance_id)
|
||||
instance = next(
|
||||
(item for item in db.list_instances() if item["id"] == instance_id),
|
||||
None,
|
||||
)
|
||||
if instance is None:
|
||||
raise HTTPException(status_code=404, detail="instance not found")
|
||||
return {"stopped": instance_id}
|
||||
@@ -0,0 +1,95 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from app.db import Database
|
||||
from app.node_manager import NodeManager
|
||||
from app.schemas import InvokePayload, NodeCreate
|
||||
from wov_sdk.models import InvokeRequest
|
||||
|
||||
router = APIRouter(prefix="/api/admin/nodes", tags=["nodes"])
|
||||
|
||||
|
||||
def _get_db() -> Database:
|
||||
from app.main import app
|
||||
|
||||
return app.state.db
|
||||
|
||||
|
||||
def _get_manager() -> NodeManager:
|
||||
from app.main import app
|
||||
|
||||
return app.state.node_manager
|
||||
|
||||
|
||||
@router.post("")
|
||||
def register_node(payload: NodeCreate, db: Database = Depends(_get_db)) -> dict:
|
||||
manifest = payload.to_manifest()
|
||||
try:
|
||||
manifest.validate()
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
db.upsert_node(manifest)
|
||||
return manifest.to_dict()
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_nodes(db: Database = Depends(_get_db)) -> list[dict]:
|
||||
return [manifest.to_dict() for manifest in db.list_nodes()]
|
||||
|
||||
|
||||
@router.get("/{node_id}")
|
||||
def get_node(node_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
manifest = db.get_node(node_id)
|
||||
if manifest is None:
|
||||
raise HTTPException(status_code=404, detail="node not found")
|
||||
return manifest.to_dict()
|
||||
|
||||
|
||||
@router.delete("/{node_id}")
|
||||
def delete_node(
|
||||
node_id: str,
|
||||
db: Database = Depends(_get_db),
|
||||
manager: NodeManager = Depends(_get_manager),
|
||||
) -> dict:
|
||||
if db.get_node(node_id) is None:
|
||||
raise HTTPException(status_code=404, detail="node not found")
|
||||
manager.stop_all_for_node(node_id)
|
||||
db.delete_node(node_id)
|
||||
return {"deleted": node_id}
|
||||
|
||||
|
||||
@router.post("/{node_id}/invoke")
|
||||
def invoke_node(
|
||||
node_id: str,
|
||||
payload: InvokePayload,
|
||||
manager: NodeManager = Depends(_get_manager),
|
||||
) -> dict:
|
||||
from app.config import STORAGE_DIR
|
||||
|
||||
output_dir = (
|
||||
STORAGE_DIR / "runs" / payload.run_id / "steps" / node_id
|
||||
)
|
||||
request = InvokeRequest(
|
||||
run_id=payload.run_id,
|
||||
node_instance_id="",
|
||||
inputs=payload.inputs,
|
||||
params=payload.params,
|
||||
output_dir=str(output_dir),
|
||||
)
|
||||
response = manager.invoke(node_id, request)
|
||||
return response.to_dict()
|
||||
|
||||
|
||||
@router.get("/{node_id}/instances")
|
||||
def list_node_instances(
|
||||
node_id: str,
|
||||
db: Database = Depends(_get_db),
|
||||
) -> list[dict]:
|
||||
if db.get_node(node_id) is None:
|
||||
raise HTTPException(status_code=404, detail="node not found")
|
||||
return [
|
||||
instance
|
||||
for instance in db.list_instances()
|
||||
if instance["node_id"] == node_id
|
||||
]
|
||||
@@ -0,0 +1,121 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from app.db import Database
|
||||
from app.schemas import WorkflowCreate
|
||||
from wov_sdk.models import WorkflowDefinition
|
||||
|
||||
router = APIRouter(prefix="/api/admin/workflows", tags=["workflows"])
|
||||
|
||||
|
||||
def _get_db() -> Database:
|
||||
from app.main import app
|
||||
|
||||
return app.state.db
|
||||
|
||||
|
||||
def _slugify(value: str) -> str:
|
||||
slug = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
|
||||
return slug or uuid.uuid4().hex[:8]
|
||||
|
||||
|
||||
def _validate_definition(raw: dict) -> WorkflowDefinition:
|
||||
try:
|
||||
definition = WorkflowDefinition.from_dict(raw)
|
||||
definition.validate()
|
||||
return definition
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_workflows(db: Database = Depends(_get_db)) -> list[dict]:
|
||||
return db.list_workflows()
|
||||
|
||||
|
||||
@router.post("")
|
||||
def create_workflow(
|
||||
payload: WorkflowCreate,
|
||||
db: Database = Depends(_get_db),
|
||||
) -> dict:
|
||||
definition = _validate_definition(payload.definition)
|
||||
workflow_id = payload.id or _slugify(payload.name)
|
||||
existing = db.get_workflow(workflow_id)
|
||||
version = (existing or {}).get("latest_version", 0) + 1
|
||||
db.upsert_workflow(
|
||||
{
|
||||
"id": workflow_id,
|
||||
"name": payload.name,
|
||||
"description": payload.description,
|
||||
"published": 0,
|
||||
"latest_version": version,
|
||||
}
|
||||
)
|
||||
db.create_workflow_version(workflow_id, version, definition.to_dict())
|
||||
return {
|
||||
"id": workflow_id,
|
||||
"name": payload.name,
|
||||
"description": payload.description,
|
||||
"published": False,
|
||||
"latest_version": version,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/{workflow_id}")
|
||||
def get_workflow(workflow_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
workflow = db.get_workflow(workflow_id)
|
||||
if workflow is None:
|
||||
raise HTTPException(status_code=404, detail="workflow not found")
|
||||
latest = db.get_latest_workflow_version(workflow_id)
|
||||
workflow["latest_version_data"] = latest
|
||||
return workflow
|
||||
|
||||
|
||||
@router.delete("/{workflow_id}")
|
||||
def delete_workflow(workflow_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
if db.get_workflow(workflow_id) is None:
|
||||
raise HTTPException(status_code=404, detail="workflow not found")
|
||||
db.delete_workflow(workflow_id)
|
||||
return {"deleted": workflow_id}
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/validate")
|
||||
def validate_workflow(
|
||||
workflow_id: str,
|
||||
definition: dict,
|
||||
db: Database = Depends(_get_db),
|
||||
) -> dict:
|
||||
if db.get_workflow(workflow_id) is None:
|
||||
raise HTTPException(status_code=404, detail="workflow not found")
|
||||
parsed = _validate_definition(definition)
|
||||
return {"valid": True, "node_ids": [node.id for node in parsed.nodes]}
|
||||
|
||||
|
||||
@router.post("/{workflow_id}/publish")
|
||||
def publish_workflow(workflow_id: str, db: Database = Depends(_get_db)) -> dict:
|
||||
workflow = db.get_workflow(workflow_id)
|
||||
if workflow is None:
|
||||
raise HTTPException(status_code=404, detail="workflow not found")
|
||||
if workflow["latest_version"] == 0:
|
||||
raise HTTPException(status_code=422, detail="workflow has no version")
|
||||
db.upsert_workflow(
|
||||
{
|
||||
"id": workflow_id,
|
||||
"name": workflow["name"],
|
||||
"description": workflow["description"],
|
||||
"published": 1,
|
||||
"latest_version": workflow["latest_version"],
|
||||
}
|
||||
)
|
||||
return {"published": workflow_id}
|
||||
|
||||
|
||||
@router.get("/{workflow_id}/versions")
|
||||
def list_versions(workflow_id: str, db: Database = Depends(_get_db)) -> list[dict]:
|
||||
if db.get_workflow(workflow_id) is None:
|
||||
raise HTTPException(status_code=404, detail="workflow not found")
|
||||
return db.list_workflow_versions(workflow_id)
|
||||
@@ -0,0 +1,215 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from wov_sdk.models import InvokeRequest, WorkflowDefinition
|
||||
|
||||
from app.db import Database
|
||||
from app.node_manager import NodeManager
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def topological_sort(definition: WorkflowDefinition) -> list[str]:
|
||||
nodes = {node.id: node for node in definition.nodes}
|
||||
indegree = {node_id: 0 for node_id in nodes}
|
||||
dependents: dict[str, list[str]] = {node_id: [] for node_id in nodes}
|
||||
|
||||
for edge in definition.edges:
|
||||
if edge.from_node not in nodes or edge.to_node not in nodes:
|
||||
raise ValueError(f"unknown edge: {edge.from_node} -> {edge.to_node}")
|
||||
indegree[edge.to_node] += 1
|
||||
dependents[edge.from_node].append(edge.to_node)
|
||||
|
||||
queue = [node_id for node_id, degree in indegree.items() if degree == 0]
|
||||
ordered: list[str] = []
|
||||
while queue:
|
||||
current = queue.pop(0)
|
||||
ordered.append(current)
|
||||
for dependent in dependents[current]:
|
||||
indegree[dependent] -= 1
|
||||
if indegree[dependent] == 0:
|
||||
queue.append(dependent)
|
||||
|
||||
if len(ordered) != len(nodes):
|
||||
raise ValueError("workflow contains a cycle")
|
||||
return ordered
|
||||
|
||||
|
||||
class WorkflowScheduler:
|
||||
def __init__(
|
||||
self,
|
||||
db: Database,
|
||||
node_manager: NodeManager,
|
||||
storage_dir: Path,
|
||||
interval_seconds: float = 1.0,
|
||||
) -> None:
|
||||
self.db = db
|
||||
self.node_manager = node_manager
|
||||
self.storage_dir = storage_dir
|
||||
self.interval_seconds = interval_seconds
|
||||
self._thread: threading.Thread | None = None
|
||||
self._stopping = False
|
||||
|
||||
def start(self) -> None:
|
||||
if self._thread is not None:
|
||||
return
|
||||
self._stopping = False
|
||||
self._thread = threading.Thread(
|
||||
target=self._loop,
|
||||
name="wov-workflow-scheduler",
|
||||
daemon=True,
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._stopping = True
|
||||
if self._thread is not None:
|
||||
self._thread.join(timeout=5)
|
||||
self._thread = None
|
||||
|
||||
def _loop(self) -> None:
|
||||
while not self._stopping:
|
||||
run = self.db.next_queued_run()
|
||||
if run is not None:
|
||||
self.execute_run(run["id"])
|
||||
else:
|
||||
time.sleep(self.interval_seconds)
|
||||
|
||||
def _resolve_ref(
|
||||
self,
|
||||
ref: str,
|
||||
run_input_uri: str | None,
|
||||
outputs_by_node: dict[str, dict[str, str]],
|
||||
) -> str | None:
|
||||
if ref.startswith("input."):
|
||||
return run_input_uri
|
||||
node_id, separator, key = ref.partition(".")
|
||||
if not separator:
|
||||
return None
|
||||
return outputs_by_node.get(node_id, {}).get(key)
|
||||
|
||||
def execute_run(self, run_id: str) -> None:
|
||||
run = self.db.get_run(run_id)
|
||||
if run is None or run["status"] != "QUEUED":
|
||||
return
|
||||
|
||||
workflow = self.db.get_workflow(run["workflow_id"])
|
||||
if workflow is None:
|
||||
self.db.update_run(run_id, status="FAILED", error="workflow not found", updated_at=_now_iso())
|
||||
return
|
||||
|
||||
version = self.db.get_workflow_version(run["workflow_id"], run["workflow_version"])
|
||||
if version is None:
|
||||
self.db.update_run(run_id, status="FAILED", error="workflow version not found", updated_at=_now_iso())
|
||||
return
|
||||
|
||||
definition = WorkflowDefinition.from_dict(version["definition"])
|
||||
definition.validate()
|
||||
ordered = topological_sort(definition)
|
||||
outputs_by_node: dict[str, dict[str, str]] = {}
|
||||
self.db.update_run(run_id, status="RUNNING", progress=0, updated_at=_now_iso())
|
||||
|
||||
try:
|
||||
for index, node_id in enumerate(ordered):
|
||||
node_spec = next(item for item in definition.nodes if item.id == node_id)
|
||||
self.db.update_run(
|
||||
run_id,
|
||||
current_node_id=node_id,
|
||||
progress=index / len(ordered),
|
||||
updated_at=_now_iso(),
|
||||
)
|
||||
invoke_inputs: dict[str, str] = {}
|
||||
for input_name, ref in node_spec.inputs.items():
|
||||
value = self._resolve_ref(ref, run.get("input_uri"), outputs_by_node)
|
||||
if value is None:
|
||||
raise ValueError(f"missing input {input_name} for node {node_id}")
|
||||
invoke_inputs[input_name] = value
|
||||
|
||||
output_dir = (
|
||||
self.storage_dir
|
||||
/ "runs"
|
||||
/ run_id
|
||||
/ "steps"
|
||||
/ node_id
|
||||
)
|
||||
response = self.node_manager.invoke(
|
||||
node_spec.node_type,
|
||||
InvokeRequest(
|
||||
run_id=run_id,
|
||||
node_instance_id="",
|
||||
inputs=invoke_inputs,
|
||||
params=node_spec.params,
|
||||
output_dir=str(output_dir),
|
||||
),
|
||||
)
|
||||
if response.status != "completed":
|
||||
raise RuntimeError(response.error or f"node {node_id} failed")
|
||||
|
||||
outputs_by_node[node_id] = {
|
||||
str(key): str(value) for key, value in response.outputs.items()
|
||||
}
|
||||
for key, uri in outputs_by_node[node_id].items():
|
||||
artifact = {
|
||||
"run_id": run_id,
|
||||
"node_id": node_id,
|
||||
"name": f"{node_id}.{key}",
|
||||
"uri": uri,
|
||||
"mime_type": self._mime_type(uri),
|
||||
"size": self._file_size(uri),
|
||||
}
|
||||
self.db.create_artifact(artifact)
|
||||
|
||||
for alias, ref in definition.final_outputs.items():
|
||||
resolved = self._resolve_ref(ref, run.get("input_uri"), outputs_by_node)
|
||||
if resolved is not None:
|
||||
self.db.create_artifact(
|
||||
{
|
||||
"run_id": run_id,
|
||||
"node_id": ref.partition(".")[0],
|
||||
"name": alias,
|
||||
"uri": resolved,
|
||||
"mime_type": self._mime_type(resolved),
|
||||
"size": self._file_size(resolved),
|
||||
}
|
||||
)
|
||||
|
||||
self.db.update_run(
|
||||
run_id,
|
||||
status="COMPLETED",
|
||||
current_node_id=None,
|
||||
progress=1.0,
|
||||
updated_at=_now_iso(),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.db.update_run(
|
||||
run_id,
|
||||
status="FAILED",
|
||||
error=str(exc),
|
||||
updated_at=_now_iso(),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _mime_type(uri: str) -> str:
|
||||
path = Path(uri)
|
||||
suffix = path.suffix.lower()
|
||||
return {
|
||||
".srt": "application/x-subrip",
|
||||
".ass": "text/plain",
|
||||
".wav": "audio/wav",
|
||||
".mp4": "video/mp4",
|
||||
".txt": "text/plain",
|
||||
}.get(suffix, "application/octet-stream")
|
||||
|
||||
@staticmethod
|
||||
def _file_size(uri: str) -> int:
|
||||
try:
|
||||
return Path(uri).stat().st_size
|
||||
except OSError:
|
||||
return 0
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from wov_sdk.models import NodeManifest
|
||||
|
||||
|
||||
class NodeCreate(BaseModel):
|
||||
id: str = Field(min_length=1)
|
||||
name: str = Field(min_length=1)
|
||||
version: str = Field(min_length=1)
|
||||
capability: str = Field(min_length=1)
|
||||
command: list[str] = Field(min_length=1)
|
||||
repo_dir: str = "."
|
||||
env: dict[str, str] = Field(default_factory=dict)
|
||||
input_schema: dict[str, Any] = Field(default_factory=dict)
|
||||
output_schema: dict[str, Any] = Field(default_factory=dict)
|
||||
max_concurrency: int = Field(default=1, ge=1)
|
||||
idle_ttl_seconds: int = Field(default=300, ge=0)
|
||||
health_timeout_seconds: int = Field(default=10, ge=1)
|
||||
keep_warm: bool = False
|
||||
|
||||
def to_manifest(self) -> NodeManifest:
|
||||
return NodeManifest(
|
||||
id=self.id,
|
||||
name=self.name,
|
||||
version=self.version,
|
||||
capability=self.capability,
|
||||
command=list(self.command),
|
||||
repo_dir=self.repo_dir,
|
||||
env=dict(self.env),
|
||||
input_schema=dict(self.input_schema),
|
||||
output_schema=dict(self.output_schema),
|
||||
max_concurrency=self.max_concurrency,
|
||||
idle_ttl_seconds=self.idle_ttl_seconds,
|
||||
health_timeout_seconds=self.health_timeout_seconds,
|
||||
keep_warm=self.keep_warm,
|
||||
)
|
||||
|
||||
|
||||
class InvokePayload(BaseModel):
|
||||
run_id: str = Field(default_factory=lambda: "run_admin")
|
||||
inputs: dict[str, Any] = Field(default_factory=dict)
|
||||
params: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class WorkflowCreate(BaseModel):
|
||||
id: str | None = None
|
||||
name: str = Field(min_length=1)
|
||||
description: str = ""
|
||||
definition: dict[str, Any]
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from wov_sdk.models import NodeManifest, WorkflowDefinition, WorkflowEdge, WorkflowNode
|
||||
|
||||
from app.db import Database
|
||||
|
||||
|
||||
def seed_nodes(db: Database, workspace_root: Path) -> None:
|
||||
for node_dir in sorted(workspace_root.glob("wov-node-*")):
|
||||
manifest_path = node_dir / "node.manifest.json"
|
||||
if not manifest_path.is_file():
|
||||
continue
|
||||
manifest = NodeManifest.load(str(manifest_path))
|
||||
manifest.repo_dir = node_dir.name
|
||||
db.upsert_node(manifest)
|
||||
|
||||
|
||||
def seed_demo_workflow(db: Database) -> None:
|
||||
if db.get_workflow("demo") is not None:
|
||||
return
|
||||
definition = WorkflowDefinition(
|
||||
name="视频字幕生成",
|
||||
version=1,
|
||||
nodes=[
|
||||
WorkflowNode(
|
||||
id="extract",
|
||||
node_type="ffmpeg-extract",
|
||||
params={"sample_rate": 16000, "channels": 1},
|
||||
inputs={"video_uri": "input.video_uri"},
|
||||
),
|
||||
WorkflowNode(
|
||||
id="asr",
|
||||
node_type="faster-whisper",
|
||||
params={"language": "ja"},
|
||||
inputs={"audio_uri": "extract.audio_uri"},
|
||||
),
|
||||
WorkflowNode(
|
||||
id="translate",
|
||||
node_type="llm-translate",
|
||||
params={"target_language": "zh-CN"},
|
||||
inputs={"srt_uri": "asr.srt_uri"},
|
||||
),
|
||||
WorkflowNode(
|
||||
id="ass",
|
||||
node_type="srt-to-dual-eye-ass",
|
||||
params={"resolution": "3840x1920"},
|
||||
inputs={"cn_srt_uri": "translate.cn_srt_uri"},
|
||||
),
|
||||
],
|
||||
edges=[
|
||||
WorkflowEdge(from_node="extract", to_node="asr"),
|
||||
WorkflowEdge(from_node="asr", to_node="translate"),
|
||||
WorkflowEdge(from_node="translate", to_node="ass"),
|
||||
],
|
||||
entry_inputs={"video_uri": "file"},
|
||||
final_outputs={
|
||||
"cn_srt": "translate.cn_srt_uri",
|
||||
"ass": "ass.ass_uri",
|
||||
},
|
||||
)
|
||||
db.upsert_workflow(
|
||||
{
|
||||
"id": "demo",
|
||||
"name": definition.name,
|
||||
"description": "上传视频，自动生成中文字幕和 VR 双眼 ASS。",
|
||||
"published": 1,
|
||||
"latest_version": 1,
|
||||
}
|
||||
)
|
||||
db.create_workflow_version("demo", 1, definition.to_dict())
|
||||
Reference in New Issue
Block a user