feat: 完成工作流调度与上传下载 API
This commit is contained in:
cat-shark
@@ -0,0 +1,394 @@
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.db import Database
|
||||
from app.node_manager import NodeManager, NodeRuntime, _idle_seconds, _now_iso
|
||||
from wov_sdk.models import InvokeRequest, NodeManifest
|
||||
|
||||
WORKSPACE = Path(__file__).resolve().parent.parent.parent
|
||||
_BASE_ECHO_MANIFEST = NodeManifest.load(WORKSPACE / "wov-node-echo" / "node.manifest.json")
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(tmp_path) -> Database:
|
||||
return Database(tmp_path / "wov.db")
|
||||
|
||||
|
||||
def _register(db: Database, manifest: NodeManifest) -> None:
|
||||
db.upsert_node(manifest)
|
||||
|
||||
|
||||
def echo_manifest() -> NodeManifest:
|
||||
return NodeManifest.from_dict(_BASE_ECHO_MANIFEST.to_dict())
|
||||
|
||||
|
||||
def test_time_helpers() -> None:
|
||||
now = _now_iso()
|
||||
assert datetime.fromisoformat(now)
|
||||
assert _idle_seconds(now) < 0.1
|
||||
old = (datetime.now(timezone.utc) - timedelta(seconds=10)).isoformat()
|
||||
assert _idle_seconds(old) >= 9
|
||||
assert _idle_seconds("") == 0
|
||||
assert _idle_seconds("invalid") == 0
|
||||
|
||||
|
||||
def test_resolve_command_and_env(db: Database) -> None:
|
||||
manager = NodeManager(db)
|
||||
python_command = NodeManifest(
|
||||
id="x",
|
||||
name="x",
|
||||
version="1",
|
||||
capability="x",
|
||||
command=["python", "-m", "x"],
|
||||
)
|
||||
assert manager._resolve_command(python_command)[0] == sys.executable
|
||||
|
||||
fixed_command = NodeManifest(
|
||||
id="x",
|
||||
name="x",
|
||||
version="1",
|
||||
capability="x",
|
||||
command=["node", "index.js"],
|
||||
)
|
||||
assert manager._resolve_command(fixed_command) == ["node", "index.js"]
|
||||
|
||||
env = manager._node_env(python_command)
|
||||
assert "wov-sdk" in env["PYTHONPATH"]
|
||||
|
||||
|
||||
def test_resolve_command_prefers_node_venv(db: Database, tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.setattr("app.node_manager.WORKSPACE_ROOT", tmp_path)
|
||||
repo = tmp_path / "wov-node-demo"
|
||||
python_exe = repo / ".venv" / "Scripts" / "python.exe"
|
||||
python_exe.parent.mkdir(parents=True)
|
||||
python_exe.write_bytes(b"")
|
||||
manifest = NodeManifest(
|
||||
id="x",
|
||||
name="x",
|
||||
version="1",
|
||||
capability="x",
|
||||
command=["python", "-m", "x"],
|
||||
repo_dir="wov-node-demo",
|
||||
)
|
||||
manager = NodeManager(db)
|
||||
assert manager._resolve_command(manifest)[0] == str(python_exe)
|
||||
|
||||
python_exe.unlink()
|
||||
unix_python = repo / ".venv" / "bin" / "python"
|
||||
unix_python.parent.mkdir(parents=True)
|
||||
unix_python.write_bytes(b"")
|
||||
assert manager._resolve_command(manifest)[0] == str(unix_python)
|
||||
|
||||
|
||||
def test_acquire_unregistered(db: Database) -> None:
|
||||
manager = NodeManager(db)
|
||||
with pytest.raises(ValueError):
|
||||
manager.acquire("missing")
|
||||
|
||||
|
||||
def test_start_missing_repo(db: Database) -> None:
|
||||
manifest = echo_manifest()
|
||||
manifest.repo_dir = "missing-repo"
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
with pytest.raises(RuntimeError, match="repo_dir not found"):
|
||||
manager.acquire("echo")
|
||||
assert db.list_instances()[0]["status"] == "error"
|
||||
|
||||
|
||||
def test_start_command_not_found(db: Database) -> None:
|
||||
manifest = echo_manifest()
|
||||
manifest.command = ["definitely-not-a-real-wov-command"]
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
with pytest.raises(RuntimeError):
|
||||
manager.acquire("echo")
|
||||
assert db.list_instances()[0]["status"] == "error"
|
||||
|
||||
|
||||
def test_start_no_ready_timeout(db: Database, monkeypatch) -> None:
|
||||
monkeypatch.setattr("app.node_manager.NODE_READY_TIMEOUT_SECONDS", 0.2)
|
||||
manifest = echo_manifest()
|
||||
manifest.command = ["python", "-c", "import time; time.sleep(0.5)"]
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
with pytest.raises(RuntimeError, match="did not report readiness"):
|
||||
manager.acquire("echo")
|
||||
assert db.list_instances()[0]["status"] == "error"
|
||||
|
||||
|
||||
def test_start_bad_ready_line(db: Database) -> None:
|
||||
manifest = echo_manifest()
|
||||
manifest.command = ["python", "-u", str(FIXTURES / "bad_ready_node.py")]
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
with pytest.raises(RuntimeError, match="did not report readiness"):
|
||||
manager.acquire("echo")
|
||||
assert db.list_instances()[0]["status"] == "error"
|
||||
|
||||
|
||||
def test_start_health_check_failure(db: Database) -> None:
|
||||
manifest = echo_manifest()
|
||||
manifest.command = ["python", "-u", str(FIXTURES / "bad_node.py")]
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
with pytest.raises(RuntimeError, match="health check failed"):
|
||||
manager.acquire("echo")
|
||||
assert db.list_instances()[0]["status"] == "error"
|
||||
|
||||
|
||||
def test_start_health_check_bad_status(db: Database) -> None:
|
||||
manifest = echo_manifest()
|
||||
manifest.command = ["python", "-u", str(FIXTURES / "bad_status_node.py")]
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
with pytest.raises(RuntimeError, match="health check returned 201"):
|
||||
manager.acquire("echo")
|
||||
assert db.list_instances()[0]["status"] == "error"
|
||||
|
||||
|
||||
def test_stderr_is_captured(db: Database) -> None:
|
||||
manifest = echo_manifest()
|
||||
manifest.command = ["python", "-u", str(FIXTURES / "stderr_node.py")]
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
try:
|
||||
runtime, _ = manager.acquire("echo")
|
||||
time.sleep(0.2)
|
||||
runtime = next(iter(manager._runtimes.values()))
|
||||
assert "stderr warning" in runtime.stderr_tail
|
||||
manager.release(runtime.instance_id)
|
||||
finally:
|
||||
manager.shutdown()
|
||||
|
||||
|
||||
def test_acquire_reuses_ready_instance(db: Database) -> None:
|
||||
_register(db, echo_manifest())
|
||||
manager = NodeManager(db)
|
||||
try:
|
||||
first, first_address = manager.acquire("echo")
|
||||
manager.release(first.instance_id)
|
||||
second, second_address = manager.acquire("echo")
|
||||
assert first.instance_id == second.instance_id
|
||||
assert first_address == second_address
|
||||
manager.release(second.instance_id)
|
||||
finally:
|
||||
manager.shutdown()
|
||||
|
||||
|
||||
def test_invoke_success(db: Database, tmp_path) -> None:
|
||||
_register(db, echo_manifest())
|
||||
manager = NodeManager(db)
|
||||
try:
|
||||
response = manager.invoke(
|
||||
"echo",
|
||||
InvokeRequest(
|
||||
run_id="run_ok",
|
||||
node_instance_id="",
|
||||
inputs={"text": "real path"},
|
||||
output_dir=str(tmp_path),
|
||||
),
|
||||
)
|
||||
assert response.status == "completed"
|
||||
assert response.outputs["text"] == "real path"
|
||||
finally:
|
||||
manager.shutdown()
|
||||
|
||||
|
||||
def test_invoke_http_error(db: Database, tmp_path) -> None:
|
||||
manifest = echo_manifest()
|
||||
manifest.command = ["python", "-u", str(FIXTURES / "failing_node.py")]
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
try:
|
||||
response = manager.invoke(
|
||||
"echo",
|
||||
InvokeRequest(
|
||||
run_id="run_fail",
|
||||
node_instance_id="",
|
||||
inputs={},
|
||||
output_dir=str(tmp_path),
|
||||
),
|
||||
)
|
||||
assert response.status == "failed"
|
||||
assert response.error == "boom"
|
||||
finally:
|
||||
manager.shutdown()
|
||||
|
||||
|
||||
def test_invoke_http_error_invalid_json(db: Database, tmp_path) -> None:
|
||||
manifest = echo_manifest()
|
||||
manifest.command = ["python", "-u", str(FIXTURES / "failing_node.py")]
|
||||
manifest.env["WOV_FAIL_INVALID_JSON"] = "1"
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
try:
|
||||
response = manager.invoke(
|
||||
"echo",
|
||||
InvokeRequest(
|
||||
run_id="run_bad_json",
|
||||
node_instance_id="",
|
||||
inputs={},
|
||||
output_dir=str(tmp_path),
|
||||
),
|
||||
)
|
||||
assert response.status == "failed"
|
||||
assert "node returned 500" in response.error
|
||||
finally:
|
||||
manager.shutdown()
|
||||
|
||||
|
||||
def test_invoke_network_error(db: Database, tmp_path) -> None:
|
||||
_register(db, echo_manifest())
|
||||
manager = NodeManager(db)
|
||||
try:
|
||||
runtime, _ = manager.acquire("echo")
|
||||
manager.release(runtime.instance_id)
|
||||
runtime.process.terminate()
|
||||
runtime.process.wait(timeout=5)
|
||||
response = manager.invoke(
|
||||
"echo",
|
||||
InvokeRequest(
|
||||
run_id="run_net",
|
||||
node_instance_id="",
|
||||
inputs={},
|
||||
output_dir=str(tmp_path),
|
||||
),
|
||||
)
|
||||
assert response.status == "failed"
|
||||
finally:
|
||||
manager.shutdown()
|
||||
|
||||
|
||||
def test_release_unknown(db: Database) -> None:
|
||||
manager = NodeManager(db)
|
||||
manager.release("missing")
|
||||
|
||||
|
||||
def test_stop_unknown_instance(db: Database) -> None:
|
||||
manager = NodeManager(db)
|
||||
manager.stop_instance("missing")
|
||||
|
||||
|
||||
def test_stop_all_for_node(db: Database) -> None:
|
||||
_register(db, echo_manifest())
|
||||
manager = NodeManager(db)
|
||||
runtime, _ = manager.acquire("echo")
|
||||
manager.stop_all_for_node("echo")
|
||||
assert runtime.status == "stopped"
|
||||
assert runtime.instance_id not in manager._runtimes
|
||||
|
||||
|
||||
def test_stop_locked_without_process(db: Database) -> None:
|
||||
_register(db, echo_manifest())
|
||||
manager = NodeManager(db)
|
||||
runtime = NodeRuntime(
|
||||
instance_id="ni_static",
|
||||
node_id="echo",
|
||||
manifest=echo_manifest(),
|
||||
status="ready",
|
||||
)
|
||||
manager._runtimes[runtime.instance_id] = runtime
|
||||
manager.stop_instance(runtime.instance_id)
|
||||
assert runtime.status == "stopped"
|
||||
assert runtime.instance_id not in manager._runtimes
|
||||
|
||||
|
||||
def test_stop_process_already_exited() -> None:
|
||||
class FakeProcess:
|
||||
def poll(self):
|
||||
return 0
|
||||
|
||||
manager = object.__new__(NodeManager)
|
||||
manager._stop_process(FakeProcess())
|
||||
|
||||
|
||||
def test_stop_process_timeout() -> None:
|
||||
class FakeProcess:
|
||||
def poll(self):
|
||||
return None
|
||||
|
||||
def terminate(self):
|
||||
self.terminated = True
|
||||
|
||||
def wait(self, timeout):
|
||||
if not self.killed:
|
||||
raise subprocess.TimeoutExpired("fake", timeout)
|
||||
|
||||
def kill(self):
|
||||
self.killed = True
|
||||
|
||||
fake = FakeProcess()
|
||||
fake.killed = False
|
||||
manager = object.__new__(NodeManager)
|
||||
manager._stop_process(fake)
|
||||
assert fake.terminated
|
||||
assert fake.killed
|
||||
|
||||
|
||||
def test_reaper_recycles_idle(db: Database, tmp_path) -> None:
|
||||
manifest = echo_manifest()
|
||||
manifest.idle_ttl_seconds = 0
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
manager.start_reaper()
|
||||
try:
|
||||
response = manager.invoke(
|
||||
"echo",
|
||||
InvokeRequest(
|
||||
run_id="run_reap",
|
||||
node_instance_id="",
|
||||
inputs={"text": "reap"},
|
||||
output_dir=str(tmp_path),
|
||||
),
|
||||
)
|
||||
assert response.status == "completed"
|
||||
time.sleep(0.4)
|
||||
assert db.list_instances()[0]["status"] == "stopped"
|
||||
finally:
|
||||
manager.shutdown()
|
||||
|
||||
|
||||
def test_reaper_keeps_warm(db: Database, tmp_path) -> None:
|
||||
manifest = echo_manifest()
|
||||
manifest.idle_ttl_seconds = 0
|
||||
manifest.keep_warm = True
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
manager.start_reaper()
|
||||
try:
|
||||
response = manager.invoke(
|
||||
"echo",
|
||||
InvokeRequest(
|
||||
run_id="run_warm",
|
||||
node_instance_id="",
|
||||
inputs={"text": "warm"},
|
||||
output_dir=str(tmp_path),
|
||||
),
|
||||
)
|
||||
assert response.status == "completed"
|
||||
time.sleep(0.4)
|
||||
assert db.list_instances()[0]["status"] == "ready"
|
||||
finally:
|
||||
manager.shutdown()
|
||||
|
||||
|
||||
def test_reaper_keeps_busy(db: Database) -> None:
|
||||
manifest = echo_manifest()
|
||||
manifest.idle_ttl_seconds = 0
|
||||
_register(db, manifest)
|
||||
manager = NodeManager(db)
|
||||
manager.start_reaper()
|
||||
try:
|
||||
runtime, _ = manager.acquire("echo")
|
||||
time.sleep(0.4)
|
||||
assert runtime.status == "ready"
|
||||
manager.release(runtime.instance_id)
|
||||
finally:
|
||||
manager.shutdown()
|
||||
Reference in New Issue
Block a user