395 lines
12 KiB
Python
395 lines
12 KiB
Python
import subprocess
|
|
import sys
|
|
import time
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.db import Database
|
|
from app.node_manager import NodeManager, NodeRuntime, _idle_seconds, _now_iso
|
|
from wov_sdk.models import InvokeRequest, NodeManifest
|
|
|
|
WORKSPACE = Path(__file__).resolve().parent.parent.parent
|
|
_BASE_ECHO_MANIFEST = NodeManifest.load(WORKSPACE / "wov-node-echo" / "node.manifest.json")
|
|
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
|
|
|
|
|
@pytest.fixture
|
|
def db(tmp_path) -> Database:
|
|
return Database(tmp_path / "wov.db")
|
|
|
|
|
|
def _register(db: Database, manifest: NodeManifest) -> None:
|
|
db.upsert_node(manifest)
|
|
|
|
|
|
def echo_manifest() -> NodeManifest:
|
|
return NodeManifest.from_dict(_BASE_ECHO_MANIFEST.to_dict())
|
|
|
|
|
|
def test_time_helpers() -> None:
|
|
now = _now_iso()
|
|
assert datetime.fromisoformat(now)
|
|
assert _idle_seconds(now) < 0.1
|
|
old = (datetime.now(timezone.utc) - timedelta(seconds=10)).isoformat()
|
|
assert _idle_seconds(old) >= 9
|
|
assert _idle_seconds("") == 0
|
|
assert _idle_seconds("invalid") == 0
|
|
|
|
|
|
def test_resolve_command_and_env(db: Database) -> None:
|
|
manager = NodeManager(db)
|
|
python_command = NodeManifest(
|
|
id="x",
|
|
name="x",
|
|
version="1",
|
|
capability="x",
|
|
command=["python", "-m", "x"],
|
|
)
|
|
assert manager._resolve_command(python_command)[0] == sys.executable
|
|
|
|
fixed_command = NodeManifest(
|
|
id="x",
|
|
name="x",
|
|
version="1",
|
|
capability="x",
|
|
command=["node", "index.js"],
|
|
)
|
|
assert manager._resolve_command(fixed_command) == ["node", "index.js"]
|
|
|
|
env = manager._node_env(python_command)
|
|
assert "wov-sdk" in env["PYTHONPATH"]
|
|
|
|
|
|
def test_resolve_command_prefers_node_venv(db: Database, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.setattr("app.node_manager.WORKSPACE_ROOT", tmp_path)
|
|
repo = tmp_path / "wov-node-demo"
|
|
python_exe = repo / ".venv" / "Scripts" / "python.exe"
|
|
python_exe.parent.mkdir(parents=True)
|
|
python_exe.write_bytes(b"")
|
|
manifest = NodeManifest(
|
|
id="x",
|
|
name="x",
|
|
version="1",
|
|
capability="x",
|
|
command=["python", "-m", "x"],
|
|
repo_dir="wov-node-demo",
|
|
)
|
|
manager = NodeManager(db)
|
|
assert manager._resolve_command(manifest)[0] == str(python_exe)
|
|
|
|
python_exe.unlink()
|
|
unix_python = repo / ".venv" / "bin" / "python"
|
|
unix_python.parent.mkdir(parents=True)
|
|
unix_python.write_bytes(b"")
|
|
assert manager._resolve_command(manifest)[0] == str(unix_python)
|
|
|
|
|
|
def test_acquire_unregistered(db: Database) -> None:
|
|
manager = NodeManager(db)
|
|
with pytest.raises(ValueError):
|
|
manager.acquire("missing")
|
|
|
|
|
|
def test_start_missing_repo(db: Database) -> None:
|
|
manifest = echo_manifest()
|
|
manifest.repo_dir = "missing-repo"
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
with pytest.raises(RuntimeError, match="repo_dir not found"):
|
|
manager.acquire("echo")
|
|
assert db.list_instances()[0]["status"] == "error"
|
|
|
|
|
|
def test_start_command_not_found(db: Database) -> None:
|
|
manifest = echo_manifest()
|
|
manifest.command = ["definitely-not-a-real-wov-command"]
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
with pytest.raises(RuntimeError):
|
|
manager.acquire("echo")
|
|
assert db.list_instances()[0]["status"] == "error"
|
|
|
|
|
|
def test_start_no_ready_timeout(db: Database, monkeypatch) -> None:
|
|
monkeypatch.setattr("app.node_manager.NODE_READY_TIMEOUT_SECONDS", 0.2)
|
|
manifest = echo_manifest()
|
|
manifest.command = ["python", "-c", "import time; time.sleep(0.5)"]
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
with pytest.raises(RuntimeError, match="did not report readiness"):
|
|
manager.acquire("echo")
|
|
assert db.list_instances()[0]["status"] == "error"
|
|
|
|
|
|
def test_start_bad_ready_line(db: Database) -> None:
|
|
manifest = echo_manifest()
|
|
manifest.command = ["python", "-u", str(FIXTURES / "bad_ready_node.py")]
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
with pytest.raises(RuntimeError, match="did not report readiness"):
|
|
manager.acquire("echo")
|
|
assert db.list_instances()[0]["status"] == "error"
|
|
|
|
|
|
def test_start_health_check_failure(db: Database) -> None:
|
|
manifest = echo_manifest()
|
|
manifest.command = ["python", "-u", str(FIXTURES / "bad_node.py")]
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
with pytest.raises(RuntimeError, match="health check failed"):
|
|
manager.acquire("echo")
|
|
assert db.list_instances()[0]["status"] == "error"
|
|
|
|
|
|
def test_start_health_check_bad_status(db: Database) -> None:
|
|
manifest = echo_manifest()
|
|
manifest.command = ["python", "-u", str(FIXTURES / "bad_status_node.py")]
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
with pytest.raises(RuntimeError, match="health check returned 201"):
|
|
manager.acquire("echo")
|
|
assert db.list_instances()[0]["status"] == "error"
|
|
|
|
|
|
def test_stderr_is_captured(db: Database) -> None:
|
|
manifest = echo_manifest()
|
|
manifest.command = ["python", "-u", str(FIXTURES / "stderr_node.py")]
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
try:
|
|
runtime, _ = manager.acquire("echo")
|
|
time.sleep(0.2)
|
|
runtime = next(iter(manager._runtimes.values()))
|
|
assert "stderr warning" in runtime.stderr_tail
|
|
manager.release(runtime.instance_id)
|
|
finally:
|
|
manager.shutdown()
|
|
|
|
|
|
def test_acquire_reuses_ready_instance(db: Database) -> None:
|
|
_register(db, echo_manifest())
|
|
manager = NodeManager(db)
|
|
try:
|
|
first, first_address = manager.acquire("echo")
|
|
manager.release(first.instance_id)
|
|
second, second_address = manager.acquire("echo")
|
|
assert first.instance_id == second.instance_id
|
|
assert first_address == second_address
|
|
manager.release(second.instance_id)
|
|
finally:
|
|
manager.shutdown()
|
|
|
|
|
|
def test_invoke_success(db: Database, tmp_path) -> None:
|
|
_register(db, echo_manifest())
|
|
manager = NodeManager(db)
|
|
try:
|
|
response = manager.invoke(
|
|
"echo",
|
|
InvokeRequest(
|
|
run_id="run_ok",
|
|
node_instance_id="",
|
|
inputs={"text": "real path"},
|
|
output_dir=str(tmp_path),
|
|
),
|
|
)
|
|
assert response.status == "completed"
|
|
assert response.outputs["text"] == "real path"
|
|
finally:
|
|
manager.shutdown()
|
|
|
|
|
|
def test_invoke_http_error(db: Database, tmp_path) -> None:
|
|
manifest = echo_manifest()
|
|
manifest.command = ["python", "-u", str(FIXTURES / "failing_node.py")]
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
try:
|
|
response = manager.invoke(
|
|
"echo",
|
|
InvokeRequest(
|
|
run_id="run_fail",
|
|
node_instance_id="",
|
|
inputs={},
|
|
output_dir=str(tmp_path),
|
|
),
|
|
)
|
|
assert response.status == "failed"
|
|
assert response.error == "boom"
|
|
finally:
|
|
manager.shutdown()
|
|
|
|
|
|
def test_invoke_http_error_invalid_json(db: Database, tmp_path) -> None:
|
|
manifest = echo_manifest()
|
|
manifest.command = ["python", "-u", str(FIXTURES / "failing_node.py")]
|
|
manifest.env["WOV_FAIL_INVALID_JSON"] = "1"
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
try:
|
|
response = manager.invoke(
|
|
"echo",
|
|
InvokeRequest(
|
|
run_id="run_bad_json",
|
|
node_instance_id="",
|
|
inputs={},
|
|
output_dir=str(tmp_path),
|
|
),
|
|
)
|
|
assert response.status == "failed"
|
|
assert "node returned 500" in response.error
|
|
finally:
|
|
manager.shutdown()
|
|
|
|
|
|
def test_invoke_network_error(db: Database, tmp_path) -> None:
|
|
_register(db, echo_manifest())
|
|
manager = NodeManager(db)
|
|
try:
|
|
runtime, _ = manager.acquire("echo")
|
|
manager.release(runtime.instance_id)
|
|
runtime.process.terminate()
|
|
runtime.process.wait(timeout=5)
|
|
response = manager.invoke(
|
|
"echo",
|
|
InvokeRequest(
|
|
run_id="run_net",
|
|
node_instance_id="",
|
|
inputs={},
|
|
output_dir=str(tmp_path),
|
|
),
|
|
)
|
|
assert response.status == "failed"
|
|
finally:
|
|
manager.shutdown()
|
|
|
|
|
|
def test_release_unknown(db: Database) -> None:
|
|
manager = NodeManager(db)
|
|
manager.release("missing")
|
|
|
|
|
|
def test_stop_unknown_instance(db: Database) -> None:
|
|
manager = NodeManager(db)
|
|
manager.stop_instance("missing")
|
|
|
|
|
|
def test_stop_all_for_node(db: Database) -> None:
|
|
_register(db, echo_manifest())
|
|
manager = NodeManager(db)
|
|
runtime, _ = manager.acquire("echo")
|
|
manager.stop_all_for_node("echo")
|
|
assert runtime.status == "stopped"
|
|
assert runtime.instance_id not in manager._runtimes
|
|
|
|
|
|
def test_stop_locked_without_process(db: Database) -> None:
|
|
_register(db, echo_manifest())
|
|
manager = NodeManager(db)
|
|
runtime = NodeRuntime(
|
|
instance_id="ni_static",
|
|
node_id="echo",
|
|
manifest=echo_manifest(),
|
|
status="ready",
|
|
)
|
|
manager._runtimes[runtime.instance_id] = runtime
|
|
manager.stop_instance(runtime.instance_id)
|
|
assert runtime.status == "stopped"
|
|
assert runtime.instance_id not in manager._runtimes
|
|
|
|
|
|
def test_stop_process_already_exited() -> None:
|
|
class FakeProcess:
|
|
def poll(self):
|
|
return 0
|
|
|
|
manager = object.__new__(NodeManager)
|
|
manager._stop_process(FakeProcess())
|
|
|
|
|
|
def test_stop_process_timeout() -> None:
|
|
class FakeProcess:
|
|
def poll(self):
|
|
return None
|
|
|
|
def terminate(self):
|
|
self.terminated = True
|
|
|
|
def wait(self, timeout):
|
|
if not self.killed:
|
|
raise subprocess.TimeoutExpired("fake", timeout)
|
|
|
|
def kill(self):
|
|
self.killed = True
|
|
|
|
fake = FakeProcess()
|
|
fake.killed = False
|
|
manager = object.__new__(NodeManager)
|
|
manager._stop_process(fake)
|
|
assert fake.terminated
|
|
assert fake.killed
|
|
|
|
|
|
def test_reaper_recycles_idle(db: Database, tmp_path) -> None:
|
|
manifest = echo_manifest()
|
|
manifest.idle_ttl_seconds = 0
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
manager.start_reaper()
|
|
try:
|
|
response = manager.invoke(
|
|
"echo",
|
|
InvokeRequest(
|
|
run_id="run_reap",
|
|
node_instance_id="",
|
|
inputs={"text": "reap"},
|
|
output_dir=str(tmp_path),
|
|
),
|
|
)
|
|
assert response.status == "completed"
|
|
time.sleep(0.4)
|
|
assert db.list_instances()[0]["status"] == "stopped"
|
|
finally:
|
|
manager.shutdown()
|
|
|
|
|
|
def test_reaper_keeps_warm(db: Database, tmp_path) -> None:
|
|
manifest = echo_manifest()
|
|
manifest.idle_ttl_seconds = 0
|
|
manifest.keep_warm = True
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
manager.start_reaper()
|
|
try:
|
|
response = manager.invoke(
|
|
"echo",
|
|
InvokeRequest(
|
|
run_id="run_warm",
|
|
node_instance_id="",
|
|
inputs={"text": "warm"},
|
|
output_dir=str(tmp_path),
|
|
),
|
|
)
|
|
assert response.status == "completed"
|
|
time.sleep(0.4)
|
|
assert db.list_instances()[0]["status"] == "ready"
|
|
finally:
|
|
manager.shutdown()
|
|
|
|
|
|
def test_reaper_keeps_busy(db: Database) -> None:
|
|
manifest = echo_manifest()
|
|
manifest.idle_ttl_seconds = 0
|
|
_register(db, manifest)
|
|
manager = NodeManager(db)
|
|
manager.start_reaper()
|
|
try:
|
|
runtime, _ = manager.acquire("echo")
|
|
time.sleep(0.4)
|
|
assert runtime.status == "ready"
|
|
manager.release(runtime.instance_id)
|
|
finally:
|
|
manager.shutdown()
|