import subprocess import sys import time from datetime import datetime, timedelta, timezone from pathlib import Path import pytest from app.db import Database from app.node_manager import NodeManager, NodeRuntime, _idle_seconds, _now_iso from wov_sdk.models import InvokeRequest, NodeManifest WORKSPACE = Path(__file__).resolve().parent.parent.parent _BASE_ECHO_MANIFEST = NodeManifest.load(WORKSPACE / "wov-node-echo" / "node.manifest.json") FIXTURES = Path(__file__).resolve().parent / "fixtures" @pytest.fixture def db(tmp_path) -> Database: return Database(tmp_path / "wov.db") def _register(db: Database, manifest: NodeManifest) -> None: db.upsert_node(manifest) def echo_manifest() -> NodeManifest: return NodeManifest.from_dict(_BASE_ECHO_MANIFEST.to_dict()) def test_time_helpers() -> None: now = _now_iso() assert datetime.fromisoformat(now) assert _idle_seconds(now) < 0.1 old = (datetime.now(timezone.utc) - timedelta(seconds=10)).isoformat() assert _idle_seconds(old) >= 9 assert _idle_seconds("") == 0 assert _idle_seconds("invalid") == 0 def test_resolve_command_and_env(db: Database) -> None: manager = NodeManager(db) python_command = NodeManifest( id="x", name="x", version="1", capability="x", command=["python", "-m", "x"], ) assert manager._resolve_command(python_command)[0] == sys.executable fixed_command = NodeManifest( id="x", name="x", version="1", capability="x", command=["node", "index.js"], ) assert manager._resolve_command(fixed_command) == ["node", "index.js"] env = manager._node_env(python_command) assert "wov-sdk" in env["PYTHONPATH"] def test_resolve_command_prefers_node_venv(db: Database, tmp_path, monkeypatch) -> None: monkeypatch.setattr("app.node_manager.WORKSPACE_ROOT", tmp_path) repo = tmp_path / "wov-node-demo" python_exe = repo / ".venv" / "Scripts" / "python.exe" python_exe.parent.mkdir(parents=True) python_exe.write_bytes(b"") manifest = NodeManifest( id="x", name="x", version="1", capability="x", command=["python", "-m", "x"], repo_dir="wov-node-demo", ) manager = NodeManager(db) assert manager._resolve_command(manifest)[0] == str(python_exe) python_exe.unlink() unix_python = repo / ".venv" / "bin" / "python" unix_python.parent.mkdir(parents=True) unix_python.write_bytes(b"") assert manager._resolve_command(manifest)[0] == str(unix_python) def test_acquire_unregistered(db: Database) -> None: manager = NodeManager(db) with pytest.raises(ValueError): manager.acquire("missing") def test_start_missing_repo(db: Database) -> None: manifest = echo_manifest() manifest.repo_dir = "missing-repo" _register(db, manifest) manager = NodeManager(db) with pytest.raises(RuntimeError, match="repo_dir not found"): manager.acquire("echo") assert db.list_instances()[0]["status"] == "error" def test_start_command_not_found(db: Database) -> None: manifest = echo_manifest() manifest.command = ["definitely-not-a-real-wov-command"] _register(db, manifest) manager = NodeManager(db) with pytest.raises(RuntimeError): manager.acquire("echo") assert db.list_instances()[0]["status"] == "error" def test_start_no_ready_timeout(db: Database, monkeypatch) -> None: monkeypatch.setattr("app.node_manager.NODE_READY_TIMEOUT_SECONDS", 0.2) manifest = echo_manifest() manifest.command = ["python", "-c", "import time; time.sleep(0.5)"] _register(db, manifest) manager = NodeManager(db) with pytest.raises(RuntimeError, match="did not report readiness"): manager.acquire("echo") assert db.list_instances()[0]["status"] == "error" def test_start_bad_ready_line(db: Database) -> None: manifest = echo_manifest() manifest.command = ["python", "-u", str(FIXTURES / "bad_ready_node.py")] _register(db, manifest) manager = NodeManager(db) with pytest.raises(RuntimeError, match="did not report readiness"): manager.acquire("echo") assert db.list_instances()[0]["status"] == "error" def test_start_health_check_failure(db: Database) -> None: manifest = echo_manifest() manifest.command = ["python", "-u", str(FIXTURES / "bad_node.py")] _register(db, manifest) manager = NodeManager(db) with pytest.raises(RuntimeError, match="health check failed"): manager.acquire("echo") assert db.list_instances()[0]["status"] == "error" def test_start_health_check_bad_status(db: Database) -> None: manifest = echo_manifest() manifest.command = ["python", "-u", str(FIXTURES / "bad_status_node.py")] _register(db, manifest) manager = NodeManager(db) with pytest.raises(RuntimeError, match="health check returned 201"): manager.acquire("echo") assert db.list_instances()[0]["status"] == "error" def test_stderr_is_captured(db: Database) -> None: manifest = echo_manifest() manifest.command = ["python", "-u", str(FIXTURES / "stderr_node.py")] _register(db, manifest) manager = NodeManager(db) try: runtime, _ = manager.acquire("echo") time.sleep(0.2) runtime = next(iter(manager._runtimes.values())) assert "stderr warning" in runtime.stderr_tail manager.release(runtime.instance_id) finally: manager.shutdown() def test_acquire_reuses_ready_instance(db: Database) -> None: _register(db, echo_manifest()) manager = NodeManager(db) try: first, first_address = manager.acquire("echo") manager.release(first.instance_id) second, second_address = manager.acquire("echo") assert first.instance_id == second.instance_id assert first_address == second_address manager.release(second.instance_id) finally: manager.shutdown() def test_invoke_success(db: Database, tmp_path) -> None: _register(db, echo_manifest()) manager = NodeManager(db) try: response = manager.invoke( "echo", InvokeRequest( run_id="run_ok", node_instance_id="", inputs={"text": "real path"}, output_dir=str(tmp_path), ), ) assert response.status == "completed" assert response.outputs["text"] == "real path" finally: manager.shutdown() def test_invoke_http_error(db: Database, tmp_path) -> None: manifest = echo_manifest() manifest.command = ["python", "-u", str(FIXTURES / "failing_node.py")] _register(db, manifest) manager = NodeManager(db) try: response = manager.invoke( "echo", InvokeRequest( run_id="run_fail", node_instance_id="", inputs={}, output_dir=str(tmp_path), ), ) assert response.status == "failed" assert response.error == "boom" finally: manager.shutdown() def test_invoke_http_error_invalid_json(db: Database, tmp_path) -> None: manifest = echo_manifest() manifest.command = ["python", "-u", str(FIXTURES / "failing_node.py")] manifest.env["WOV_FAIL_INVALID_JSON"] = "1" _register(db, manifest) manager = NodeManager(db) try: response = manager.invoke( "echo", InvokeRequest( run_id="run_bad_json", node_instance_id="", inputs={}, output_dir=str(tmp_path), ), ) assert response.status == "failed" assert "node returned 500" in response.error finally: manager.shutdown() def test_invoke_network_error(db: Database, tmp_path) -> None: _register(db, echo_manifest()) manager = NodeManager(db) try: runtime, _ = manager.acquire("echo") manager.release(runtime.instance_id) runtime.process.terminate() runtime.process.wait(timeout=5) response = manager.invoke( "echo", InvokeRequest( run_id="run_net", node_instance_id="", inputs={}, output_dir=str(tmp_path), ), ) assert response.status == "failed" finally: manager.shutdown() def test_release_unknown(db: Database) -> None: manager = NodeManager(db) manager.release("missing") def test_stop_unknown_instance(db: Database) -> None: manager = NodeManager(db) manager.stop_instance("missing") def test_stop_all_for_node(db: Database) -> None: _register(db, echo_manifest()) manager = NodeManager(db) runtime, _ = manager.acquire("echo") manager.stop_all_for_node("echo") assert runtime.status == "stopped" assert runtime.instance_id not in manager._runtimes def test_stop_locked_without_process(db: Database) -> None: _register(db, echo_manifest()) manager = NodeManager(db) runtime = NodeRuntime( instance_id="ni_static", node_id="echo", manifest=echo_manifest(), status="ready", ) manager._runtimes[runtime.instance_id] = runtime manager.stop_instance(runtime.instance_id) assert runtime.status == "stopped" assert runtime.instance_id not in manager._runtimes def test_stop_process_already_exited() -> None: class FakeProcess: def poll(self): return 0 manager = object.__new__(NodeManager) manager._stop_process(FakeProcess()) def test_stop_process_timeout() -> None: class FakeProcess: def poll(self): return None def terminate(self): self.terminated = True def wait(self, timeout): if not self.killed: raise subprocess.TimeoutExpired("fake", timeout) def kill(self): self.killed = True fake = FakeProcess() fake.killed = False manager = object.__new__(NodeManager) manager._stop_process(fake) assert fake.terminated assert fake.killed def test_reaper_recycles_idle(db: Database, tmp_path) -> None: manifest = echo_manifest() manifest.idle_ttl_seconds = 0 _register(db, manifest) manager = NodeManager(db) manager.start_reaper() try: response = manager.invoke( "echo", InvokeRequest( run_id="run_reap", node_instance_id="", inputs={"text": "reap"}, output_dir=str(tmp_path), ), ) assert response.status == "completed" time.sleep(0.4) assert db.list_instances()[0]["status"] == "stopped" finally: manager.shutdown() def test_reaper_keeps_warm(db: Database, tmp_path) -> None: manifest = echo_manifest() manifest.idle_ttl_seconds = 0 manifest.keep_warm = True _register(db, manifest) manager = NodeManager(db) manager.start_reaper() try: response = manager.invoke( "echo", InvokeRequest( run_id="run_warm", node_instance_id="", inputs={"text": "warm"}, output_dir=str(tmp_path), ), ) assert response.status == "completed" time.sleep(0.4) assert db.list_instances()[0]["status"] == "ready" finally: manager.shutdown() def test_reaper_keeps_busy(db: Database) -> None: manifest = echo_manifest() manifest.idle_ttl_seconds = 0 _register(db, manifest) manager = NodeManager(db) manager.start_reaper() try: runtime, _ = manager.acquire("echo") time.sleep(0.4) assert runtime.status == "ready" manager.release(runtime.instance_id) finally: manager.shutdown()