chore: add smoke test script for print + RPC mode validation

tests/test-smoke.mjs

@@ -0,0 +1,339 @@
+/**
+ * Integration smoke test for pi-commandcode-provider.
+ *
+ * Tests that the extension:
+ * 1. Loads without crashing in print mode
+ * 2. Registers the provider and models
+ * 3. Can complete a simple query (requires COMMANDCODE_API_KEY)
+ *
+ * Run with: node tests/test-smoke.mjs
+ * Requires: pi on PATH, COMMANDCODE_API_KEY env var set (or test is skipped)
+ */
+
+import { spawn } from "node:child_process";
+import { resolve, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PROJECT_DIR = resolve(__dirname, "..");
+const EXT_PATH = resolve(PROJECT_DIR, "index.ts");
+
+const PRINT_MODE_TIMEOUT = 120_000; // 2 minutes for print mode
+const RPC_START_TIMEOUT = 15_000;
+const RPC_QUERY_TIMEOUT = 60_000;
+
+const HAS_API_KEY = !!process.env.COMMANDCODE_API_KEY;
+
+let passed = 0;
+let failed = 0;
+let skipped = 0;
+
+// -------------------------------------------------------------------------
+// Helpers
+// -------------------------------------------------------------------------
+
+function kill(child) {
+  try {
+    child.kill();
+  } catch {
+    // ignore
+  }
+}
+
+// -------------------------------------------------------------------------
+// Test 1: Print mode — extension loads and agent runs
+// -------------------------------------------------------------------------
+
+async function runPrintMode() {
+  if (!HAS_API_KEY) {
+    console.log("[smoke] SKIP — COMMANDCODE_API_KEY not set, skipping print mode test\n");
+    skipped++;
+    return;
+  }
+
+  console.log(`[smoke] Running pi in print mode with extension: ${EXT_PATH}`);
+  console.log(`[smoke]   pi -e ${EXT_PATH} -p "say hi" --provider commandcode --model claude-sonnet-4-6\n`);
+
+  const child = spawn("pi", [
+    "-e", EXT_PATH,
+    "-p", "say hi in one word",
+    "--provider", "commandcode",
+    "--model", "claude-sonnet-4-6",
+    "--thinking-level", "minimal",
+  ], {
+    env: { ...process.env },
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  let stdout = "";
+  let stderr = "";
+
+  child.stdout.on("data", (d) => { stdout += d.toString(); });
+  child.stderr.on("data", (d) => { stderr += d.toString(); });
+
+  const done = new Promise((resolve) => {
+    const timer = setTimeout(() => {
+      kill(child);
+      console.log("[smoke] TIMEOUT — pi print mode took too long");
+      resolve(false);
+    }, PRINT_MODE_TIMEOUT);
+
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      if (code === 0) {
+        console.log("[smoke] PASS — extension loaded and agent ran without crash");
+        console.log(`[smoke] stdout (last 300 chars): ${stdout.slice(-300).trim()}`);
+      } else {
+        console.log(`[smoke] FAIL — exit code ${code}`);
+        console.log(`[smoke] stderr (last 500 chars): ${stderr.slice(-500).trim()}`);
+      }
+      resolve(code === 0);
+    });
+  });
+
+  const ok = await done;
+  if (ok) passed++;
+  else failed++;
+}
+
+// -------------------------------------------------------------------------
+// Test 2: Print mode — provider discovery (list models)
+// -------------------------------------------------------------------------
+
+async function runListModels() {
+  if (!HAS_API_KEY) {
+    console.log("[smoke] SKIP — no API key, skipping model list test\n");
+    skipped++;
+    return;
+  }
+
+  console.log(`[smoke] Checking that models are discoverable via pi --list-models\n`);
+
+  const child = spawn("pi", [
+    "-e", EXT_PATH,
+    "--list-models",
+  ], {
+    env: { ...process.env },
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  let stdout = "";
+
+  child.stdout.on("data", (d) => { stdout += d.toString(); });
+
+  const done = new Promise((resolve) => {
+    const timer = setTimeout(() => {
+      kill(child);
+      console.log("[smoke] TIMEOUT — model listing took too long");
+      resolve(false);
+    }, 15_000);
+
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      if (code === 0 && stdout.includes("commandcode")) {
+        console.log("[smoke] PASS — commandcode provider models are listed");
+      } else {
+        console.log("[smoke] FAIL — commandcode models not found or error listing");
+        console.log(`[smoke] stdout (last 500 chars): ${stdout.slice(-500).trim()}`);
+      }
+      resolve(code === 0 && stdout.includes("commandcode"));
+    });
+  });
+
+  const ok = await done;
+  if (ok) passed++;
+  else failed++;
+}
+
+// -------------------------------------------------------------------------
+// Test 3: RPC mode — extension loads, session starts
+// -------------------------------------------------------------------------
+
+async function runRpcStartup() {
+  if (!HAS_API_KEY) {
+    console.log("[smoke] SKIP — no API key, skipping RPC startup test\n");
+    skipped++;
+    return;
+  }
+
+  console.log(`[smoke] Testing RPC mode startup with extension\n`);
+  console.log(`[smoke]   pi --mode rpc -e ${EXT_PATH}\n`);
+
+  const child = spawn("pi", [
+    "--mode", "rpc",
+    "-e", EXT_PATH,
+  ], {
+    env: { ...process.env },
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+
+  let sawSessionStart = false;
+  let sawError = false;
+  const events = [];
+
+  let buf = "";
+  child.stdout.on("data", (chunk) => {
+    buf += chunk.toString("utf-8");
+    const lines = buf.split("\n");
+    buf = lines.pop() ?? "";
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      try {
+        const msg = JSON.parse(trimmed);
+        events.push(msg);
+        if (msg.type === "session_start") {
+          sawSessionStart = true;
+          console.log("[smoke] RPC received session_start event");
+        }
+        if (msg.type === "error" || msg.type === "fatal") {
+          sawError = true;
+          console.error(`[smoke] RPC error: ${JSON.stringify(msg).slice(0, 300)}`);
+        }
+      } catch {
+        // ignore non-JSON
+      }
+    }
+  });
+
+  const result = new Promise((resolve) => {
+    const timer = setTimeout(async () => {
+      if (sawSessionStart) {
+        console.log("[smoke] PASS — extension loaded, session started in RPC mode");
+        resolve(true);
+      } else {
+        console.log("[smoke] FAIL — session_start not received within 5s");
+        resolve(false);
+      }
+      // Send quit
+      try { child.stdin.write(JSON.stringify({ type: "quit" }) + "\n"); } catch {}
+      kill(child);
+    }, RPC_START_TIMEOUT);
+
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      if (!sawSessionStart && !sawError) {
+        console.log(`[smoke] FAIL — pi exited with code ${code} before session_start`);
+        resolve(false);
+      }
+    });
+  });
+
+  const ok = await result;
+  if (ok) passed++;
+  else failed++;
+}
+
+// -------------------------------------------------------------------------
+// Test 4: RPC mode — send query and receive assistant message
+// -------------------------------------------------------------------------
+
+async function runRpcQuery() {
+  if (!HAS_API_KEY) {
+    console.log("[smoke] SKIP — no API key, skipping RPC query test\n");
+    skipped++;
+    return;
+  }
+
+  console.log(`[smoke] Testing RPC query flow\n`);
+  console.log(`[smoke]   pi --mode rpc -e ${EXT_PATH} → query "say hi" → expect response\n`);
+
+  const child = spawn("pi", [
+    "--mode", "rpc",
+    "-e", EXT_PATH,
+    "--provider", "commandcode",
+    "--model", "claude-sonnet-4-6",
+    "--thinking-level", "minimal",
+  ], {
+    env: { ...process.env },
+    stdio: ["pipe", "pipe", "pipe"],
+  });
+
+  let sawSessionStart = false;
+  let sawAssistantMessage = false;
+  let querySent = false;
+
+  let buf = "";
+  child.stdout.on("data", (chunk) => {
+    buf += chunk.toString("utf-8");
+    const lines = buf.split("\n");
+    buf = lines.pop() ?? "";
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      try {
+        const msg = JSON.parse(trimmed);
+        if (msg.type === "session_start") {
+          sawSessionStart = true;
+          // Now send a query
+          if (!querySent) {
+            querySent = true;
+            const query = {
+              type: "query",
+              query: "say hi in one word",
+              sessionId: msg.sessionId,
+            };
+            child.stdin.write(JSON.stringify(query) + "\n");
+            console.log("[smoke] Sent RPC query");
+          }
+        }
+        if (msg.type === "assistant_message") {
+          sawAssistantMessage = true;
+          console.log("[smoke] PASS — received assistant_message in RPC mode");
+        }
+      } catch {
+        // ignore
+      }
+    }
+  });
+
+  const result = new Promise((resolve) => {
+    const timer = setTimeout(() => {
+      if (sawAssistantMessage) {
+        console.log("[smoke] PASS — full RPC query/response cycle works");
+        resolve(true);
+      } else {
+        console.log("[smoke] WARN — no assistant_message received (may still be streaming)");
+        resolve(false);
+      }
+      try { child.stdin.write(JSON.stringify({ type: "quit" }) + "\n"); } catch {}
+      kill(child);
+    }, RPC_QUERY_TIMEOUT);
+
+    child.on("close", (code) => {
+      clearTimeout(timer);
+      if (!sawAssistantMessage) {
+        console.log(`[smoke] FAIL — pi exited before assistant_message`);
+        resolve(false);
+      }
+    });
+  });
+
+  const ok = await result;
+  if (ok) passed++;
+  else failed++;
+}
+
+// -------------------------------------------------------------------------
+// Main
+// -------------------------------------------------------------------------
+
+console.log("=".repeat(60));
+console.log("  pi-commandcode-provider Integration Smoke Test");
+console.log("=".repeat(60));
+console.log(`  API key: ${HAS_API_KEY ? "✓ found" : "✗ not set (tests will be skipped)"}`);
+console.log(`  Extension: ${EXT_PATH}`);
+console.log("=".repeat(60));
+console.log("");
+
+await runPrintMode();
+await runListModels();
+await runRpcStartup();
+await runRpcQuery();
+
+console.log("");
+console.log("=".repeat(60));
+console.log(`  SUITE RESULT: ${passed} passed, ${failed} failed, ${skipped} skipped`);
+console.log("=".repeat(60));
+
+process.exit(failed > 0 ? 1 : 0);