pi-commandcode-provider/tests/test-smoke.mjs

/**
 * Integration smoke test for pi-commandcode-provider.
 *
 * Tests that the extension:
 * 1. Loads without crashing in print mode
 * 2. Registers the provider and models
 * 3. Can complete a simple prompt (requires Command Code auth)
 *
 * Run with: node tests/test-smoke.mjs
 * Requires: pi on PATH plus COMMANDCODE_API_KEY or live pi auth files.
 */

import { spawn } from "node:child_process";
import { accessSync, constants, existsSync } from "node:fs";
import { homedir } from "node:os";
import { delimiter, resolve, dirname, join } from "node:path";
import { fileURLToPath } from "node:url";

const __dirname = dirname(fileURLToPath(import.meta.url));
const PROJECT_DIR = resolve(__dirname, "..");
const EXT_PATH = resolve(PROJECT_DIR, "index.ts");
const TEST_MODEL = "deepseek/deepseek-v4-flash";

function findPiBinary() {
  if (process.env.PI_BIN) return process.env.PI_BIN;
  const localBin = resolve(PROJECT_DIR, "node_modules", ".bin");
  const candidates = (process.env.PATH ?? "")
    .split(delimiter)
    .map((entry) => resolve(entry, "pi"))
    .filter((candidate) => !candidate.startsWith(localBin));
  for (const candidate of candidates) {
    try {
      accessSync(candidate, constants.X_OK);
      return candidate;
    } catch {
      // Try next PATH entry.
    }
  }
  return undefined;
}

const PI_BIN = findPiBinary();
const HAS_PI = !!PI_BIN;

const PRINT_MODE_TIMEOUT = 120_000; // 2 minutes for print mode
const RPC_START_TIMEOUT = 15_000;
const RPC_QUERY_TIMEOUT = 60_000;

function hasCommandCodeAuth() {
  return (
    !!process.env.COMMANDCODE_API_KEY ||
    existsSync(join(homedir(), ".commandcode", "auth.json")) ||
    existsSync(join(homedir(), ".pi", "agent", "auth.json"))
  );
}

const HAS_AUTH = hasCommandCodeAuth();

let passed = 0;
let failed = 0;
let skipped = 0;

// -------------------------------------------------------------------------
// Helpers
// -------------------------------------------------------------------------

function kill(child) {
  try {
    child.kill();
  } catch {
    // ignore
  }
}

// -------------------------------------------------------------------------
// Test 1: Print mode — extension loads and agent runs
// -------------------------------------------------------------------------

async function runPrintMode() {
  if (!HAS_AUTH) {
    console.log(
      "[smoke] SKIP — Command Code auth not found, skipping print mode test\n",
    );
    skipped++;
    return;
  }
  if (!HAS_PI) {
    console.log("[smoke] SKIP — pi is not on PATH, skipping print mode test\n");
    skipped++;
    return;
  }

  console.log(`[smoke] Running pi in print mode with extension: ${EXT_PATH}`);
  console.log(
    `[smoke]   ${PI_BIN} -e ${EXT_PATH} -p "say hi" --provider commandcode --model ${TEST_MODEL}\n`,
  );

  const child = spawn(
    PI_BIN,
    [
      "-e",
      EXT_PATH,
      "-p",
      "say hi in one word",
      "--provider",
      "commandcode",
      "--model",
      TEST_MODEL,
    ],
    {
      env: { ...process.env },
      stdio: ["ignore", "pipe", "pipe"],
    },
  );

  let stdout = "";
  let stderr = "";

  child.stdout.on("data", (d) => {
    stdout += d.toString();
  });
  child.stderr.on("data", (d) => {
    stderr += d.toString();
  });

  const done = new Promise((resolve) => {
    const timer = setTimeout(() => {
      kill(child);
      console.log("[smoke] TIMEOUT — pi print mode took too long");
      resolve(false);
    }, PRINT_MODE_TIMEOUT);

    child.on("close", (code) => {
      clearTimeout(timer);
      if (code === 0) {
        console.log(
          "[smoke] PASS — extension loaded and agent ran without crash",
        );
        console.log(
          `[smoke] stdout (last 300 chars): ${stdout.slice(-300).trim()}`,
        );
      } else {
        console.log(`[smoke] FAIL — exit code ${code}`);
        console.log(
          `[smoke] stderr (last 500 chars): ${stderr.slice(-500).trim()}`,
        );
      }
      resolve(code === 0);
    });
  });

  const ok = await done;
  if (ok) passed++;
  else failed++;
}

// -------------------------------------------------------------------------
// Test 2: Print mode — provider discovery (list models)
// -------------------------------------------------------------------------

async function runListModels() {
  if (!HAS_AUTH) {
    console.log(
      "[smoke] SKIP — Command Code auth not found, skipping model list test\n",
    );
    skipped++;
    return;
  }
  if (!HAS_PI) {
    console.log("[smoke] SKIP — pi is not on PATH, skipping model list test\n");
    skipped++;
    return;
  }

  console.log(
    `[smoke] Checking that models are discoverable via pi --list-models\n`,
  );

  const child = spawn(PI_BIN, ["-e", EXT_PATH, "--list-models"], {
    env: { ...process.env },
    stdio: ["ignore", "pipe", "pipe"],
  });

  let stdout = "";

  child.stdout.on("data", (d) => {
    stdout += d.toString();
  });

  const done = new Promise((resolve) => {
    const timer = setTimeout(() => {
      kill(child);
      console.log("[smoke] TIMEOUT — model listing took too long");
      resolve(false);
    }, 15_000);

    child.on("close", (code) => {
      clearTimeout(timer);
      if (code === 0 && stdout.includes("commandcode")) {
        console.log("[smoke] PASS — commandcode provider models are listed");
      } else {
        console.log(
          "[smoke] FAIL — commandcode models not found or error listing",
        );
        console.log(
          `[smoke] stdout (last 500 chars): ${stdout.slice(-500).trim()}`,
        );
      }
      resolve(code === 0 && stdout.includes("commandcode"));
    });
  });

  const ok = await done;
  if (ok) passed++;
  else failed++;
}

// -------------------------------------------------------------------------
// Test 3: RPC mode — extension loads and answers get_state
// -------------------------------------------------------------------------

async function runRpcStartup() {
  if (!HAS_AUTH) {
    console.log(
      "[smoke] SKIP — Command Code auth not found, skipping RPC startup test\n",
    );
    skipped++;
    return;
  }
  if (!HAS_PI) {
    console.log(
      "[smoke] SKIP — pi is not on PATH, skipping RPC startup test\n",
    );
    skipped++;
    return;
  }

  console.log(`[smoke] Testing RPC mode startup with extension\n`);
  console.log(`[smoke]   ${PI_BIN} --mode rpc -e ${EXT_PATH}\n`);

  const child = spawn(PI_BIN, ["--mode", "rpc", "-e", EXT_PATH], {
    env: { ...process.env },
    stdio: ["pipe", "pipe", "pipe"],
  });

  let sawStateResponse = false;
  let sawError = false;
  const events = [];

  let buf = "";
  child.stdout.on("data", (chunk) => {
    buf += chunk.toString("utf-8");
    const lines = buf.split("\n");
    buf = lines.pop() ?? "";
    for (const line of lines) {
      const trimmed = line.trim();
      if (!trimmed) continue;
      try {
        const msg = JSON.parse(trimmed);
        events.push(msg);
        if (
          msg.type === "response" &&
          msg.id === "state-1" &&
          msg.command === "get_state" &&
          msg.success === true
        ) {
          sawStateResponse = true;
          console.log("[smoke] RPC received get_state response");
        }
        if (msg.type === "error" || msg.type === "fatal") {
          sawError = true;
          console.error(
            `[smoke] RPC error: ${JSON.stringify(msg).slice(0, 300)}`,
          );
        }
      } catch {
        // ignore non-JSON
      }
    }
  });

  const result = new Promise((resolve) => {
    child.stdin.write(
      JSON.stringify({ id: "state-1", type: "get_state" }) + "\n",
    );

    const timer = setTimeout(async () => {
      if (sawStateResponse) {
        console.log("[smoke] PASS — extension loaded and RPC get_state works");
        resolve(true);
      } else {
        console.log("[smoke] FAIL — get_state response not received");
        resolve(false);
      }
      // Send quit
      try {
        child.stdin.write(JSON.stringify({ type: "quit" }) + "\n");
      } catch {}
      kill(child);
    }, RPC_START_TIMEOUT);

    child.on("close", (code) => {
      clearTimeout(timer);
      if (!sawStateResponse && !sawError) {
        console.log(
          `[smoke] FAIL — pi exited with code ${code} before get_state response`,
        );
        resolve(false);
      }
    });
  });

  const ok = await result;
  if (ok) passed++;
  else failed++;
}

// -------------------------------------------------------------------------
// Test 4: RPC mode — send prompt and receive assistant message
// -------------------------------------------------------------------------

async function runRpcQuery() {
  if (!HAS_AUTH) {
    console.log(
      "[smoke] SKIP — Command Code auth not found, skipping RPC prompt test\n",
    );
    skipped++;
    return;
  }
  if (!HAS_PI) {
    console.log("[smoke] SKIP — pi is not on PATH, skipping RPC prompt test\n");
    skipped++;
    return;
  }

  console.log(`[smoke] Testing RPC prompt flow\n`);
  console.log(
    `[smoke]   pi --mode rpc -e ${EXT_PATH} → prompt "say hi" → expect response\n`,
  );

  const child = spawn(
    PI_BIN,
    [
      "--mode",
      "rpc",
      "-e",
      EXT_PATH,
      "--provider",
      "commandcode",
      "--model",
      TEST_MODEL,
    ],
    {
      env: { ...process.env },
      stdio: ["pipe", "pipe", "pipe"],
    },
  );

  let sawPromptAccepted = false;
  let sawAssistantMessage = false;

  let buf = "";
  child.stdout.on("data", (chunk) => {
    buf += chunk.toString("utf-8");
    const lines = buf.split("\n");
    buf = lines.pop() ?? "";
    for (const line of lines) {
      const trimmed = line.trim();
      if (!trimmed) continue;
      try {
        const msg = JSON.parse(trimmed);
        if (
          msg.type === "response" &&
          msg.id === "prompt-1" &&
          msg.command === "prompt" &&
          msg.success === true
        ) {
          sawPromptAccepted = true;
        }
        if (msg.type === "message_end" && msg.message?.role === "assistant") {
          sawAssistantMessage = true;
          console.log(
            "[smoke] PASS — received assistant message_end in RPC mode",
          );
        }
      } catch {
        // ignore
      }
    }
  });

  const result = new Promise((resolve) => {
    child.stdin.write(
      JSON.stringify({
        id: "prompt-1",
        type: "prompt",
        message: "say hi in one word",
      }) + "\n",
    );
    console.log("[smoke] Sent RPC prompt");

    const timer = setTimeout(() => {
      if (sawPromptAccepted && sawAssistantMessage) {
        console.log("[smoke] PASS — full RPC prompt/response cycle works");
        resolve(true);
      } else {
        console.log(
          "[smoke] WARN — no assistant message_end received (may still be streaming)",
        );
        resolve(false);
      }
      try {
        child.stdin.write(JSON.stringify({ type: "quit" }) + "\n");
      } catch {}
      kill(child);
    }, RPC_QUERY_TIMEOUT);

    child.on("close", (code) => {
      clearTimeout(timer);
      if (!sawAssistantMessage) {
        console.log(`[smoke] FAIL — pi exited before assistant message_end`);
        resolve(false);
      }
    });
  });

  const ok = await result;
  if (ok) passed++;
  else failed++;
}

// -------------------------------------------------------------------------
// Main
// -------------------------------------------------------------------------

console.log("=".repeat(60));
console.log("  pi-commandcode-provider Integration Smoke Test");
console.log("=".repeat(60));
console.log(
  `  Auth: ${HAS_AUTH ? "✓ found" : "✗ not found (tests will be skipped)"}`,
);
console.log(`  Extension: ${EXT_PATH}`);
console.log("=".repeat(60));
console.log("");

await runPrintMode();
await runListModels();
await runRpcStartup();
await runRpcQuery();

console.log("");
console.log("=".repeat(60));
console.log(
  `  SUITE RESULT: ${passed} passed, ${failed} failed, ${skipped} skipped`,
);
console.log("=".repeat(60));

process.exit(failed > 0 ? 1 : 0);