#!/usr/bin/env node /** * Local end-to-end test: loads the real extension through the pi CLI while the * Command Code API is replaced by a deterministic local mock server. */ import assert from "node:assert/strict" import { spawn, spawnSync } from "node:child_process" import { accessSync, constants, mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs" import { createServer } from "node:http" import { tmpdir } from "node:os" import { delimiter, dirname, join, resolve } from "node:path" import { fileURLToPath } from "node:url" const __dirname = dirname(fileURLToPath(import.meta.url)) const PROJECT_DIR = resolve(__dirname, "..") const EXT_PATH = resolve(PROJECT_DIR, "index.ts") const TEST_MODEL = "gpt-5.4" const CLAUDE_TEST_MODEL = "claude-sonnet-4-6" function findPiBinary() { if (process.env.PI_BIN) return process.env.PI_BIN const localBin = resolve(PROJECT_DIR, "node_modules", ".bin") const candidates = (process.env.PATH ?? "") .split(delimiter) .map((entry) => resolve(entry, "pi")) .filter((candidate) => !candidate.startsWith(localBin)) for (const candidate of candidates) { try { accessSync(candidate, constants.X_OK) return candidate } catch { // Try next PATH entry. } } return undefined } const PI_BIN = findPiBinary() if (!PI_BIN) { console.log("[pi-local] SKIP — pi is not on PATH") process.exit(0) } const piCheck = spawnSync(PI_BIN, ["--help"], { stdio: "ignore" }) if (piCheck.error) { console.log(`[pi-local] SKIP — pi failed to start: ${piCheck.error.message}`) process.exit(0) } let requestCount = 0 let modelListRequestCount = 0 let lastRequestBody let lastRequestHeaders = {} let overflowMode = false let overflowRequestCount = 0 let modelsDelayMs = 0 let includeRefreshedModel = false function modelCatalog() { const data = [ { id: TEST_MODEL, object: "model", created: 1779824324, owned_by: "command-code", name: "GPT 5.4", context_length: 1_000_000, }, { id: CLAUDE_TEST_MODEL, object: "model", created: 1779824324, owned_by: "command-code", name: "Claude Sonnet 4.6", context_length: 200_000, }, { id: "cc-second-model", object: "model", created: 1779824324, owned_by: "command-code", name: "Qwen 3.7 Max", context_length: 1_000_000, }, ] if (includeRefreshedModel) { data.push({ id: "cc-refreshed-model", object: "model", created: 1779824324, owned_by: "command-code", name: "Refreshed Model", context_length: 200_000, }) } return { object: "list", data } } const server = createServer((req, res) => { if (req.method === "GET" && req.url === "/provider/v1/models") { modelListRequestCount += 1 const respond = () => { if (res.destroyed) return res.writeHead(200, { "Content-Type": "application/json; charset=utf-8" }) res.end(JSON.stringify(modelCatalog())) } if (modelsDelayMs > 0) setTimeout(respond, modelsDelayMs) else respond() return } const isOpenAIRequest = req.method === "POST" && req.url === "/provider/v1/chat/completions" const isAnthropicRequest = req.method === "POST" && req.url === "/provider/v1/messages" if (!isOpenAIRequest && !isAnthropicRequest) { res.writeHead(404) res.end("Not found") return } requestCount += 1 if (overflowMode) overflowRequestCount += 1 lastRequestHeaders = Object.fromEntries( Object.entries(req.headers).map(([key, value]) => [ key, Array.isArray(value) ? value.join(", ") : (value ?? ""), ]), ) let body = "" req.on("data", (chunk) => { body += chunk.toString("utf-8") }) req.on("end", () => { try { lastRequestBody = JSON.parse(body) } catch { lastRequestBody = undefined } if (overflowMode && overflowRequestCount === 2) { res.writeHead(400, { "Content-Type": "application/json; charset=utf-8" }) res.end( JSON.stringify({ error: { message: "Input exceeds context limit", type: "invalid_request_error", code: "context_length_exceeded", }, }), ) return } res.writeHead(200, { "Content-Type": "text/event-stream; charset=utf-8", "Transfer-Encoding": "chunked", }) const text = overflowMode ? overflowRequestCount === 1 ? "overflow-initial" : overflowRequestCount === 3 ? "compaction-summary" : "overflow-recovered" : "mock-pi-ok" if (isAnthropicRequest) { res.write( `event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "mock", type: "message", role: "assistant", content: [], model: CLAUDE_TEST_MODEL, stop_reason: null, stop_sequence: null, usage: { input_tokens: 1, output_tokens: 0 } } })}\n\n`, ) res.write( `event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } })}\n\n`, ) res.write( `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text } })}\n\n`, ) res.write( `event: content_block_stop\ndata: ${JSON.stringify({ type: "content_block_stop", index: 0 })}\n\n`, ) res.write( `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 1 } })}\n\n`, ) res.end(`event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`) return } res.write( `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: text }, finish_reason: null }] })}\n\n`, ) res.write( `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n`, ) res.write( `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })}\n\n`, ) res.end("data: [DONE]\n\n") }) }) await new Promise((resolve) => server.listen(0, resolve)) const address = server.address() const port = typeof address === "object" && address ? address.port : 0 const apiBase = `http://127.0.0.1:${port}` const tempHome = mkdtempSync(join(tmpdir(), "pi-cc-home-")) const agentDir = join(tempHome, "custom-pi-agent") mkdirSync(agentDir, { recursive: true }) writeFileSync( join(agentDir, "settings.json"), JSON.stringify({ compaction: { enabled: true, reserveTokens: 10, keepRecentTokens: 10 } }), ) const env = { ...process.env, HOME: tempHome, USERPROFILE: tempHome, PI_CODING_AGENT_DIR: agentDir, PI_CODING_AGENT_SESSION_DIR: join(tempHome, "sessions"), COMMANDCODE_API_BASE: `${apiBase}/provider/v1`, COMMANDCODE_API_KEY: "mock-key", COMMANDCODE_ZDR: "1", COMMANDCODE_MODELS_URL: `${apiBase}/provider/v1/models`, } function runPi(args, timeoutMs = 30_000) { return new Promise((resolve) => { const child = spawn(PI_BIN, args, { cwd: PROJECT_DIR, env, stdio: ["ignore", "pipe", "pipe"], }) let stdout = "" let stderr = "" const timer = setTimeout(() => { child.kill() resolve({ code: -1, stdout, stderr: `${stderr}\nTIMEOUT after ${timeoutMs}ms`, }) }, timeoutMs) child.stdout.on("data", (chunk) => { stdout += chunk.toString("utf-8") }) child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf-8") }) child.on("close", (code) => { clearTimeout(timer) resolve({ code, stdout, stderr }) }) }) } async function runRpcQuery( timeoutMs = 30_000, promptMessage = "say mock token", extraArgs = [], promptFields = {}, ) { const child = spawn( PI_BIN, [ "--no-extensions", "--mode", "rpc", "-e", EXT_PATH, "--provider", "commandcode", "--model", TEST_MODEL, ...extraArgs, ], { cwd: PROJECT_DIR, env, stdio: ["pipe", "pipe", "pipe"], }, ) let stdout = "" let stderr = "" let buffer = "" let sawPromptAccepted = false let sawAssistantMessage = false let sawTextDelta = false const events = [] const done = new Promise((resolve) => { const timer = setTimeout(() => { child.kill() resolve(false) }, timeoutMs) const finish = (ok) => { clearTimeout(timer) try { child.stdin.write(`${JSON.stringify({ type: "quit" })}\n`) } catch { // ignore shutdown race } child.kill() resolve(ok) } child.stdin.write( `${JSON.stringify({ id: "prompt-1", type: "prompt", message: promptMessage, ...promptFields, })}\n`, ) child.stdout.on("data", (chunk) => { const text = chunk.toString("utf-8") stdout += text buffer += text const lines = buffer.split("\n") buffer = lines.pop() ?? "" for (const line of lines) { const trimmed = line.trim() if (!trimmed) continue try { const event = JSON.parse(trimmed) events.push(event) if (event.type === "response" && event.id === "prompt-1" && event.success === true) { sawPromptAccepted = true } if ( event.type === "message_update" && event.assistantMessageEvent?.type === "text_delta" ) { sawTextDelta = true } if (event.type === "message_end" && event.message?.role === "assistant") { sawAssistantMessage = true finish(true) } } catch { // ignore non-JSON output } } }) child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf-8") }) child.on("close", () => { if (!sawAssistantMessage) finish(false) }) }) const ok = await done return { ok, stdout, stderr, events, sawPromptAccepted, sawAssistantMessage, sawTextDelta, } } async function runRpcExtensionCommands(timeoutMs = 30_000) { const child = spawn( PI_BIN, [ "--no-extensions", "--mode", "rpc", "-e", EXT_PATH, "--provider", "commandcode", "--model", TEST_MODEL, ], { cwd: PROJECT_DIR, env, stdio: ["pipe", "pipe", "pipe"], }, ) let buffer = "" let stderr = "" const events = [] const waiters = [] const publish = (event) => { events.push(event) for (let index = waiters.length - 1; index >= 0; index -= 1) { const waiter = waiters[index] if (!waiter.predicate(event)) continue waiters.splice(index, 1) clearTimeout(waiter.timer) waiter.resolve(event) } } child.stdout.on("data", (chunk) => { buffer += chunk.toString("utf-8") const lines = buffer.split("\n") buffer = lines.pop() ?? "" for (const line of lines) { if (!line.trim()) continue try { publish(JSON.parse(line)) } catch { // Ignore non-JSON output. } } }) child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf-8") }) const waitFor = (predicate) => new Promise((resolve, reject) => { const existing = events.find(predicate) if (existing) { resolve(existing) return } const timer = setTimeout(() => { const index = waiters.findIndex((waiter) => waiter.timer === timer) if (index >= 0) waiters.splice(index, 1) reject(new Error(`RPC event timeout. stderr: ${stderr.slice(-500)}`)) }, timeoutMs) waiters.push({ predicate, resolve, timer }) }) const send = (value) => child.stdin.write(`${JSON.stringify(value)}\n`) try { send({ id: "commands", type: "get_commands" }) const commandsResponse = await waitFor( (event) => event.type === "response" && event.id === "commands", ) const commandNames = commandsResponse.data?.commands?.map((command) => command.name) ?? [] send({ id: "status-before", type: "prompt", message: "/commandcode-status" }) await waitFor( (event) => event.type === "response" && event.id === "status-before" && event.success, ) const statusBefore = await waitFor( (event) => event.type === "extension_ui_request" && event.method === "notify" && typeof event.message === "string" && event.message.includes("model count: 3"), ) includeRefreshedModel = true send({ id: "refresh", type: "prompt", message: "/commandcode-refresh" }) await waitFor((event) => event.type === "response" && event.id === "refresh" && event.success) const refreshNotification = await waitFor( (event) => event.type === "extension_ui_request" && event.method === "notify" && typeof event.message === "string" && event.message.includes("4 models from live"), ) send({ id: "status-after", type: "prompt", message: "/commandcode-status" }) await waitFor( (event) => event.type === "response" && event.id === "status-after" && event.success, ) const statusAfter = await waitFor( (event) => event.type === "extension_ui_request" && event.method === "notify" && typeof event.message === "string" && event.message.includes("model count: 4"), ) return { commandNames, statusBefore: statusBefore.message, refreshNotification: refreshNotification.message, statusAfter: statusAfter.message, stderr, } } finally { child.kill() } } async function runRpcOverflowRecovery(timeoutMs = 60_000) { const child = spawn( PI_BIN, [ "--no-extensions", "--mode", "rpc", "-e", EXT_PATH, "--provider", "commandcode", "--model", TEST_MODEL, ], { cwd: PROJECT_DIR, env, stdio: ["pipe", "pipe", "pipe"], }, ) let buffer = "" let stderr = "" const events = [] let firstSettled = false let recovered = false const result = new Promise((resolve) => { const timer = setTimeout(() => { child.kill() resolve({ ok: false }) }, timeoutMs) const finish = (ok) => { clearTimeout(timer) child.kill() resolve({ ok }) } child.stdout.on("data", (chunk) => { buffer += chunk.toString("utf-8") const lines = buffer.split("\n") buffer = lines.pop() ?? "" for (const line of lines) { if (!line.trim()) continue let event try { event = JSON.parse(line) } catch { continue } events.push(event) if (event.type === "agent_settled" && !firstSettled) { firstSettled = true child.stdin.write( `${JSON.stringify({ id: "overflow-prompt", type: "prompt", message: "trigger overflow recovery" })}\n`, ) } if ( event.type === "compaction_end" && event.reason === "overflow" && event.willRetry === true ) { recovered = true } if (recovered && event.type === "agent_settled") finish(true) } }) child.stderr.on("data", (chunk) => { stderr += chunk.toString("utf-8") }) child.stdin.write( `${JSON.stringify({ id: "initial-prompt", type: "prompt", message: "initial turn" })}\n`, ) }) const outcome = await result return { ...outcome, requests: overflowRequestCount, sawNormalizedOverflow: events.some( (event) => event.type === "message_end" && event.message?.role === "assistant" && typeof event.message.errorMessage === "string" && event.message.errorMessage.startsWith("context_length_exceeded:"), ), sawCompactionRetry: events.some( (event) => event.type === "compaction_end" && event.reason === "overflow" && event.willRetry, ), stderrHasSecrets: /mock-key|user_secret|api_key/i.test(stderr), } } try { console.log("[pi-local] first offline start without a cache") const onlineModelsUrl = env.COMMANDCODE_MODELS_URL env.COMMANDCODE_MODELS_URL = "http://127.0.0.1:1/provider/v1/models" const modelsCachePath = join(env.PI_CODING_AGENT_DIR, "commandcode-models.json") rmSync(modelsCachePath, { force: true }) const firstOfflineList = await runPi( ["--no-extensions", "-e", EXT_PATH, "--list-models", "commandcode"], 20_000, ) assert.equal(firstOfflineList.code, 0, firstOfflineList.stderr) assert.doesNotMatch(firstOfflineList.stderr, /Failed to load extension/) assert.match( firstOfflineList.stdout || firstOfflineList.stderr, /No models matching|No models available/, ) assert.match(firstOfflineList.stderr, /no valid cached catalog/) assert.match(firstOfflineList.stderr, /until \/commandcode-refresh succeeds/) assert.throws(() => accessSync(modelsCachePath, constants.R_OK), /ENOENT|no such file/i) // A fresh process re-runs the extension entrypoint, which is the same path /reload uses. console.log("[pi-local] recover models after empty offline start") env.COMMANDCODE_MODELS_URL = onlineModelsUrl modelListRequestCount = 0 const recoveryList = await runPi( ["--no-extensions", "-e", EXT_PATH, "--list-models", "commandcode"], 20_000, ) assert.equal(recoveryList.code, 0, recoveryList.stderr) const recoveryOutput = recoveryList.stdout || recoveryList.stderr assert.match(recoveryOutput, /gpt-5\.4/) assert.match(recoveryOutput, /cc-second-model/) assert.doesNotMatch(recoveryList.stderr, /no valid cached catalog/) assert.doesNotMatch(recoveryList.stderr, /Failed to load extension/) assert.equal(modelListRequestCount, 1) assert.doesNotThrow(() => accessSync(modelsCachePath, constants.R_OK)) console.log("[pi-local] list models through real extension") modelListRequestCount = 0 const list = await runPi(["--no-extensions", "-e", EXT_PATH, "--list-models"], 20_000) assert.equal(list.code, 0, list.stderr) const listOutput = list.stdout || list.stderr assert.match(listOutput, /commandcode/) assert.match(listOutput, /gpt-5\.4/) assert.match(listOutput, /cc-second-model/) assert.equal(modelListRequestCount, 1) assert.doesNotThrow(() => accessSync(modelsCachePath, constants.R_OK)) console.log("[pi-local] list cached models while model discovery is offline") env.COMMANDCODE_MODELS_URL = "http://127.0.0.1:1/provider/v1/models" const offlineList = await runPi( ["--no-extensions", "-e", EXT_PATH, "--list-models", "commandcode"], 20_000, ) assert.equal(offlineList.code, 0, offlineList.stderr) const offlineListOutput = offlineList.stdout || offlineList.stderr assert.match(offlineListOutput, /gpt-5\.4/) assert.match(offlineListOutput, /cc-second-model/) assert.match(offlineList.stderr, /Using the cached catalog/) console.log("[pi-local] use a cached model while model discovery is offline") requestCount = 0 const offlinePrint = await runPi( [ "--no-extensions", "-e", EXT_PATH, "-p", "say mock token", "--provider", "commandcode", "--model", TEST_MODEL, ], 30_000, ) assert.equal(offlinePrint.code, 0, offlinePrint.stderr) assert.match(offlinePrint.stdout, /mock-pi-ok/) assert.match(offlinePrint.stderr, /Using the cached catalog/) assert.equal(requestCount, 1) env.COMMANDCODE_MODELS_URL = onlineModelsUrl console.log("[pi-local] discovery timeout through real extension") rmSync(modelsCachePath, { force: true }) modelsDelayMs = 5_000 env.COMMANDCODE_MODELS_TIMEOUT_MS = "50" const timeoutStartedAt = Date.now() const timedOutList = await runPi( ["--no-extensions", "-e", EXT_PATH, "--list-models", "commandcode"], 5_000, ) const timeoutElapsedMs = Date.now() - timeoutStartedAt assert.equal(timedOutList.code, 0, timedOutList.stderr) assert.ok(timeoutElapsedMs < 2_000, `model discovery took ${timeoutElapsedMs}ms`) assert.match(timedOutList.stderr, /timed out after 50ms/i) modelsDelayMs = 0 delete env.COMMANDCODE_MODELS_TIMEOUT_MS console.log("[pi-local] print mode with reasoning and tool schemas") requestCount = 0 const print = await runPi( [ "--no-extensions", "-e", EXT_PATH, "-p", "say mock token", "--provider", "commandcode", "--model", TEST_MODEL, "--thinking", "high", ], 30_000, ) assert.equal(print.code, 0, print.stderr) assert.match(print.stdout, /mock-pi-ok/) assert.equal(requestCount, 1) assert.ok( typeof lastRequestHeaders.authorization === "string" && lastRequestHeaders.authorization.startsWith("Bearer "), "should send a bearer Authorization header", ) assert.equal(lastRequestHeaders["x-cmd-zdr"], "1") assert.equal(lastRequestBody?.model, TEST_MODEL) assert.equal(lastRequestBody?.reasoning_effort, "high") const sentTools = lastRequestBody?.tools assert.ok(Array.isArray(sentTools) && sentTools.length > 0) const editTool = sentTools.find((tool) => tool.function?.name === "edit") assert.equal(editTool?.function?.parameters?.type, "object") assert.equal(editTool?.function?.parameters?.properties?.edits?.type, "array") assert.equal(editTool?.function?.parameters?.properties?.edits?.items?.type, "object") assert.equal( editTool?.function?.parameters?.properties?.edits?.items?.properties?.oldText?.type, "string", ) console.log("[pi-local] Claude request through Anthropic Messages endpoint") requestCount = 0 const claudePrint = await runPi( [ "--no-extensions", "-e", EXT_PATH, "-p", "say mock token", "--provider", "commandcode", "--model", CLAUDE_TEST_MODEL, "--thinking", "high", ], 30_000, ) assert.equal(claudePrint.code, 0, claudePrint.stderr) assert.match(claudePrint.stdout, /mock-pi-ok/) assert.equal(requestCount, 1) assert.equal(lastRequestBody?.model, CLAUDE_TEST_MODEL) assert.equal(lastRequestBody?.thinking?.type, "adaptive") assert.deepEqual(lastRequestBody?.output_config, { effort: "high" }) assert.equal(lastRequestHeaders["x-api-key"], "mock-key") assert.equal(lastRequestHeaders["x-cmd-zdr"], "1") console.log("[pi-local] runtime commands through real RPC extension lifecycle") includeRefreshedModel = false const runtimeCommands = await runRpcExtensionCommands() assert.ok(runtimeCommands.commandNames.includes("commandcode-refresh")) assert.ok(runtimeCommands.commandNames.includes("commandcode-status")) assert.match(runtimeCommands.statusBefore, /source: live/) assert.match(runtimeCommands.refreshNotification, /4 models from live/) assert.match(runtimeCommands.statusAfter, /model count: 4/) assert.doesNotMatch( `${runtimeCommands.statusBefore}\n${runtimeCommands.statusAfter}\n${runtimeCommands.stderr}`, /mock-key/, ) console.log("[pi-local] RPC prompt through real extension and mock API") requestCount = 0 const rpc = await runRpcQuery() assert.equal( rpc.ok, true, JSON.stringify( { stderr: rpc.stderr, stdout: rpc.stdout, events: rpc.events.slice(-10) }, null, 2, ), ) assert.equal(rpc.sawPromptAccepted, true) assert.equal(rpc.sawAssistantMessage, true) assert.equal(rpc.sawTextDelta, true) assert.equal(requestCount, 1) console.log("[pi-local] forward image input through the documented provider schema") requestCount = 0 const imageRpc = await runRpcQuery(10_000, "describe image", [], { images: [ { type: "image", data: "iVBORw0KGgo=", mimeType: "image/png", }, ], }) assert.equal(imageRpc.ok, true, imageRpc.stderr) assert.equal(requestCount, 1) const imageContent = lastRequestBody?.messages?.find( (message) => message.role === "user", )?.content assert.ok(Array.isArray(imageContent), JSON.stringify(lastRequestBody?.messages)) assert.ok( imageContent.some((part) => part.type === "image_url"), JSON.stringify(imageContent), ) console.log("[pi-local] verify overflow normalization and compaction recovery") overflowMode = true overflowRequestCount = 0 const overflowRpc = await runRpcOverflowRecovery() assert.equal(overflowRpc.ok, true) assert.ok(overflowRpc.requests >= 4) assert.equal(overflowRpc.sawCompactionRetry, true, JSON.stringify(overflowRpc)) assert.equal(overflowRpc.stderrHasSecrets, false) overflowMode = false console.log("[pi-local] PASS") } finally { await new Promise((resolve) => server.close(resolve)) rmSync(tempHome, { recursive: true, force: true }) }