From 75552ef4b90e08be96de011bd5ae3623b28a5a7d Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 18 Aug 2026 17:20:44 +0200 Subject: [PATCH 1/2] test(e2e): add live account profiles --- index.ts | 1 + package.json | 3 ++ scripts/live-e2e-profile.mjs | 74 +++++++++++++++++++++++++++++++++++ src/runtime.ts | 15 ++++--- tests/test-live-e2e.mjs | 76 ++++++++++++++++++++++++------------ tests/test-runtime.ts | 2 + 6 files changed, 141 insertions(+), 30 deletions(-) create mode 100644 scripts/live-e2e-profile.mjs diff --git a/index.ts b/index.ts index d263138..1f4aeb2 100644 --- a/index.ts +++ b/index.ts @@ -127,6 +127,7 @@ export default async function (pi: ExtensionAPI) { timeoutMs: modelsTimeoutMs, }), createProviderConfig: (models) => createProviderConfig(models, apiBase, transport.stream), + getTransport: transport.getTransport, }) await runtime.initialize() diff --git a/package.json b/package.json index c99a0aa..f284d3b 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,9 @@ "test:pi-local": "node tests/test-pi-local.mjs", "test:smoke": "node tests/test-smoke.mjs", "test:e2e:live": "node tests/test-live-e2e.mjs", + "test:e2e:live:go": "node scripts/live-e2e-profile.mjs go", + "test:e2e:live:provider": "node scripts/live-e2e-profile.mjs provider", + "test:e2e:live:all": "node scripts/live-e2e-profile.mjs go provider", "test:cost": "tsx tests/test-cost.ts" }, "pi": { diff --git a/scripts/live-e2e-profile.mjs b/scripts/live-e2e-profile.mjs new file mode 100644 index 0000000..01e9bab --- /dev/null +++ b/scripts/live-e2e-profile.mjs @@ -0,0 +1,74 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process" +import { readFile } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +const projectDir = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const liveTest = resolve(projectDir, "tests", "test-live-e2e.mjs") +const profiles = process.argv.slice(2) + +if ( + profiles.length === 0 || + profiles.some((profile) => profile !== "go" && profile !== "provider") +) { + console.error("Usage: node scripts/live-e2e-profile.mjs [go|provider]") + process.exit(2) +} + +async function credentialFor(profile) { + const prefix = profile === "go" ? "COMMANDCODE_E2E_GO" : "COMMANDCODE_E2E_PROVIDER" + const direct = process.env[`${prefix}_API_KEY`]?.trim() + const file = process.env[`${prefix}_API_KEY_FILE`] + + if (direct && file) + throw new Error(`${prefix}_API_KEY and ${prefix}_API_KEY_FILE are mutually exclusive`) + if (direct) return direct + if (file) { + const credential = (await readFile(file, "utf-8")).trim() + if (credential) return credential + } + + throw new Error(`Set ${prefix}_API_KEY_FILE (recommended) or ${prefix}_API_KEY`) +} + +function runProfile(profile, apiKey) { + const modelVariable = + profile === "go" ? "COMMANDCODE_E2E_GO_MODEL" : "COMMANDCODE_E2E_PROVIDER_MODEL" + const model = process.env[modelVariable] ?? "deepseek/deepseek-v4-flash" + const env = { + ...process.env, + COMMANDCODE_API_KEY: apiKey, + COMMANDCODE_E2E_MODEL: model, + COMMANDCODE_E2E_PROFILE: profile, + } + delete env.COMMANDCODE_E2E_GO_API_KEY + delete env.COMMANDCODE_E2E_PROVIDER_API_KEY + + return new Promise((resolveRun, reject) => { + console.log(`[live-e2e:${profile}] model ${model}`) + const child = spawn(process.execPath, [liveTest], { + cwd: projectDir, + env, + stdio: "inherit", + }) + child.on("error", reject) + child.on("close", (code, signal) => { + if (code === 0) { + resolveRun() + return + } + reject(new Error(`[live-e2e:${profile}] failed (${signal ?? `exit ${code}`})`)) + }) + }) +} + +try { + for (const profile of profiles) { + await runProfile(profile, await credentialFor(profile)) + } +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exit(1) +} diff --git a/src/runtime.ts b/src/runtime.ts index 34c034a..e103f96 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -28,11 +28,13 @@ export interface CommandCodeRuntimeOptions { cachePath: string loadModels: () => Promise createProviderConfig: (models: readonly CommandCodeModel[]) => TProviderConfig + getTransport?: () => "unknown" | "provider" | "generate" now?: () => number logWarning?: (message: string) => void } export interface CommandCodeRuntimeStatus { + transport: "unknown" | "provider" | "generate" source: LoadCommandCodeModelsResult["source"] modelCount: number lastSuccess?: number @@ -86,6 +88,7 @@ function formatTimestamp(timestamp: number | undefined): string { export function formatCommandCodeStatus(status: CommandCodeRuntimeStatus): string { const lines = [ + `transport: ${status.transport}`, `source: ${status.source}`, `model count: ${status.modelCount}`, `last success: ${formatTimestamp(status.lastSuccess)}`, @@ -113,6 +116,7 @@ export class CommandCodeRuntime console.warn(`[commandcode] ${message}`)) const initialStatus: CommandCodeRuntimeStatus = { + transport: "unknown", source: "empty", modelCount: 0, cachePath: options.cachePath, @@ -123,7 +127,10 @@ export class CommandCodeRuntime { @@ -259,10 +266,8 @@ export class CommandCodeRuntime { - ctx.ui.notify( - formatCommandCodeStatus(this.status), - this.status.warning ? "warning" : "info", - ) + const status = this.getStatus() + ctx.ui.notify(formatCommandCodeStatus(status), status.warning ? "warning" : "info") }, }) } diff --git a/tests/test-live-e2e.mjs b/tests/test-live-e2e.mjs index 160882a..88ca636 100644 --- a/tests/test-live-e2e.mjs +++ b/tests/test-live-e2e.mjs @@ -25,6 +25,9 @@ import { fileURLToPath } from "node:url" const projectDir = resolve(dirname(fileURLToPath(import.meta.url)), "..") const extensionPath = join(projectDir, "index.ts") const testModel = process.env.COMMANDCODE_E2E_MODEL ?? "deepseek/deepseek-v4-flash" +const testProfile = process.env.COMMANDCODE_E2E_PROFILE +const expectedTransport = + testProfile === "go" ? "generate" : testProfile === "provider" ? "provider" : undefined const marker = "commandcode-live-e2e-ok" function findPiBinary() { @@ -57,9 +60,18 @@ if (!piBin || !hasAuthMetadata()) { process.exit(0) } +const profileAgentDir = testProfile + ? mkdtempSync(join(tmpdir(), `pi-commandcode-live-${testProfile}-agent-`)) + : undefined + function safeEnv(overrides = {}) { const env = { ...process.env, PI_SKIP_VERSION_CHECK: "1", ...overrides } - delete env.COMMANDCODE_API_KEY + if (testProfile && profileAgentDir) { + env.PI_CODING_AGENT_DIR = profileAgentDir + env.COMMANDCODE_MODELS_CACHE = join(profileAgentDir, "commandcode-models.json") + } else { + delete env.COMMANDCODE_API_KEY + } return env } @@ -228,12 +240,22 @@ try { return { firstThinkingDeltas, secondThinkingDeltas, stderr: getStderr() } }) - assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning") - assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning") + if (testProfile !== "provider") { + assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning") + assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning") + } assert.doesNotMatch(multiTurn.stderr, /Bearer\s+\S+/i) console.log("[live-e2e] live runtime refresh/status commands") const runtime = await runRpc(extensionPath, async ({ send, waitFor, getStderr }) => { + if (expectedTransport) { + send({ id: "transport-probe", type: "prompt", message: `Reply exactly: ${marker}` }) + await waitFor( + (event) => event.type === "response" && event.id === "transport-probe" && event.success, + ) + await waitFor((event) => event.type === "agent_settled") + } + send({ id: "commands", type: "get_commands" }) const commands = await waitFor( (event) => event.type === "response" && event.id === "commands" && event.success, @@ -264,6 +286,7 @@ try { assert.ok(runtime.names.includes("commandcode-refresh")) assert.ok(runtime.names.includes("commandcode-status")) assert.match(runtime.refresh, /model catalog (?:refreshed|unchanged)/) + if (expectedTransport) assert.match(runtime.status, new RegExp(`transport: ${expectedTransport}`)) assert.match(runtime.status, /source: (?:live|cache)/) assert.match(runtime.status, /model count: [1-9][0-9]*/) assert.doesNotMatch(`${runtime.refresh}\n${runtime.status}\n${runtime.stderr}`, /Bearer\s+\S+/i) @@ -296,30 +319,32 @@ try { assert.match(toolResult.stdout, new RegExp(marker)) assert.equal(readFileSync(targetPath, "utf-8"), marker) - console.log("[live-e2e] image rejection through real RPC host") - const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => { - send({ - id: "image", - type: "prompt", - message: "Describe this image", - images: [{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }], + if (testProfile !== "provider") { + console.log("[live-e2e] image rejection through real RPC host") + const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => { + send({ + id: "image", + type: "prompt", + message: "Describe this image", + images: [{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }], + }) + await waitFor((event) => event.type === "response" && event.id === "image") + await waitFor( + (event) => + event.type === "message_end" && + event.message?.role === "assistant" && + event.message?.stopReason === "error", + ) + return events }) - await waitFor((event) => event.type === "response" && event.id === "image") - await waitFor( - (event) => - event.type === "message_end" && - event.message?.role === "assistant" && - event.message?.stopReason === "error", + assert.ok( + image.some( + (event) => + event.type === "message_end" && + /does not support image content/i.test(event.message?.errorMessage ?? ""), + ), ) - return events - }) - assert.ok( - image.some( - (event) => - event.type === "message_end" && - /does not support image content/i.test(event.message?.errorMessage ?? ""), - ), - ) + } console.log("[live-e2e] packed artifact with existing authentication") const packDir = join(tempRoot, "pack") @@ -361,4 +386,5 @@ try { console.log("[live-e2e] PASS") } finally { rmSync(tempRoot, { recursive: true, force: true }) + if (profileAgentDir) rmSync(profileAgentDir, { recursive: true, force: true }) } diff --git a/tests/test-runtime.ts b/tests/test-runtime.ts index 397c54e..2a89f12 100644 --- a/tests/test-runtime.ts +++ b/tests/test-runtime.ts @@ -98,6 +98,7 @@ describe("Command Code runtime", () => { cachePath: "/tmp/commandcode-models.json", loadModels: () => firstLoad.promise, createProviderConfig: (models) => ({ models }), + getTransport: () => "provider", now: () => now, logWarning: () => {}, }) @@ -115,6 +116,7 @@ describe("Command Code runtime", () => { assert.ok(statusCommand) await statusCommand("", context) const statusMessage = context.notifications.at(-1)?.message ?? "" + assert.match(statusMessage, /transport: provider/) assert.match(statusMessage, /source: live/) assert.match(statusMessage, /model count: 1/) assert.match(statusMessage, /last success:/) From 33c405c06005f0c430f9ad260d53d90173c6853a Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 18 Aug 2026 17:20:51 +0200 Subject: [PATCH 2/2] docs(tests): document live account credentials --- CHANGELOG.md | 1 + CONTRIBUTING.md | 9 +++++++++ README.md | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b58dd8a..cf4942c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Let `/login` use browser authentication, an explicit API-key prompt, or a directly pasted API key. - Add optional zero-data-retention headers through `COMMANDCODE_ZDR=1`. - Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended. +- Add isolated live E2E profiles for separate Go-plan and Provider-API credentials, including an explicit selected-transport assertion and packed-package validation. ## 0.5.1 - 2026-08-11 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da4d0a4..6adb7bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,15 @@ npm run pi:authenticated Both commands accept additional pi arguments after `--`, for example `npm run pi:authenticated -- --model claude-sonnet-4-6`. +Run the transport-specific live tests with separate credentials: + +```sh +COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key npm run test:e2e:live:go +COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key npm run test:e2e:live:provider +``` + +Use `npm run test:e2e:live:all` with both file variables to run them sequentially. Store the keys in a secret manager and export each one to a new mode-`0600` temporary file for the test; never add key files to the repository. The direct `COMMANDCODE_E2E_GO_API_KEY` and `COMMANDCODE_E2E_PROVIDER_API_KEY` variables are intended primarily for protected CI secrets. + Before opening a PR, run: ```sh diff --git a/README.md b/README.md index e1e4a3a..6c37492 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,26 @@ npm run pi:authenticated Both commands accept additional pi arguments after `--`, for example `npm run pi:authenticated -- --model claude-sonnet-4-6`. +### Live transport tests + +Keep the Go-plan and Provider-API test keys in separate secret-manager entries. Pass them through protected files so the keys do not enter shell history: + +```sh +COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \ + npm run test:e2e:live:go + +COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key \ + npm run test:e2e:live:provider + +COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \ +COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key \ + npm run test:e2e:live:all +``` + +Each profile runs with an isolated Pi agent directory and asserts the selected transport through `/commandcode-status`: Go must select `generate`, while a Provider API account must select `provider`. The profile-specific `*_API_KEY` environment variables are also supported for CI secrets, but key files are preferred for local use. + +Override the default DeepSeek test model with `COMMANDCODE_E2E_GO_MODEL` or `COMMANDCODE_E2E_PROVIDER_MODEL`. A successful live Anthropic `/provider/v1/messages` test requires a Provider API account whose plan includes the selected Claude model. + See [CONTRIBUTING.md](CONTRIBUTING.md) for local setup and tests. See [RELEASE.md](RELEASE.md) for the release process. ## License