test(e2e): merge live account profiles

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Let `/login` use browser authentication, an explicit API-key prompt, or a directly pasted API key.
 - Add optional zero-data-retention headers through `COMMANDCODE_ZDR=1`.
 - Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended.
+- Add isolated live E2E profiles for separate Go-plan and Provider-API credentials, including an explicit selected-transport assertion and packed-package validation.
 
 ## 0.5.1 - 2026-08-11
 

CONTRIBUTING.md

@@ -42,6 +42,15 @@ npm run pi:authenticated
 
 Both commands accept additional pi arguments after `--`, for example `npm run pi:authenticated -- --model claude-sonnet-4-6`.
 
+Run the transport-specific live tests with separate credentials:
+
+```sh
+COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key npm run test:e2e:live:go
+COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key npm run test:e2e:live:provider
+```
+
+Use `npm run test:e2e:live:all` with both file variables to run them sequentially. Store the keys in a secret manager and export each one to a new mode-`0600` temporary file for the test; never add key files to the repository. The direct `COMMANDCODE_E2E_GO_API_KEY` and `COMMANDCODE_E2E_PROVIDER_API_KEY` variables are intended primarily for protected CI secrets.
+
 Before opening a PR, run:
 
 ```sh

README.md

@@ -190,6 +190,26 @@ npm run pi:authenticated
 
 Both commands accept additional pi arguments after `--`, for example `npm run pi:authenticated -- --model claude-sonnet-4-6`.
 
+### Live transport tests
+
+Keep the Go-plan and Provider-API test keys in separate secret-manager entries. Pass them through protected files so the keys do not enter shell history:
+
+```sh
+COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \
+  npm run test:e2e:live:go
+
+COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key \
+  npm run test:e2e:live:provider
+
+COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \
+COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key \
+  npm run test:e2e:live:all
+```
+
+Each profile runs with an isolated Pi agent directory and asserts the selected transport through `/commandcode-status`: Go must select `generate`, while a Provider API account must select `provider`. The profile-specific `*_API_KEY` environment variables are also supported for CI secrets, but key files are preferred for local use.
+
+Override the default DeepSeek test model with `COMMANDCODE_E2E_GO_MODEL` or `COMMANDCODE_E2E_PROVIDER_MODEL`. A successful live Anthropic `/provider/v1/messages` test requires a Provider API account whose plan includes the selected Claude model.
+
 See [CONTRIBUTING.md](CONTRIBUTING.md) for local setup and tests. See [RELEASE.md](RELEASE.md) for the release process.
 
 ## License

index.ts

@@ -127,6 +127,7 @@ export default async function (pi: ExtensionAPI) {
         timeoutMs: modelsTimeoutMs,
       }),
     createProviderConfig: (models) => createProviderConfig(models, apiBase, transport.stream),
+    getTransport: transport.getTransport,
   })
 
   await runtime.initialize()

package.json

@@ -51,6 +51,9 @@
     "test:pi-local": "node tests/test-pi-local.mjs",
     "test:smoke": "node tests/test-smoke.mjs",
     "test:e2e:live": "node tests/test-live-e2e.mjs",
+    "test:e2e:live:go": "node scripts/live-e2e-profile.mjs go",
+    "test:e2e:live:provider": "node scripts/live-e2e-profile.mjs provider",
+    "test:e2e:live:all": "node scripts/live-e2e-profile.mjs go provider",
     "test:cost": "tsx tests/test-cost.ts"
   },
   "pi": {

scripts/live-e2e-profile.mjs

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process"
+import { readFile } from "node:fs/promises"
+import { dirname, resolve } from "node:path"
+import { fileURLToPath } from "node:url"
+
+const projectDir = resolve(dirname(fileURLToPath(import.meta.url)), "..")
+const liveTest = resolve(projectDir, "tests", "test-live-e2e.mjs")
+const profiles = process.argv.slice(2)
+
+if (
+  profiles.length === 0 ||
+  profiles.some((profile) => profile !== "go" && profile !== "provider")
+) {
+  console.error("Usage: node scripts/live-e2e-profile.mjs <go|provider> [go|provider]")
+  process.exit(2)
+}
+
+async function credentialFor(profile) {
+  const prefix = profile === "go" ? "COMMANDCODE_E2E_GO" : "COMMANDCODE_E2E_PROVIDER"
+  const direct = process.env[`${prefix}_API_KEY`]?.trim()
+  const file = process.env[`${prefix}_API_KEY_FILE`]
+
+  if (direct && file)
+    throw new Error(`${prefix}_API_KEY and ${prefix}_API_KEY_FILE are mutually exclusive`)
+  if (direct) return direct
+  if (file) {
+    const credential = (await readFile(file, "utf-8")).trim()
+    if (credential) return credential
+  }
+
+  throw new Error(`Set ${prefix}_API_KEY_FILE (recommended) or ${prefix}_API_KEY`)
+}
+
+function runProfile(profile, apiKey) {
+  const modelVariable =
+    profile === "go" ? "COMMANDCODE_E2E_GO_MODEL" : "COMMANDCODE_E2E_PROVIDER_MODEL"
+  const model = process.env[modelVariable] ?? "deepseek/deepseek-v4-flash"
+  const env = {
+    ...process.env,
+    COMMANDCODE_API_KEY: apiKey,
+    COMMANDCODE_E2E_MODEL: model,
+    COMMANDCODE_E2E_PROFILE: profile,
+  }
+  delete env.COMMANDCODE_E2E_GO_API_KEY
+  delete env.COMMANDCODE_E2E_PROVIDER_API_KEY
+
+  return new Promise((resolveRun, reject) => {
+    console.log(`[live-e2e:${profile}] model ${model}`)
+    const child = spawn(process.execPath, [liveTest], {
+      cwd: projectDir,
+      env,
+      stdio: "inherit",
+    })
+    child.on("error", reject)
+    child.on("close", (code, signal) => {
+      if (code === 0) {
+        resolveRun()
+        return
+      }
+      reject(new Error(`[live-e2e:${profile}] failed (${signal ?? `exit ${code}`})`))
+    })
+  })
+}
+
+try {
+  for (const profile of profiles) {
+    await runProfile(profile, await credentialFor(profile))
+  }
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error))
+  process.exit(1)
+}

src/runtime.ts

@@ -28,11 +28,13 @@ export interface CommandCodeRuntimeOptions<TProviderConfig> {
   cachePath: string
   loadModels: () => Promise<LoadCommandCodeModelsResult>
   createProviderConfig: (models: readonly CommandCodeModel[]) => TProviderConfig
+  getTransport?: () => "unknown" | "provider" | "generate"
   now?: () => number
   logWarning?: (message: string) => void
 }
 
 export interface CommandCodeRuntimeStatus {
+  transport: "unknown" | "provider" | "generate"
   source: LoadCommandCodeModelsResult["source"]
   modelCount: number
   lastSuccess?: number
@@ -86,6 +88,7 @@ function formatTimestamp(timestamp: number | undefined): string {
 
 export function formatCommandCodeStatus(status: CommandCodeRuntimeStatus): string {
   const lines = [
+    `transport: ${status.transport}`,
     `source: ${status.source}`,
     `model count: ${status.modelCount}`,
     `last success: ${formatTimestamp(status.lastSuccess)}`,
@@ -113,6 +116,7 @@ export class CommandCodeRuntime<TProviderConfig, TContext extends CommandCodeCom
     this.now = options.now ?? Date.now
     this.logWarning = options.logWarning ?? ((message) => console.warn(`[commandcode] ${message}`))
     const initialStatus: CommandCodeRuntimeStatus = {
+      transport: "unknown",
       source: "empty",
       modelCount: 0,
       cachePath: options.cachePath,
@@ -123,7 +127,10 @@ export class CommandCodeRuntime<TProviderConfig, TContext extends CommandCodeCom
   }
 
   getStatus(): CommandCodeRuntimeStatus {
-    return { ...this.status }
+    return {
+      ...this.status,
+      transport: this.options.getTransport?.() ?? "unknown",
+    }
   }
 
   async initialize(): Promise<void> {
@@ -259,10 +266,8 @@ export class CommandCodeRuntime<TProviderConfig, TContext extends CommandCodeCom
     this.pi.registerCommand("commandcode-status", {
       description: "Show redacted Command Code provider diagnostics",
       handler: async (_args, ctx) => {
-        ctx.ui.notify(
+        const status = this.getStatus()
-          formatCommandCodeStatus(this.status),
+        ctx.ui.notify(formatCommandCodeStatus(status), status.warning ? "warning" : "info")
-          this.status.warning ? "warning" : "info",
-        )
       },
     })
   }

tests/test-live-e2e.mjs

@@ -25,6 +25,9 @@ import { fileURLToPath } from "node:url"
 const projectDir = resolve(dirname(fileURLToPath(import.meta.url)), "..")
 const extensionPath = join(projectDir, "index.ts")
 const testModel = process.env.COMMANDCODE_E2E_MODEL ?? "deepseek/deepseek-v4-flash"
+const testProfile = process.env.COMMANDCODE_E2E_PROFILE
+const expectedTransport =
+  testProfile === "go" ? "generate" : testProfile === "provider" ? "provider" : undefined
 const marker = "commandcode-live-e2e-ok"
 
 function findPiBinary() {
@@ -57,9 +60,18 @@ if (!piBin || !hasAuthMetadata()) {
   process.exit(0)
 }
 
+const profileAgentDir = testProfile
+  ? mkdtempSync(join(tmpdir(), `pi-commandcode-live-${testProfile}-agent-`))
+  : undefined
+
 function safeEnv(overrides = {}) {
   const env = { ...process.env, PI_SKIP_VERSION_CHECK: "1", ...overrides }
-  delete env.COMMANDCODE_API_KEY
+  if (testProfile && profileAgentDir) {
+    env.PI_CODING_AGENT_DIR = profileAgentDir
+    env.COMMANDCODE_MODELS_CACHE = join(profileAgentDir, "commandcode-models.json")
+  } else {
+    delete env.COMMANDCODE_API_KEY
+  }
   return env
 }
 
@@ -228,12 +240,22 @@ try {
 
     return { firstThinkingDeltas, secondThinkingDeltas, stderr: getStderr() }
   })
-  assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning")
+  if (testProfile !== "provider") {
-  assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning")
+    assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning")
+    assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning")
+  }
   assert.doesNotMatch(multiTurn.stderr, /Bearer\s+\S+/i)
 
   console.log("[live-e2e] live runtime refresh/status commands")
   const runtime = await runRpc(extensionPath, async ({ send, waitFor, getStderr }) => {
+    if (expectedTransport) {
+      send({ id: "transport-probe", type: "prompt", message: `Reply exactly: ${marker}` })
+      await waitFor(
+        (event) => event.type === "response" && event.id === "transport-probe" && event.success,
+      )
+      await waitFor((event) => event.type === "agent_settled")
+    }
+
     send({ id: "commands", type: "get_commands" })
     const commands = await waitFor(
       (event) => event.type === "response" && event.id === "commands" && event.success,
@@ -264,6 +286,7 @@ try {
   assert.ok(runtime.names.includes("commandcode-refresh"))
   assert.ok(runtime.names.includes("commandcode-status"))
   assert.match(runtime.refresh, /model catalog (?:refreshed|unchanged)/)
+  if (expectedTransport) assert.match(runtime.status, new RegExp(`transport: ${expectedTransport}`))
   assert.match(runtime.status, /source: (?:live|cache)/)
   assert.match(runtime.status, /model count: [1-9][0-9]*/)
   assert.doesNotMatch(`${runtime.refresh}\n${runtime.status}\n${runtime.stderr}`, /Bearer\s+\S+/i)
@@ -296,30 +319,32 @@ try {
   assert.match(toolResult.stdout, new RegExp(marker))
   assert.equal(readFileSync(targetPath, "utf-8"), marker)
 
-  console.log("[live-e2e] image rejection through real RPC host")
+  if (testProfile !== "provider") {
-  const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => {
+    console.log("[live-e2e] image rejection through real RPC host")
-    send({
+    const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => {
-      id: "image",
+      send({
-      type: "prompt",
+        id: "image",
-      message: "Describe this image",
+        type: "prompt",
-      images: [{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }],
+        message: "Describe this image",
+        images: [{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }],
+      })
+      await waitFor((event) => event.type === "response" && event.id === "image")
+      await waitFor(
+        (event) =>
+          event.type === "message_end" &&
+          event.message?.role === "assistant" &&
+          event.message?.stopReason === "error",
+      )
+      return events
     })
-    await waitFor((event) => event.type === "response" && event.id === "image")
+    assert.ok(
-    await waitFor(
+      image.some(
-      (event) =>
+        (event) =>
-        event.type === "message_end" &&
+          event.type === "message_end" &&
-        event.message?.role === "assistant" &&
+          /does not support image content/i.test(event.message?.errorMessage ?? ""),
-        event.message?.stopReason === "error",
+      ),
     )
-    return events
+  }
-  })
-  assert.ok(
-    image.some(
-      (event) =>
-        event.type === "message_end" &&
-        /does not support image content/i.test(event.message?.errorMessage ?? ""),
-    ),
-  )
 
   console.log("[live-e2e] packed artifact with existing authentication")
   const packDir = join(tempRoot, "pack")
@@ -361,4 +386,5 @@ try {
   console.log("[live-e2e] PASS")
 } finally {
   rmSync(tempRoot, { recursive: true, force: true })
+  if (profileAgentDir) rmSync(profileAgentDir, { recursive: true, force: true })
 }

tests/test-runtime.ts

@@ -98,6 +98,7 @@ describe("Command Code runtime", () => {
       cachePath: "/tmp/commandcode-models.json",
       loadModels: () => firstLoad.promise,
       createProviderConfig: (models) => ({ models }),
+      getTransport: () => "provider",
       now: () => now,
       logWarning: () => {},
     })
@@ -115,6 +116,7 @@ describe("Command Code runtime", () => {
     assert.ok(statusCommand)
     await statusCommand("", context)
     const statusMessage = context.notifications.at(-1)?.message ?? ""
+    assert.match(statusMessage, /transport: provider/)
     assert.match(statusMessage, /source: live/)
     assert.match(statusMessage, /model count: 1/)
     assert.match(statusMessage, /last success:/)