test(e2e): cover live Go and GOAT accounts

2026-08-25 16:37:24 +02:00
parent 804ba5862e
commit cee0d7cc96
6 changed files with 157 additions and 26 deletions
@@ -18,7 +18,7 @@
 - Let `/login` use browser authentication, an explicit API-key prompt, or a directly pasted API key.
 - Add optional zero-data-retention headers through `CMD_ZDR=1` and the legacy `COMMANDCODE_ZDR=1` alias.
 - Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended, and display the current DeepSeek V4 off-peak rates for its time-dependent pricing.
- Add isolated live E2E profiles for separate Go-plan and Provider-API credentials, including an explicit selected-transport assertion and packed-package validation.
+- Add isolated live E2E profiles for separate Go-, GOAT-, and Provider-plan credentials, covering transport selection, reasoning across turns, quota identity, aborts, tools, GOAT vision, Go image rejection, and packed-package validation.
 - Fix extension load failure on newer pi hosts that reject registering a custom API under a built-in name (`openai-completions`); register under `commandcode-custom` instead and restore the real wire API before native compat dispatch.

 ## 0.5.1 - 2026-08-11
@@ -46,10 +46,11 @@ Run the transport-specific live tests with separate credentials:

 ```sh
 COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key npm run test:e2e:live:go
+COMMANDCODE_E2E_GOAT_API_KEY_FILE=/path/to/goat-key npm run test:e2e:live:goat
 COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key npm run test:e2e:live:provider
 ```

-Use `npm run test:e2e:live:all` with both file variables to run them sequentially. Store the keys in a secret manager and export each one to a new mode-`0600` temporary file for the test; never add key files to the repository. The direct `COMMANDCODE_E2E_GO_API_KEY` and `COMMANDCODE_E2E_PROVIDER_API_KEY` variables are intended primarily for protected CI secrets.
+Use `npm run test:e2e:live:all` with the Go and GOAT file variables to run both subscription transports sequentially. Store keys in a secret manager and export each one to a new mode-`0600` temporary file for the test; never add key files to the repository. Direct `*_API_KEY` variables are intended primarily for protected CI secrets.

 Before opening a PR, run:

@@ -195,23 +195,26 @@ Both commands accept additional pi arguments after `--`, for example `npm run pi

 ### Live transport tests

-Keep the Go-plan and Provider-API test keys in separate secret-manager entries. Pass them through protected files so the keys do not enter shell history:
+Keep Go-, GOAT-, and optional Provider-plan test keys in separate secret-manager entries. Pass them through protected files so the keys do not enter shell history:

 ```sh
 COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \
  npm run test:e2e:live:go

+COMMANDCODE_E2E_GOAT_API_KEY_FILE=/path/to/goat-key \
+  npm run test:e2e:live:goat
+
 COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key \
  npm run test:e2e:live:provider

 COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \
-COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key \
+COMMANDCODE_E2E_GOAT_API_KEY_FILE=/path/to/goat-key \
  npm run test:e2e:live:all
 ```

-Each profile runs with an isolated Pi agent directory and asserts the selected transport through `/commandcode-status`: Go must select `generate`, while a Provider API account must select `provider`. The profile-specific `*_API_KEY` environment variables are also supported for CI secrets, but key files are preferred for local use.
+Each profile runs with an isolated Pi agent directory and asserts transport selection, reasoning across turns, quota plan identity, abort handling, tool calls, and the packed npm artifact. Go must select `generate` and reject unsupported images; GOAT must select `provider` and complete a live vision request. The profile-specific `*_API_KEY` environment variables are also supported for CI secrets, but key files are preferred for local use.

-Override the default DeepSeek test model with `COMMANDCODE_E2E_GO_MODEL` or `COMMANDCODE_E2E_PROVIDER_MODEL`. A successful live Anthropic `/provider/v1/messages` test requires a Provider API account whose plan includes the selected Claude model.
+The Go profile defaults to DeepSeek V4 Flash; GOAT defaults to Grok 4.6 because its Provider API stream exposes reasoning consistently across consecutive turns. Override them with `COMMANDCODE_E2E_GO_MODEL`, `COMMANDCODE_E2E_GOAT_MODEL`, or `COMMANDCODE_E2E_PROVIDER_MODEL`. A successful live Anthropic `/provider/v1/messages` test requires a paid account whose plan includes the selected Claude model.

 See [CONTRIBUTING.md](CONTRIBUTING.md) for local setup and tests. See [RELEASE.md](RELEASE.md) for the release process.

@@ -55,8 +55,9 @@
    "test:smoke": "node tests/test-smoke.mjs",
    "test:e2e:live": "node tests/test-live-e2e.mjs",
    "test:e2e:live:go": "node scripts/live-e2e-profile.mjs go",
+    "test:e2e:live:goat": "node scripts/live-e2e-profile.mjs goat",
    "test:e2e:live:provider": "node scripts/live-e2e-profile.mjs provider",
-    "test:e2e:live:all": "node scripts/live-e2e-profile.mjs go provider",
+    "test:e2e:live:all": "node scripts/live-e2e-profile.mjs go goat",
    "test:cost": "tsx tests/test-cost.ts"
  },
  "pi": {
@@ -11,14 +11,19 @@ const profiles = process.argv.slice(2)

 if (
  profiles.length === 0 ||
-  profiles.some((profile) => profile !== "go" && profile !== "provider")
+  profiles.some((profile) => profile !== "go" && profile !== "goat" && profile !== "provider")
 ) {
-  console.error("Usage: node scripts/live-e2e-profile.mjs <go|provider> [go|provider]")
+  console.error("Usage: node scripts/live-e2e-profile.mjs <go|goat|provider> [go|goat|provider]")
  process.exit(2)
 }

 async function credentialFor(profile) {
-  const prefix = profile === "go" ? "COMMANDCODE_E2E_GO" : "COMMANDCODE_E2E_PROVIDER"
+  const prefix =
+    profile === "go"
+      ? "COMMANDCODE_E2E_GO"
+      : profile === "goat"
+        ? "COMMANDCODE_E2E_GOAT"
+        : "COMMANDCODE_E2E_PROVIDER"
  const direct = process.env[`${prefix}_API_KEY`]?.trim()
  const file = process.env[`${prefix}_API_KEY_FILE`]

@@ -35,8 +40,14 @@ async function credentialFor(profile) {

 function runProfile(profile, apiKey) {
  const modelVariable =
-    profile === "go" ? "COMMANDCODE_E2E_GO_MODEL" : "COMMANDCODE_E2E_PROVIDER_MODEL"
-  const model = process.env[modelVariable] ?? "deepseek/deepseek-v4-flash"
+    profile === "go"
+      ? "COMMANDCODE_E2E_GO_MODEL"
+      : profile === "goat"
+        ? "COMMANDCODE_E2E_GOAT_MODEL"
+        : "COMMANDCODE_E2E_PROVIDER_MODEL"
+  const model =
+    process.env[modelVariable] ??
+    (profile === "goat" ? "xai/grok-4.6" : "deepseek/deepseek-v4-flash")
  const env = {
    ...process.env,
    COMMAND_CODE_API_KEY: apiKey,
@@ -45,6 +56,7 @@ function runProfile(profile, apiKey) {
  }
  delete env.COMMANDCODE_API_KEY
  delete env.COMMANDCODE_E2E_GO_API_KEY
+  delete env.COMMANDCODE_E2E_GOAT_API_KEY
  delete env.COMMANDCODE_E2E_PROVIDER_API_KEY

  return new Promise((resolveRun, reject) => {
@@ -27,7 +27,20 @@ const extensionPath = join(projectDir, "index.ts")
 const testModel = process.env.COMMANDCODE_E2E_MODEL ?? "deepseek/deepseek-v4-flash"
 const testProfile = process.env.COMMANDCODE_E2E_PROFILE
 const expectedTransport =
-  testProfile === "go" ? "generate" : testProfile === "provider" ? "provider" : undefined
+  testProfile === "go"
+    ? "generate"
+    : testProfile === "goat" || testProfile === "provider"
+      ? "provider"
+      : undefined
+const expectedPlan =
+  testProfile === "go"
+    ? "go"
+    : testProfile === "goat"
+      ? "goat"
+      : testProfile === "provider"
+        ? "provider"
+        : undefined
+const goatVisionModel = process.env.COMMANDCODE_E2E_GOAT_VISION_MODEL ?? "google/gemini-3.7-flash"
 const marker = "commandcode-live-e2e-ok"

 function findPiBinary() {
@@ -104,7 +117,7 @@ function run(command, args, options = {}) {
  })
 }

-async function runRpc(extension, action, timeoutMs = 120_000) {
+async function runRpc(extension, action, timeoutMs = 120_000, model = testModel) {
  const child = spawn(
    piBin,
    [
@@ -116,7 +129,9 @@ async function runRpc(extension, action, timeoutMs = 120_000) {
      "--provider",
      "commandcode",
      "--model",
-      testModel,
+      model,
+      "--thinking",
+      "high",
    ],
    { cwd: projectDir, env: safeEnv(), stdio: ["pipe", "pipe", "pipe"] },
  )
@@ -224,8 +239,19 @@ try {
    await waitFor(
      (event) => event.type === "response" && event.id === "reasoning-turn-1" && event.success,
    )
-    await waitFor((event) => event.type === "agent_settled")
-    const firstThinkingDeltas = countThinkingDeltas(firstStart)
+    const firstSettled = await waitFor(
+      (event) => event.type === "agent_settled" && events.indexOf(event) >= firstStart,
+    )
+    const firstSettledIndex = events.indexOf(firstSettled)
+    const firstThinkingDeltas = events
+      .slice(firstStart, firstSettledIndex + 1)
+      .filter(
+        (event) =>
+          event.type === "message_update" &&
+          event.assistantMessageEvent?.type === "thinking_delta" &&
+          typeof event.assistantMessageEvent.delta === "string" &&
+          event.assistantMessageEvent.delta.length > 0,
+      ).length

    const secondStart = events.length
    send({
@@ -237,15 +263,16 @@ try {
    await waitFor(
      (event) => event.type === "response" && event.id === "reasoning-turn-2" && event.success,
    )
-    await waitFor((event) => event.type === "agent_settled" && events.indexOf(event) >= secondStart)
+    const secondSettled = await waitFor(
+      (event) => event.type === "agent_settled" && events.indexOf(event) >= secondStart,
+    )
    const secondThinkingDeltas = countThinkingDeltas(secondStart)
+    assert.ok(events.indexOf(secondSettled) >= secondStart)

    return { firstThinkingDeltas, secondThinkingDeltas, stderr: getStderr() }
  })
-  if (testProfile !== "provider") {
  assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning")
  assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning")
-  }
  assert.doesNotMatch(multiTurn.stderr, /Bearer\s+\S+/i)

  console.log("[live-e2e] live runtime refresh/status commands")
@@ -283,15 +310,66 @@ try {
        typeof event.message === "string" &&
        event.message.includes("source:"),
    )
-    return { names, refresh: refresh.message, status: status.message, stderr: getStderr() }
+
+    send({ id: "quota", type: "prompt", message: "/commandcode-quota" })
+    await waitFor((event) => event.type === "response" && event.id === "quota" && event.success)
+    const quota = await waitFor(
+      (event) =>
+        event.type === "extension_ui_request" &&
+        event.method === "notify" &&
+        typeof event.message === "string" &&
+        event.message.includes("Plan:"),
+    )
+    return {
+      names,
+      refresh: refresh.message,
+      status: status.message,
+      quota: quota.message,
+      stderr: getStderr(),
+    }
  })
  assert.ok(runtime.names.includes("commandcode-refresh"))
  assert.ok(runtime.names.includes("commandcode-status"))
+  assert.ok(runtime.names.includes("commandcode-quota"))
  assert.match(runtime.refresh, /model catalog (?:refreshed|unchanged)/)
  if (expectedTransport) assert.match(runtime.status, new RegExp(`transport: ${expectedTransport}`))
  assert.match(runtime.status, /source: (?:live|cache)/)
  assert.match(runtime.status, /model count: [1-9][0-9]*/)
-  assert.doesNotMatch(`${runtime.refresh}\n${runtime.status}\n${runtime.stderr}`, /Bearer\s+\S+/i)
+  if (expectedPlan) assert.match(runtime.quota, new RegExp(`Plan:.*\\b${expectedPlan}\\b`, "i"))
+  assert.doesNotMatch(
+    `${runtime.refresh}\n${runtime.status}\n${runtime.quota}\n${runtime.stderr}`,
+    /Bearer\s+\S+/i,
+  )
+
+  console.log("[live-e2e] live abort through real RPC host")
+  const abortResult = await runRpc(extensionPath, async ({ send, waitFor, events, getStderr }) => {
+    const startIndex = events.length
+    send({
+      id: "abort-turn",
+      type: "prompt",
+      message: "Write a very long detailed explanation of every integer from 1 to 10000.",
+    })
+    await waitFor(
+      (event) => event.type === "response" && event.id === "abort-turn" && event.success,
+    )
+    await waitFor((event) => event.type === "message_update" && events.indexOf(event) >= startIndex)
+    send({ id: "abort", type: "abort" })
+    await waitFor((event) => event.type === "response" && event.id === "abort" && event.success)
+    await waitFor((event) => event.type === "agent_settled" && events.indexOf(event) >= startIndex)
+    return {
+      aborted: events
+        .slice(startIndex)
+        .some(
+          (event) =>
+            event.type === "message_end" &&
+            event.message?.role === "assistant" &&
+            event.message?.stopReason === "aborted",
+        ),
+      stderr: getStderr(),
+    }
+  })
+  assert.equal(abortResult.aborted, true)
+  assert.doesNotMatch(abortResult.stderr, /Bearer\s+\S+/i)

  console.log("[live-e2e] live tool-call round trip")
  const toolRoot = join(tempRoot, "tool-roundtrip")
@@ -319,9 +397,45 @@ try {
  )
  assert.equal(toolResult.code, 0, toolResult.stderr)
  assert.match(toolResult.stdout, new RegExp(marker))
-  assert.equal(readFileSync(targetPath, "utf-8"), marker)
+  assert.equal(readFileSync(targetPath, "utf-8").trimEnd(), marker)

-  if (testProfile !== "provider") {
+  if (testProfile === "goat") {
+    console.log("[live-e2e] live vision request through Provider API")
+    const vision = await runRpc(
+      extensionPath,
+      async ({ send, waitFor, events, getStderr }) => {
+        const startIndex = events.length
+        send({
+          id: "vision",
+          type: "prompt",
+          message: "Describe the attached image briefly.",
+          images: [
+            {
+              type: "image",
+              data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
+              mimeType: "image/png",
+            },
+          ],
+        })
+        await waitFor(
+          (event) => event.type === "response" && event.id === "vision" && event.success,
+        )
+        await waitFor(
+          (event) => event.type === "agent_settled" && events.indexOf(event) >= startIndex,
+        )
+        const messageEnd = events
+          .slice(startIndex)
+          .find((event) => event.type === "message_end" && event.message?.role === "assistant")
+        return { messageEnd, stderr: getStderr() }
+      },
+      180_000,
+      goatVisionModel,
+    )
+    assert.notEqual(vision.messageEnd?.message?.stopReason, "error")
+    assert.doesNotMatch(vision.stderr, /Bearer\s+\S+/i)
+  }
+
+  if (testProfile === "go") {
    console.log("[live-e2e] image rejection through real RPC host")
    const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => {
      send({