From 06032913963f1e28921fd8805c305757b60a7b37 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 18 Aug 2026 10:55:58 +0200 Subject: [PATCH 01/40] feat(auth): support direct api key login --- src/api-key.ts | 68 +++++++++++++++++++++++++++++++++++++++++++ src/oauth.ts | 67 +++++++++++++++++++++++++++++++++--------- tests/test-api-key.ts | 59 +++++++++++++++++++++++++++++++++++++ tests/test-oauth.ts | 43 +++++++++++++++++++++++---- 4 files changed, 217 insertions(+), 20 deletions(-) create mode 100644 src/api-key.ts create mode 100644 tests/test-api-key.ts diff --git a/src/api-key.ts b/src/api-key.ts new file mode 100644 index 0000000..0ab52f5 --- /dev/null +++ b/src/api-key.ts @@ -0,0 +1,68 @@ +import { existsSync, readFileSync } from "node:fs" +import { homedir } from "node:os" +import { join } from "node:path" + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined +} + +function defaultAuthPaths(home: string): string[] { + return [ + join(home, ".commandcode", "auth.json"), + join(home, ".pi", "agent", "auth.json"), + join(home, ".omp", "agent", "auth.json"), + ] +} + +function apiKeyFromCredential(value: unknown): string | undefined { + if (!isRecord(value)) return undefined + + if (stringValue(value.type) === "oauth") return stringValue(value.access) + if (stringValue(value.type) === "api") return stringValue(value.key) + return stringValue(value.access) ?? stringValue(value.key) +} + +export function getConfiguredApiKey( + options: { + env?: NodeJS.ProcessEnv + authPaths?: readonly string[] + homeDir?: () => string + } = {}, +): string | undefined { + const env = options.env ?? process.env + if (env.COMMANDCODE_API_KEY) return env.COMMANDCODE_API_KEY + + const home = options.homeDir?.() ?? homedir() + const authPaths = options.authPaths ?? defaultAuthPaths(home) + + for (const authPath of authPaths) { + try { + if (!existsSync(authPath)) continue + const parsed: unknown = JSON.parse(readFileSync(authPath, "utf-8")) + if (!isRecord(parsed)) continue + + const apiKey = stringValue(parsed.apiKey) + if (apiKey) return apiKey + + const commandcode = stringValue(parsed.commandcode) + if (commandcode) return commandcode + + const providerKey = apiKeyFromCredential(parsed.commandcode) + if (providerKey) return providerKey + + const commandCode = stringValue(parsed["command-code"]) + if (commandCode) return commandCode + + const commandCodeKey = apiKeyFromCredential(parsed["command-code"]) + if (commandCodeKey) return commandCodeKey + } catch { + // Ignore malformed or unreadable auth files. + } + } + + return undefined +} diff --git a/src/oauth.ts b/src/oauth.ts index be77391..0a68ac5 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -1,13 +1,13 @@ /** * Command Code OAuth provider for pi's /login flow. * - * Implements a browser-assisted API key retrieval flow: - * 1. Starts a local HTTP server on a Command Code CLI-compatible port - * 2. Opens the Command Code Studio auth page in the browser - * 3. The user authenticates on the Command Code website - * 4. The website POSTs the API key back to the local server - * 5. If browser transfer fails, the user can paste the API key manually - * 6. The API key is stored in pi's auth.json as OAuth credentials + * Implements two API key retrieval flows: + * 1. Browser-assisted login opens Command Code Studio and waits for the + * website to POST the API key back to a local callback server. + * 2. Direct API key login prompts the user to paste a Studio API key. + * + * If browser transfer fails, the user can still paste the API key manually. + * The API key is stored in pi's auth.json as OAuth credentials. * * Since Command Code API keys don't expire, we store them as * OAuth credentials with a far-future expiry. @@ -101,13 +101,35 @@ async function promptForApiKey(callbacks: OAuthLoginCallbacks, message: string) return credentialsFromApiKey(apiKey) } -/** - * Starts the browser-based login flow for Command Code. - * - * Returns OAuth credentials where access == refresh == the user's API key. - * The keys don't expire, so we set a far-future expiry. - */ -export async function login(callbacks: OAuthLoginCallbacks): Promise { +type LoginChoice = { type: "browser" } | { type: "prompt" } | { type: "apiKey"; apiKey: string } + +async function chooseLoginFlow(callbacks: OAuthLoginCallbacks): Promise { + const input = sanitizeApiKey( + await callbacks.onPrompt({ + message: + "Command Code login: press Enter for browser login, type 'key' to paste an API key, or paste the API key directly:", + }), + ) + const normalized = input.toLowerCase() + + if (!input || normalized === "1" || normalized === "b" || normalized === "browser") { + return { type: "browser" } + } + + if ( + normalized === "2" || + normalized === "k" || + normalized === "key" || + normalized === "api" || + normalized === "paste" + ) { + return { type: "prompt" } + } + + return { type: "apiKey", apiKey: input } +} + +async function browserLogin(callbacks: OAuthLoginCallbacks): Promise { let authServer try { authServer = await startAuthServer() @@ -151,6 +173,23 @@ export async function login(callbacks: OAuthLoginCallbacks): Promise { + const choice = await chooseLoginFlow(callbacks) + + if (choice.type === "apiKey") return credentialsFromApiKey(choice.apiKey) + if (choice.type === "prompt") { + return promptForApiKey(callbacks, "Paste your Command Code API key:") + } + + return browserLogin(callbacks) +} + /** * Command Code API keys don't expire, so "refresh" is a no-op. * Returns the same credentials with an updated far-future expiry. diff --git a/tests/test-api-key.ts b/tests/test-api-key.ts new file mode 100644 index 0000000..a497978 --- /dev/null +++ b/tests/test-api-key.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "node:test" + +import { getConfiguredApiKey } from "../src/api-key.ts" + +async function withAuthFile( + value: unknown, + run: (authPath: string) => Promise, +): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-commandcode-auth-")) + const authPath = join(directory, "auth.json") + try { + await writeFile(authPath, JSON.stringify(value), "utf-8") + await run(authPath) + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +describe("getConfiguredApiKey()", () => { + it("prefers the environment variable", () => { + assert.equal( + getConfiguredApiKey({ env: { COMMANDCODE_API_KEY: "env-key" }, authPaths: [] }), + "env-key", + ) + }) + + it("reads pi OAuth and API credentials", async () => { + const cases: readonly { credential: unknown; expected: string }[] = [ + { + credential: { commandcode: { type: "oauth", access: "oauth-key" } }, + expected: "oauth-key", + }, + { credential: { commandcode: { type: "api", key: "api-key" } }, expected: "api-key" }, + { credential: { "command-code": { type: "api", key: "cli-key" } }, expected: "cli-key" }, + { credential: { apiKey: "legacy-key" }, expected: "legacy-key" }, + ] + + for (const testCase of cases) { + await withAuthFile(testCase.credential, async (authPath) => { + assert.equal(getConfiguredApiKey({ env: {}, authPaths: [authPath] }), testCase.expected) + }) + } + }) + + it("ignores malformed files", async () => { + const directory = await mkdtemp(join(tmpdir(), "pi-commandcode-auth-")) + const authPath = join(directory, "auth.json") + try { + await writeFile(authPath, "not json", "utf-8") + assert.equal(getConfiguredApiKey({ env: {}, authPaths: [authPath] }), undefined) + } finally { + await rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/test-oauth.ts b/tests/test-oauth.ts index 0ff0fab..00d94a0 100644 --- a/tests/test-oauth.ts +++ b/tests/test-oauth.ts @@ -186,7 +186,7 @@ describe("login()", () => { authUrl = params.url }, onPrompt(_params: { message: string }): Promise { - throw new Error("onPrompt should not be called in browser flow") + return Promise.resolve("") }, } @@ -239,7 +239,7 @@ describe("login()", () => { process.env.COMMANDCODE_AUTH_TIMEOUT_MS = "1" let authUrl = "" - let promptMessage = "" + const promptMessages: string[] = [] try { const result = await login({ @@ -247,13 +247,13 @@ describe("login()", () => { authUrl = params.url }, async onPrompt(params: { message: string }): Promise { - promptMessage = params.message - return "\u001b[200~ user_manualApiKey\n\u001b[201~" + promptMessages.push(params.message) + return promptMessages.length === 1 ? "" : "\u001b[200~ user_manualApiKey\n\u001b[201~" }, }) assert.match(authUrl, /^https:\/\/commandcode\.ai\/studio\/auth\/cli\?/) - assert.match(promptMessage, /Paste your Command Code API key/) + assert.match(promptMessages[1] ?? "", /Paste your Command Code API key/) assert.equal(result.access, "user_manualApiKey") assert.equal(result.refresh, "user_manualApiKey") assert.ok(result.expires > Date.now(), "expiry should be far in the future") @@ -263,6 +263,37 @@ describe("login()", () => { } }) + it("accepts a directly pasted API key", async () => { + let authOpened = false + const result = await login({ + onAuth() { + authOpened = true + }, + onPrompt(): Promise { + return Promise.resolve("user_directApiKey") + }, + }) + + assert.equal(authOpened, false) + assert.equal(result.access, "user_directApiKey") + }) + + it("offers an explicit API key prompt", async () => { + let promptCount = 0 + const result = await login({ + onAuth() { + throw new Error("browser should not open") + }, + onPrompt(): Promise { + promptCount += 1 + return Promise.resolve(promptCount === 1 ? "key" : "user_promptedApiKey") + }, + }) + + assert.equal(result.access, "user_promptedApiKey") + assert.equal(promptCount, 2) + }) + it("rejects on state token mismatch", async () => { let authUrl = "" const callbacks = { @@ -270,7 +301,7 @@ describe("login()", () => { authUrl = params.url }, onPrompt(_params: { message: string }): Promise { - throw new Error("should not prompt") + return Promise.resolve("") }, } From 84a802d91dd0e7cebc0d5458a91834a1b4e17a14 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 18 Aug 2026 10:56:06 +0200 Subject: [PATCH 02/40] feat(api): use official provider endpoints --- index.ts | 97 ++--- package.json | 9 +- src/converters.ts | 296 -------------- src/core.ts | 741 --------------------------------- src/cost.ts | 31 -- src/json-schema.ts | 382 ----------------- src/models.ts | 17 +- src/overflow.ts | 120 ------ src/types.ts | 201 --------- tests/helpers.ts | 289 ------------- tests/test-abort.ts | 85 ---- tests/test-cost.ts | 183 --------- tests/test-models.ts | 16 + tests/test-omp-compat.mjs | 21 +- tests/test-overflow.ts | 196 --------- tests/test-pi-local.mjs | 139 +++++-- tests/test-pure-functions.ts | 658 ------------------------------ tests/test-retry.ts | 470 --------------------- tests/test-runtime.ts | 2 + tests/test-stream.ts | 771 ----------------------------------- 20 files changed, 199 insertions(+), 4525 deletions(-) delete mode 100644 src/converters.ts delete mode 100644 src/core.ts delete mode 100644 src/cost.ts delete mode 100644 src/json-schema.ts delete mode 100644 src/overflow.ts delete mode 100644 src/types.ts delete mode 100644 tests/helpers.ts delete mode 100644 tests/test-abort.ts delete mode 100644 tests/test-cost.ts delete mode 100644 tests/test-overflow.ts delete mode 100644 tests/test-pure-functions.ts delete mode 100644 tests/test-retry.ts delete mode 100644 tests/test-stream.ts diff --git a/index.ts b/index.ts index 24a77b1..552e3ee 100644 --- a/index.ts +++ b/index.ts @@ -1,12 +1,10 @@ /** * Command Code provider for pi. * - * Connects pi to Command Code's API (https://api.commandcode.ai/alpha/generate). - * The provider uses pi's legacy extension registration surface because the - * current pi host exposes `registerProvider(name, config)`, including OMP. + * Uses Command Code's documented Provider API: + * https://api.commandcode.ai/provider/v1 */ -import { AssistantMessageEventStream } from "@earendil-works/pi-ai" import { getAgentDir, type ExtensionAPI, @@ -15,10 +13,11 @@ import { } from "@earendil-works/pi-coding-agent" import { join } from "node:path" -import { COMMAND_CODE_CLI_VERSION, createStreamCommandCode, DEFAULT_API_BASE } from "./src/core.ts" -import { calculateCommandCodeCost } from "./src/cost.ts" +import { getConfiguredApiKey } from "./src/api-key.ts" import { + baseUrlForModel, DEFAULT_MODELS_URL, + DEFAULT_PROVIDER_API_BASE, getModelsTimeoutMs, inputModalitiesForModel, loadCommandCodeModels, @@ -28,73 +27,67 @@ import { import { getApiKey as getOAuthApiKey, login, refreshToken } from "./src/oauth.ts" import { MODEL_COSTS, ZERO_MODEL_COST } from "./src/pricing.ts" import { createCommandCodeRuntime } from "./src/runtime.ts" -import { normalizeCommandCodeMessage } from "./src/overflow.ts" + +function commandCodeHeaders(): Record | undefined { + if (process.env.CMD_ZDR === "1" || process.env.COMMANDCODE_ZDR === "1") { + return { "x-cmd-zdr": "1" } + } + return undefined +} function createProviderConfig( models: readonly CommandCodeModel[], apiBase: string, - streamCommandCode: ProviderConfig["streamSimple"], ): ProviderConfig { + const headers = commandCodeHeaders() return { name: "Command Code", baseUrl: apiBase, - // Keep environment authentication dynamic. OAuth credentials are resolved - // by pi's oauth registration, while the custom stream retains its own - // request-time legacy-file fallback for older compatible hosts. - apiKey: "$COMMANDCODE_API_KEY", - authHeader: true, - api: "commandcode-custom", - streamSimple: streamCommandCode, - headers: { - "x-command-code-version": COMMAND_CODE_CLI_VERSION, - "x-cli-environment": "production", - }, + apiKey: getConfiguredApiKey() ?? "$COMMANDCODE_API_KEY", + api: "openai-completions", + headers, oauth: { name: "Command Code", login, refreshToken, getApiKey: getOAuthApiKey, }, - models: models.map(createProviderModel), + models: models.map((model) => ({ + id: model.id, + name: model.name, + api: model.api, + baseUrl: baseUrlForModel(apiBase, model.api), + reasoning: model.reasoning, + ...(thinkingMetadataForModel(model.id) ?? {}), + input: [...inputModalitiesForModel(model.id)], + cost: MODEL_COSTS[model.id] ?? ZERO_MODEL_COST, + contextWindow: model.contextWindow, + maxTokens: model.maxTokens, + headers, + compat: + model.api === "openai-completions" + ? { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: true, + maxTokensField: "max_tokens", + } + : { + supportsEagerToolInputStreaming: false, + supportsLongCacheRetention: false, + supportsCacheControlOnTools: false, + supportsToolReferences: false, + }, + })), } } -function createProviderModel(model: { - id: string - name: string - reasoning: boolean - contextWindow: number - maxTokens: number -}) { - return { - id: model.id, - name: model.name, - reasoning: model.reasoning, - ...(thinkingMetadataForModel(model.id) ?? {}), - input: inputModalitiesForModel(model.id), - cost: MODEL_COSTS[model.id] ?? ZERO_MODEL_COST, - contextWindow: model.contextWindow, - maxTokens: model.maxTokens, - } as const -} - export default async function (pi: ExtensionAPI) { - const apiBase = process.env.COMMANDCODE_API_BASE ?? DEFAULT_API_BASE + const apiBase = process.env.COMMANDCODE_API_BASE ?? DEFAULT_PROVIDER_API_BASE const modelsUrl = process.env.COMMANDCODE_MODELS_URL ?? DEFAULT_MODELS_URL const modelsTimeoutMs = getModelsTimeoutMs() const modelsCachePath = process.env.COMMANDCODE_MODELS_CACHE ?? join(getAgentDir(), "commandcode-models.json") - const streamCommandCode = createStreamCommandCode({ - createStream: () => new AssistantMessageEventStream(), - calculateCost: calculateCommandCodeCost, - apiBase, - }) - - pi.on("message_end", async (event, ctx) => { - if (event.message.role !== "assistant") return - const normalized = normalizeCommandCodeMessage(event.message, ctx.model?.provider) - return normalized ? { message: normalized.message } : undefined - }) const runtime = createCommandCodeRuntime(pi, { endpoint: modelsUrl, @@ -105,7 +98,7 @@ export default async function (pi: ExtensionAPI) { cachePath: modelsCachePath, timeoutMs: modelsTimeoutMs, }), - createProviderConfig: (models) => createProviderConfig(models, apiBase, streamCommandCode), + createProviderConfig: (models) => createProviderConfig(models, apiBase), }) await runtime.initialize() diff --git a/package.json b/package.json index 05a6ef0..9af81b6 100644 --- a/package.json +++ b/package.json @@ -29,21 +29,18 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-oauth.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", "pi:isolated": "node scripts/pi-isolated.mjs", "pi:authenticated": "node scripts/pi-authenticated.mjs", - "test:unit": "tsx tests/test-pure-functions.ts", + "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-oauth.ts", + "test:api-key": "tsx tests/test-api-key.ts", "test:models": "tsx tests/test-models.ts", "test:runtime": "tsx tests/test-runtime.ts", "test:pricing": "tsx tests/test-pricing.ts", "test:oauth": "tsx tests/test-oauth.ts", - "test:abort": "tsx tests/test-abort.ts", - "test:overflow": "tsx tests/test-overflow.ts", - "test:stream": "tsx tests/test-stream.ts", - "test:retry": "tsx tests/test-retry.ts", "test:pi-isolated": "node tests/test-pi-isolated.mjs", "test:pi-authenticated": "node tests/test-pi-authenticated.mjs", "test:pi-local": "node tests/test-pi-local.mjs", diff --git a/src/converters.ts b/src/converters.ts deleted file mode 100644 index 4012fbb..0000000 --- a/src/converters.ts +++ /dev/null @@ -1,296 +0,0 @@ -import { existsSync, readFileSync } from "node:fs" -import { homedir } from "node:os" -import { join } from "node:path" - -import type { MessageLike, StopReason, ToolLike } from "./types.ts" -import { toJsonSchema } from "./json-schema.ts" - -export { toJsonSchema } from "./json-schema.ts" - -export function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value) -} - -export function stringValue(value: unknown): string | undefined { - return typeof value === "string" ? value : undefined -} - -export function recordArray(value: unknown): readonly Record[] { - if (!Array.isArray(value)) return [] - return value.filter(isRecord) -} - -export function recordOrEmpty(value: unknown): Record { - if (isRecord(value)) return value - if (typeof value === "string") { - try { - const parsed: unknown = JSON.parse(value) - if (isRecord(parsed)) return parsed - } catch { - // Some providers stream incomplete JSON argument fragments. - } - } - return {} -} - -export function numberValue(value: unknown): number | undefined { - return typeof value === "number" && Number.isFinite(value) ? value : undefined -} - -function defaultAuthPaths(home: string): string[] { - return [ - join(home, ".commandcode", "auth.json"), - join(home, ".omp", "agent", "auth.json"), - join(home, ".pi", "agent", "auth.json"), - ] -} - -function apiKeyFromCredentialRecord(value: unknown): string | undefined { - if (!isRecord(value)) return undefined - - const type = stringValue(value.type) - if (type === "api") return stringValue(value.key) - if (type === "oauth") return stringValue(value.access) - - return stringValue(value.key) ?? stringValue(value.access) -} - -function imageParts(value: unknown): readonly Record[] { - if (isRecord(value)) return value.type === "image" ? [value] : [] - return recordArray(value).filter((part) => part.type === "image") -} - -function imageContentError(role: string): Error { - return new Error(`Selected Command Code model does not support image content in ${role}`) -} - -export function assertTextOnlyMessages(messages?: readonly MessageLike[]): void { - for (const message of messages ?? []) { - if (imageParts(message.content).length > 0) { - const role = message.role === "toolResult" ? "tool results" : `${message.role} messages` - throw imageContentError(role) - } - } -} - -function imageToCommandCode(part: Record): Record { - const data = stringValue(part.data) - const mimeType = stringValue(part.mimeType) - if (!data || !mimeType) - throw new Error("Invalid image content: expected base64 data and mimeType") - - return { - type: "image", - image: `data:${mimeType};base64,${data}`, - mimeType, - } -} - -function userContentToCommandCode(content: unknown, allowImages: boolean): unknown { - if (typeof content === "string") return content - - return recordArray(content).flatMap((part) => { - if (part.type === "text") return [{ type: "text", text: stringValue(part.text) ?? "" }] - if (part.type === "image") { - if (!allowImages) throw imageContentError("user messages") - return [imageToCommandCode(part)] - } - return [] - }) -} - -export function getApiKey( - options: { - env?: NodeJS.ProcessEnv - authPaths?: readonly string[] - homeDir?: () => string - } = {}, -): string | undefined { - const env = options.env ?? process.env - if (env.COMMANDCODE_API_KEY) return env.COMMANDCODE_API_KEY - - const home = options.homeDir?.() ?? homedir() - const authPaths = options.authPaths ?? defaultAuthPaths(home) - - for (const authPath of authPaths) { - try { - if (!existsSync(authPath)) continue - const parsed: unknown = JSON.parse(readFileSync(authPath, "utf-8")) - if (!isRecord(parsed)) continue - - // Legacy: direct apiKey or commandcode field. - const apiKey = stringValue(parsed.apiKey) - if (apiKey) return apiKey - const commandcode = stringValue(parsed.commandcode) - if (commandcode) return commandcode - - // pi stores OAuth credentials as {"commandcode": {"type":"oauth","access":"..."}}. - // The official Command Code CLI stores API credentials under "command-code". - const providerKey = - apiKeyFromCredentialRecord(parsed.commandcode) ?? - apiKeyFromCredentialRecord(parsed["command-code"]) - if (providerKey) return providerKey - } catch { - // Ignore malformed or unreadable auth files. - } - } - - return undefined -} - -export function textContent(message: { content?: unknown }): string { - return recordArray(message.content) - .filter((part) => part.type === "text") - .map((part) => stringValue(part.text) ?? "") - .join("\n") -} - -export function getEnvironmentInfo(): string { - return `${process.platform}-${process.arch}, Node.js ${process.version}` -} - -export function toolsToJson(tools?: readonly ToolLike[]): unknown[] { - if (!tools) return [] - return tools.map((tool) => ({ - type: "function", - name: tool.name, - description: tool.description, - input_schema: tool.parameters ? toJsonSchema(tool.parameters) : {}, - })) -} - -function completeToolCallIds(messages?: readonly MessageLike[]): Set { - const callIds = new Set() - const resultIds = new Set() - - for (const message of messages ?? []) { - if (message.role === "assistant") { - for (const content of recordArray(message.content)) { - if (content.type === "toolCall") { - const id = stringValue(content.id) - if (id) callIds.add(id) - } - } - } else if (message.role === "toolResult") { - if (message.toolCallId) resultIds.add(message.toolCallId) - } - } - - return new Set([...callIds].filter((id) => resultIds.has(id))) -} - -export function messagesToCC( - messages?: readonly MessageLike[], - options: { allowImages?: boolean } = {}, -): unknown[] { - const allowImages = options.allowImages ?? false - if (!allowImages) assertTextOnlyMessages(messages) - - const out: unknown[] = [] - const pairedToolCallIds = completeToolCallIds(messages) - - for (const message of messages ?? []) { - if (message.role === "user") { - out.push({ - role: "user", - content: userContentToCommandCode(message.content, allowImages), - }) - } else if (message.role === "assistant") { - const parts: unknown[] = [] - for (const content of recordArray(message.content)) { - if (content.type === "text") { - parts.push({ type: "text", text: stringValue(content.text) ?? "" }) - } else if (content.type === "toolCall") { - const toolCallId = stringValue(content.id) ?? "" - if (!pairedToolCallIds.has(toolCallId)) continue - parts.push({ - type: "tool-call", - toolCallId, - toolName: stringValue(content.name) ?? "", - input: recordOrEmpty(content.arguments), - }) - } - } - if (parts.length > 0) out.push({ role: "assistant", content: parts }) - } else if (message.role === "toolResult") { - if (!message.toolCallId || !pairedToolCallIds.has(message.toolCallId)) continue - out.push({ - role: "tool", - content: [ - { - type: "tool-result", - toolCallId: message.toolCallId, - toolName: message.toolName, - output: message.isError - ? { type: "error-text", value: textContent(message) } - : { type: "text", value: textContent(message) }, - }, - ], - }) - - const images = imageParts(message.content) - if (images.length > 0) { - if (!allowImages) throw imageContentError("tool results") - out.push({ - role: "user", - content: images.map(imageToCommandCode), - }) - } - } - } - return out -} - -export function parseStreamEventLine(line: string): unknown | undefined { - let trimmed = line.trim() - if (!trimmed || trimmed.startsWith(":") || trimmed.startsWith("event:")) return undefined - if (trimmed.startsWith("data:")) trimmed = trimmed.slice(5).trim() - if (!trimmed || trimmed === "[DONE]") return undefined - - try { - const parsed: unknown = JSON.parse(trimmed) - return parsed - } catch { - return undefined - } -} - -export function mapFinishReason(reason: unknown): StopReason { - if (reason === "tool-calls") return "toolUse" - if ( - reason === "length" || - reason === "max_tokens" || - reason === "max-tokens" || - reason === "max_output_tokens" - ) { - return "length" - } - return "stop" -} - -function promptPartToText(value: unknown, depth = 0): string { - if (depth > 10) return "" - if (typeof value === "string") return value - if (Array.isArray(value)) - return value - .map((v) => promptPartToText(v, depth + 1)) - .filter(Boolean) - .join("\n") - if (!isRecord(value)) return "" - const text = stringValue(value.text) - if (text) return text - const content = promptPartToText(value.content, depth + 1) - if (content) return content - return "" -} - -export function systemPromptToText(value: unknown): string { - if (value === undefined || value === null) return "" - if (typeof value === "string") return value - if (Array.isArray(value)) - return value - .map((v) => promptPartToText(v, 0)) - .filter(Boolean) - .join("\n\n") - return promptPartToText(value, 0) -} diff --git a/src/core.ts b/src/core.ts deleted file mode 100644 index f1898dd..0000000 --- a/src/core.ts +++ /dev/null @@ -1,741 +0,0 @@ -/** - * Testable Command Code provider core. - * - * The runtime imports live in index.ts; this module takes injected stream/cost - * dependencies so tests can exercise the real serialization and stream parser. - */ - -import { randomUUID } from "node:crypto" - -import { commandCodeErrorMessage, redactCommandCodeErrorText } from "./overflow.ts" -import { modelSupportsImageInput } from "./models.ts" -import { - getApiKey, - getEnvironmentInfo, - isRecord, - assertTextOnlyMessages, - mapFinishReason, - messagesToCC, - numberValue, - parseStreamEventLine, - recordOrEmpty, - stringValue, - toolsToJson, - systemPromptToText, -} from "./converters.ts" -import type { - AssistantMessageEventStreamLike, - AssistantMessageLike, - ContextLike, - CoreDependencies, - ErrorReason, - ModelLike, - StopReason, - StreamOptions, - TerminalReason, - TextContent, - ToolCallContent, - Usage, -} from "./types.ts" - -export * from "./converters.ts" -export * from "./overflow.ts" -export * from "./types.ts" - -export const DEFAULT_API_BASE = "https://api.commandcode.ai" -export const COMMAND_CODE_CLI_VERSION = "1.15.1" - -const DEFAULT_GENERATE_MAX_TOKENS = 64_000 -const DEFAULT_MAX_RETRIES = 0 -const DEFAULT_MAX_RETRY_DELAY_MS = 60_000 -const BASE_RETRY_DELAY_MS = 500 - -function isRetryableStatus(status: number): boolean { - return status === 429 || (status >= 500 && status < 600) -} - -function parseRetryAfterSeconds(value: string | null): number | undefined { - if (!value) return undefined - const seconds = Number(value) - if (Number.isFinite(seconds) && seconds >= 0) return seconds - const date = Date.parse(value) - if (!Number.isNaN(date)) return Math.max(0, (date - Date.now()) / 1000) - return undefined -} - -function effectiveMaxRetryDelayMs(value: number | undefined): number { - if (value === undefined) return DEFAULT_MAX_RETRY_DELAY_MS - if (value === 0) return Number.POSITIVE_INFINITY - return value -} - -function retryDelayMs( - attempt: number, - retryAfterHeader: string | null, - maxDelayMs: number, -): number { - const retryAfterMs = parseRetryAfterSeconds(retryAfterHeader) - if (retryAfterMs !== undefined) { - if (retryAfterMs * 1000 > maxDelayMs) return -1 - return retryAfterMs * 1000 - } - const exponential = BASE_RETRY_DELAY_MS * 2 ** attempt - const jitter = exponential * 0.2 * Math.random() - return Math.min(exponential + jitter, maxDelayMs) -} - -function defaultUsage(): Usage { - return { - input: 0, - output: 0, - cacheRead: 0, - cacheWrite: 0, - totalTokens: 0, - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, - } -} - -function commandCodeUsage(event: Record): Record | undefined { - return isRecord(event.totalUsage) ? event.totalUsage : undefined -} - -function commandCodeInputTokenDetails( - usage: Record, -): Record | undefined { - return isRecord(usage.inputTokenDetails) ? usage.inputTokenDetails : undefined -} - -function headersToRecord(headers: Headers): Record { - const out: Record = {} - headers.forEach((value, key) => { - out[key] = value - }) - return out -} - -function abortError(message = "The operation was aborted"): DOMException { - return new DOMException(message, "AbortError") -} - -function timeoutError(timeoutMs: number | undefined): Error { - return new Error( - timeoutMs === undefined - ? "Command Code API request timed out" - : `Command Code API request timed out after ${timeoutMs}ms`, - ) -} - -function successStopReason(reason: TerminalReason): StopReason { - if (reason === "length" || reason === "toolUse") return reason - return "stop" -} - -function generateMaxTokens(model: ModelLike, options?: StreamOptions): number { - return Math.min( - options?.maxTokens ?? model.maxTokens, - model.maxTokens, - DEFAULT_GENERATE_MAX_TOKENS, - ) -} - -function mappedReasoningEffort(model: ModelLike, options?: StreamOptions): string | undefined { - const level = options?.reasoning - if (!level || level === "off" || !model.reasoning) return undefined - - const effortMap = model.thinking?.effortMap ?? model.thinkingLevelMap - const mapped = effortMap?.[level] - return typeof mapped === "string" && mapped !== "off" ? mapped : undefined -} - -export function projectSlugFromPath(pathName: string): string { - const slug = pathName - .toLowerCase() - .replace(/^[a-z]:/i, "") - .replace(/[^a-z0-9]+/g, "-") - .replace(/^-+|-+$/g, "") - return slug || "project" -} - -export function createStreamCommandCode(deps: CoreDependencies) { - const apiBase = deps.apiBase ?? DEFAULT_API_BASE - const fetchImpl = deps.fetchImpl ?? fetch - const cwd = deps.cwd ?? (() => process.cwd()) - const now = deps.now ?? (() => Date.now()) - const uuid = deps.uuid ?? (() => randomUUID()) - const delay = - deps.delay ?? - ((ms: number, signal: AbortSignal) => { - if (signal.aborted) return Promise.reject(abortError()) - return new Promise((resolve, reject) => { - const id = setTimeout(() => { - signal.removeEventListener("abort", onAbort) - resolve() - }, ms) - const onAbort = () => { - clearTimeout(id) - reject(abortError()) - } - signal.addEventListener("abort", onAbort, { once: true }) - }) - }) - - function raceAbort(promise: Promise, signal: AbortSignal): Promise { - if (signal.aborted) return Promise.reject(abortError()) - - return new Promise((resolve, reject) => { - const onAbort = () => reject(abortError()) - signal.addEventListener("abort", onAbort, { once: true }) - promise.then( - (value) => { - signal.removeEventListener("abort", onAbort) - resolve(value) - }, - (error: unknown) => { - signal.removeEventListener("abort", onAbort) - reject(error) - }, - ) - }) - } - - function raceAbortWithTimeout( - promise: Promise, - controller: AbortController, - timeoutMs: number | undefined, - ): Promise { - if (timeoutMs === undefined) return raceAbort(promise, controller.signal) - - return new Promise((resolve, reject) => { - const timer = setTimeout(() => { - controller.abort() - reject(timeoutError(timeoutMs)) - }, timeoutMs) - raceAbort(promise, controller.signal).then( - (value) => { - clearTimeout(timer) - resolve(value) - }, - (error: unknown) => { - clearTimeout(timer) - reject(error) - }, - ) - }) - } - - return function streamCommandCode( - model: ModelLike, - context: ContextLike, - options?: StreamOptions, - ): AssistantMessageEventStreamLike { - const stream = deps.createStream() - - async function run() { - // OMP may pass the legacy env-var name "COMMANDCODE_API_KEY" (old pi) - // or "$COMMANDCODE_API_KEY" (new pi) as the apiKey value instead of - // resolving it. Filter out these specific strings. - const LEGACY_API_KEY_REF = "$COMMANDCODE_API_KEY" - const OLD_API_KEY_REF = "COMMANDCODE_API_KEY" - const hostKey = - options?.apiKey && - options.apiKey !== LEGACY_API_KEY_REF && - options.apiKey !== OLD_API_KEY_REF - ? options.apiKey - : undefined - - const apiKey = - hostKey ?? - getApiKey({ - env: deps.env, - authPaths: deps.authPaths, - homeDir: deps.homeDir, - }) - - if (!apiKey) { - const msg: AssistantMessageLike = { - role: "assistant", - content: [], - api: model.api, - provider: model.provider, - model: model.id, - usage: defaultUsage(), - stopReason: "error", - errorMessage: - "No Command Code API key. Run /login and select Command Code, set the COMMANDCODE_API_KEY env var, or configure ~/.commandcode/auth.json, ~/.pi/agent/auth.json or ~/.omp/agent/auth.json", - timestamp: now(), - } - stream.push({ type: "error", reason: "error", error: msg }) - stream.end() - return - } - - const output: AssistantMessageLike = { - role: "assistant", - content: [], - api: model.api, - provider: model.provider, - model: model.id, - usage: defaultUsage(), - stopReason: "stop", - timestamp: now(), - } - - const controller = new AbortController() - let reader: ReadableStreamDefaultReader | undefined - let textBlock: TextContent | undefined - let currentTextIdx = -1 - let thinkingIdx = -1 - let finished = false - - const abortUpstream = () => { - if (!controller.signal.aborted) controller.abort() - try { - reader?.cancel().catch(() => undefined) - } catch { - // Reader cancellation is best-effort. - } - } - - if (options?.signal?.aborted) { - abortUpstream() - } else { - options?.signal?.addEventListener("abort", abortUpstream, { - once: true, - }) - } - - const endTextBlock = () => { - if (!textBlock) return - stream.push({ - type: "text_end", - contentIndex: currentTextIdx, - content: textBlock.text, - partial: output, - }) - textBlock = undefined - currentTextIdx = -1 - } - - const endThinking = () => { - if (thinkingIdx < 0) return - const tc = output.content[thinkingIdx] - if (tc && tc.type === "thinking") { - stream.push({ - type: "thinking_end", - contentIndex: thinkingIdx, - content: (tc as { thinking: string }).thinking, - partial: output, - }) - } - thinkingIdx = -1 - } - - const handleEvent = (event: unknown) => { - if (!isRecord(event)) return - - switch (event.type) { - case "text-delta": { - endThinking() - if (!textBlock) { - textBlock = { type: "text", text: "" } - output.content.push(textBlock) - currentTextIdx = output.content.length - 1 - stream.push({ - type: "text_start", - contentIndex: currentTextIdx, - partial: output, - }) - } - const delta = stringValue(event.text) ?? "" - textBlock.text += delta - stream.push({ - type: "text_delta", - contentIndex: currentTextIdx, - delta, - partial: output, - }) - break - } - - case "reasoning-start": { - endTextBlock() - break - } - - case "reasoning-delta": { - endTextBlock() - const delta = stringValue(event.text) ?? "" - if (thinkingIdx < 0) { - output.content.push({ type: "thinking", thinking: delta }) - thinkingIdx = output.content.length - 1 - stream.push({ - type: "thinking_start", - contentIndex: thinkingIdx, - partial: output, - }) - } else { - const tc = output.content[thinkingIdx] - if (tc && tc.type === "thinking") { - ;(tc as { thinking: string }).thinking += delta - } - } - stream.push({ - type: "thinking_delta", - contentIndex: thinkingIdx, - delta, - partial: output, - }) - break - } - - case "reasoning-end": { - endThinking() - break - } - - case "tool-result": { - break - } - - case "tool-call": { - endTextBlock() - endThinking() - const toolCall: ToolCallContent = { - type: "toolCall", - id: stringValue(event.toolCallId) ?? "", - name: stringValue(event.toolName) ?? "", - arguments: recordOrEmpty(event.input ?? event.args ?? event.arguments), - } - output.content.push(toolCall) - const idx = output.content.length - 1 - stream.push({ - type: "toolcall_start", - contentIndex: idx, - partial: output, - }) - stream.push({ - type: "toolcall_end", - contentIndex: idx, - toolCall, - partial: output, - }) - break - } - - case "finish": { - const usage = commandCodeUsage(event) - if (usage) { - const details = commandCodeInputTokenDetails(usage) - const totalInput = numberValue(usage.inputTokens) ?? 0 - const input = numberValue(details?.noCacheTokens) - const cacheRead = numberValue(details?.cacheReadTokens) ?? 0 - const cacheWrite = numberValue(details?.cacheWriteTokens) ?? 0 - output.usage.input = input ?? Math.max(0, totalInput - cacheRead - cacheWrite) - output.usage.output = numberValue(usage.outputTokens) ?? 0 - output.usage.cacheRead = cacheRead - output.usage.cacheWrite = cacheWrite - output.usage.totalTokens = - output.usage.input + - output.usage.output + - output.usage.cacheRead + - output.usage.cacheWrite - deps.calculateCost(model, output.usage) - } - output.stopReason = mapFinishReason(event.finishReason) - finished = true - break - } - - case "error": { - const message = - commandCodeErrorMessage(event.error) ?? - commandCodeErrorMessage(event.message) ?? - "Stream error" - output.stopReason = "error" - output.errorMessage = message - throw new Error(message) - } - } - } - - try { - stream.push({ type: "start", partial: output }) - if (controller.signal.aborted) throw abortError("Aborted") - - const workingDir = cwd() - const threadId = uuid() - const reasoningEffort = mappedReasoningEffort(model, options) - const timeoutMs = options?.timeoutMs - - const allowImages = modelSupportsImageInput(model.id) - if (!allowImages) assertTextOnlyMessages(context.messages) - - let body: unknown = { - config: { - workingDir, - date: new Date(now()).toISOString().split("T")[0], - environment: getEnvironmentInfo(), - structure: [], - isGitRepo: false, - currentBranch: "", - mainBranch: "", - gitStatus: "", - recentCommits: [], - }, - memory: null, - taste: null, - skills: null, - params: { - model: model.id, - messages: messagesToCC(context.messages, { allowImages }), - tools: toolsToJson(context.tools), - system: systemPromptToText(context.systemPrompt), - max_tokens: generateMaxTokens(model, options), - temperature: 0.3, - stream: true, - ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}), - }, - threadId, - } - - const payloadController = new AbortController() - const onPayloadAbort = () => payloadController.abort() - controller.signal.addEventListener("abort", onPayloadAbort, { once: true }) - let nextBody: unknown - try { - nextBody = await raceAbortWithTimeout( - Promise.resolve(options?.onPayload?.(body, model)), - payloadController, - timeoutMs, - ) - } finally { - controller.signal.removeEventListener("abort", onPayloadAbort) - } - if (nextBody !== undefined) body = nextBody - - const maxRetries = options?.maxRetries ?? DEFAULT_MAX_RETRIES - const maxRetryDelayMs = effectiveMaxRetryDelayMs(options?.maxRetryDelayMs) - const requestHeaders = { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - "x-command-code-version": COMMAND_CODE_CLI_VERSION, - "x-cli-environment": "production", - "x-project-slug": projectSlugFromPath(workingDir), - "x-taste-learning": "true", - "x-co-flag": "false", - ...options?.headers, - } - const bodyStr = JSON.stringify(body) - - let response!: Response - retryLoop: for (let attempt = 0; ; attempt++) { - const attemptController = new AbortController() - let attemptTimedOut = false - let attemptTimeoutId: ReturnType | undefined - - const clearAttemptTimeout = () => { - if (attemptTimeoutId !== undefined) { - clearTimeout(attemptTimeoutId) - attemptTimeoutId = undefined - } - } - - if (timeoutMs !== undefined) { - attemptTimeoutId = setTimeout(() => { - attemptTimedOut = true - attemptController.abort() - }, timeoutMs) - } - const onOuterAbort = () => attemptController.abort() - controller.signal.addEventListener("abort", onOuterAbort, { once: true }) - const raceAttempt = (promise: Promise): Promise => - raceAbort(promise, attemptController.signal).catch((error: unknown) => { - if (attemptTimedOut) throw timeoutError(timeoutMs) - throw error - }) - - try { - try { - response = await fetchImpl(`${apiBase}/alpha/generate`, { - method: "POST", - headers: requestHeaders, - body: bodyStr, - signal: attemptController.signal, - }) - } catch (fetchError: unknown) { - if (controller.signal.aborted) throw abortError("Aborted") - if (attemptTimedOut) { - if (attempt < maxRetries) continue retryLoop - throw timeoutError(timeoutMs) - } - throw fetchError - } - - // --- HTTP-level retry --- - if (!response.ok && isRetryableStatus(response.status)) { - const retryAfter = response.headers.get("retry-after") - const waitMs = retryDelayMs(attempt, retryAfter, maxRetryDelayMs) - if (waitMs < 0) { - const requestedSeconds = parseRetryAfterSeconds(retryAfter) ?? 0 - const capLabel = - maxRetryDelayMs === Number.POSITIVE_INFINITY ? "disabled" : `${maxRetryDelayMs}ms` - throw new Error(`Retry-After delay ${requestedSeconds}s exceeds max ${capLabel}`) - } - if (attempt < maxRetries) { - await response.text().catch(() => "") - if (waitMs > 0) await delay(waitMs, controller.signal) - continue retryLoop - } - } - - try { - await raceAttempt( - Promise.resolve( - options?.onResponse?.( - { - status: response.status, - headers: headersToRecord(response.headers), - }, - model, - ), - ), - ) - } catch (error: unknown) { - if (attemptTimedOut && attempt < maxRetries) continue retryLoop - throw error - } - - if (!response.ok) { - const errBody = await raceAttempt(response.text().catch(() => "")) - let errorDetail: string | undefined - try { - const parsedBody: unknown = JSON.parse(errBody) - errorDetail = commandCodeErrorMessage(parsedBody) - } catch { - // Preserve useful plain-text provider errors only after secret - // redaction; upstream/proxy bodies may echo credentials. - } - const safeBody = redactCommandCodeErrorText(errBody).slice(0, 500) - const detail = redactCommandCodeErrorText( - errorDetail ?? (safeBody || "Provider returned an error"), - ) - throw new Error(`Command Code API error ${response.status}: ${detail}`) - } - - // --- Read response stream --- - reader = response.body?.getReader() - if (!reader) throw new Error("No response body") - - const decoder = new TextDecoder() - let buffer = "" - - try { - readLoop: for (;;) { - if (controller.signal.aborted) throw abortError("Aborted") - const { done, value } = await raceAbort(reader.read(), attemptController.signal) - if (done) { - if (buffer.trim()) handleEvent(parseStreamEventLine(buffer)) - break - } - if (controller.signal.aborted) throw abortError("Aborted") - - buffer += decoder.decode(value, { stream: true }) - const lines = buffer.split("\n") - buffer = lines.pop() ?? "" - - for (const line of lines) { - if (controller.signal.aborted) throw abortError("Aborted") - handleEvent(parseStreamEventLine(line)) - if (finished) break readLoop - } - } - } catch (streamError: unknown) { - // Stream-level error (e.g. API returned 200 OK but sent an error event) - // or per-attempt timeout during stream reading. - await reader.cancel().catch(() => {}) - try { - reader.releaseLock() - } catch {} - reader = undefined - - if (controller.signal.aborted) throw streamError - - // Never retry after visible content was emitted (including timeout mid-stream). - const canRetry = output.content.length === 0 && attempt < maxRetries - if (canRetry) { - output.content.length = 0 - textBlock = undefined - currentTextIdx = -1 - thinkingIdx = -1 - output.stopReason = "stop" - output.errorMessage = undefined - finished = false - const waitMs = attemptTimedOut ? 0 : retryDelayMs(attempt, null, maxRetryDelayMs) - if (waitMs > 0) await delay(waitMs, controller.signal) - continue retryLoop - } - if (attemptTimedOut) throw timeoutError(timeoutMs) - throw streamError - } - - // Stream completed successfully. - endTextBlock() - endThinking() - - stream.push({ - type: "done", - reason: successStopReason(output.stopReason), - message: output, - }) - stream.end() - break retryLoop - } finally { - controller.signal.removeEventListener("abort", onOuterAbort) - clearAttemptTimeout() - } - } - } catch (error: unknown) { - const reason: ErrorReason = controller.signal.aborted ? "aborted" : "error" - output.stopReason = reason - output.errorMessage = - reason === "aborted" - ? "Request aborted" - : redactCommandCodeErrorText(error instanceof Error ? error.message : String(error)) - stream.push({ type: "error", reason, error: output }) - stream.end() - } finally { - options?.signal?.removeEventListener("abort", abortUpstream) - try { - await reader?.cancel() - } catch { - // Reader may already be closed/cancelled. - } - try { - reader?.releaseLock() - } catch { - // Reader may already be released/cancelled by the abort path. - } - } - } - - run().catch((error: unknown) => { - const msg: AssistantMessageLike = { - role: "assistant", - content: [], - api: model.api, - provider: model.provider, - model: model.id, - usage: defaultUsage(), - stopReason: "error", - errorMessage: redactCommandCodeErrorText( - error instanceof Error ? error.message : String(error), - ), - timestamp: now(), - } - stream.push({ type: "error", reason: "error", error: msg }) - stream.end() - }) - - return stream - } -} diff --git a/src/cost.ts b/src/cost.ts deleted file mode 100644 index 55c1bd0..0000000 --- a/src/cost.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Local cost calculation for Command Code usage. - * - * Mirrors pi-ai's `calculateCost` arithmetic exactly. The provider ships its - * own copy because Oh My Pi's legacy pi-ai shim does not export - * `calculateCost`, which broke extension installation there (issue #24). - * `tests/test-cost.ts` locks this implementation to the pi-ai original. - */ - -import type { ModelLike, Usage } from "./types.ts" - -export function calculateCommandCodeCost(model: ModelLike, usage: Usage): void { - const inputTokens = usage.input + usage.cacheRead + usage.cacheWrite - let rates = model.cost - let matchedThreshold = -1 - for (const tier of model.cost.tiers ?? []) { - if (inputTokens > tier.inputTokensAbove && tier.inputTokensAbove > matchedThreshold) { - rates = tier - matchedThreshold = tier.inputTokensAbove - } - } - - const longWrite = usage.cacheWrite1h ?? 0 - const shortWrite = usage.cacheWrite - longWrite - usage.cost.input = (rates.input / 1_000_000) * usage.input - usage.cost.output = (rates.output / 1_000_000) * usage.output - usage.cost.cacheRead = (rates.cacheRead / 1_000_000) * usage.cacheRead - usage.cost.cacheWrite = (rates.cacheWrite * shortWrite + rates.input * 2 * longWrite) / 1_000_000 - usage.cost.total = - usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite -} diff --git a/src/json-schema.ts b/src/json-schema.ts deleted file mode 100644 index a2ffc02..0000000 --- a/src/json-schema.ts +++ /dev/null @@ -1,382 +0,0 @@ -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value) -} - -function stringValue(value: unknown): string | undefined { - return typeof value === "string" ? value : undefined -} - -function booleanValue(value: unknown): boolean | undefined { - return typeof value === "boolean" ? value : undefined -} - -type JsonSchemaValue = boolean | Record - -const JSON_SCHEMA_TYPES = new Set([ - "array", - "boolean", - "integer", - "null", - "number", - "object", - "string", -]) - -const LEGACY_KINDS = new Set([ - "any", - "array", - "boolean", - "enum", - "integer", - "intersect", - "intersection", - "literal", - "never", - "null", - "nullable", - "number", - "object", - "optional", - "string", - "undefined", - "union", - "unknown", -]) - -const LEGACY_FIELDS = new Set([ - "element", - "kind", - "inner", - "optional", - "value", - "values", - "variants", - "wrapped", -]) - -const SCHEMA_MAP_FIELDS = new Set([ - "$defs", - "definitions", - "dependentSchemas", - "patternProperties", - "properties", -]) - -const SCHEMA_ARRAY_FIELDS = new Set(["allOf", "anyOf", "oneOf", "prefixItems"]) - -const SCHEMA_VALUE_FIELDS = new Set([ - "additionalItems", - "additionalProperties", - "contains", - "contentSchema", - "else", - "if", - "items", - "not", - "propertyNames", - "then", - "unevaluatedItems", - "unevaluatedProperties", -]) - -const SCHEMA_KEYWORDS = new Set([ - "$anchor", - "$comment", - "$defs", - "$dynamicAnchor", - "$dynamicRef", - "$id", - "$ref", - "$schema", - "$vocabulary", - "additionalItems", - "additionalProperties", - "allOf", - "anyOf", - "const", - "contains", - "contentEncoding", - "contentMediaType", - "contentSchema", - "default", - "definitions", - "dependentRequired", - "dependentSchemas", - "description", - "else", - "enum", - "examples", - "exclusiveMaximum", - "exclusiveMinimum", - "format", - "if", - "items", - "maxContains", - "maxItems", - "maxLength", - "maxProperties", - "maximum", - "minContains", - "minItems", - "minLength", - "minProperties", - "minimum", - "multipleOf", - "not", - "oneOf", - "pattern", - "patternProperties", - "prefixItems", - "properties", - "propertyNames", - "readOnly", - "required", - "title", - "type", - "unevaluatedItems", - "unevaluatedProperties", - "uniqueItems", - "writeOnly", -]) - -function stringArray(value: unknown): string[] | undefined { - if (!Array.isArray(value)) return undefined - const values = value.filter((item): item is string => typeof item === "string") - return values.length === value.length ? values : undefined -} - -function validSchemaType(value: unknown): boolean { - if (typeof value === "string") return JSON_SCHEMA_TYPES.has(value) - if (!Array.isArray(value) || value.length === 0) return false - return value.every((item) => typeof item === "string" && JSON_SCHEMA_TYPES.has(item)) -} - -function legacyKind(schema: Record): string | undefined { - const explicitKind = stringValue(schema.kind)?.toLowerCase() - if (explicitKind && LEGACY_KINDS.has(explicitKind)) return explicitKind - - const type = stringValue(schema.type) - const normalized = type?.toLowerCase() - if (!normalized || !LEGACY_KINDS.has(normalized)) return undefined - if (!validSchemaType(type) || Object.keys(schema).some((key) => LEGACY_FIELDS.has(key))) { - return normalized - } - return undefined -} - -function looksLikeJsonSchema(schema: Record): boolean { - if (Object.keys(schema).length === 0) return true - if (schema.type !== undefined && !validSchemaType(schema.type)) return false - return Object.keys(schema).some((key) => SCHEMA_KEYWORDS.has(key)) -} - -function isOptionalSchema(schema: unknown): boolean { - if (!isRecord(schema)) return false - if (booleanValue(schema.optional) === true) return true - - const kind = legacyKind(schema) - if (kind === "optional") return true - if (kind !== "union") return false - - const variants = Array.isArray(schema.variants) - ? schema.variants - : Array.isArray(schema.anyOf) - ? schema.anyOf - : [] - return variants.some((variant) => legacyKind(isRecord(variant) ? variant : {}) === "undefined") -} - -function schemaValue(value: unknown, seen: WeakSet): JsonSchemaValue { - if (typeof value === "boolean") return value - if (!isRecord(value)) return {} - return convertSchema(value, seen) -} - -function setSchemaProperty(target: Record, key: string, value: unknown): void { - Object.defineProperty(target, key, { - configurable: true, - enumerable: true, - value, - writable: true, - }) -} - -function schemaMap(value: unknown, seen: WeakSet): Record { - if (!isRecord(value)) return {} - const out: Record = {} - for (const [key, item] of Object.entries(value)) { - setSchemaProperty(out, key, schemaValue(item, seen)) - } - return out -} - -function schemaArray(value: unknown, seen: WeakSet): unknown[] { - if (!Array.isArray(value)) return [] - return value.map((item) => schemaValue(item, seen)) -} - -function isSchemaValue(value: unknown): value is JsonSchemaValue { - return typeof value === "boolean" || isRecord(value) -} - -function copySchemaObject( - source: Record, - seen: WeakSet, - legacy: boolean, - forcedType?: string, -): JsonSchemaValue { - const out: Record = {} - - for (const [key, value] of Object.entries(source)) { - if (legacy && LEGACY_FIELDS.has(key)) continue - if (key === "nullable" || (forcedType !== undefined && key === "type")) continue - - if (key === "required") { - const required = stringArray(value) - if (required) out.required = required - } else if (SCHEMA_MAP_FIELDS.has(key)) { - out[key] = schemaMap(value, seen) - } else if (SCHEMA_ARRAY_FIELDS.has(key)) { - out[key] = schemaArray(value, seen) - } else if (SCHEMA_VALUE_FIELDS.has(key)) { - out[key] = - Array.isArray(value) && key === "items" - ? schemaArray(value, seen) - : schemaValue(value, seen) - } else { - out[key] = value - } - } - - if (forcedType !== undefined) out.type = forcedType - if (booleanValue(source.nullable) === true) return makeNullable(out) - return out -} - -function makeNullable(schema: Record): Record { - const type = schema.type - if (typeof type === "string") { - if (type === "null") return schema - return { ...schema, type: [type, "null"] } - } - if (Array.isArray(type) && !type.includes("null")) { - return { ...schema, type: [...type, "null"] } - } - if (Array.isArray(schema.anyOf)) { - return { ...schema, anyOf: [...schema.anyOf, { type: "null" }] } - } - return { anyOf: [schema, { type: "null" }] } -} - -function legacyVariants(schema: Record): unknown[] { - if (Array.isArray(schema.variants)) return schema.variants - if (Array.isArray(schema.anyOf)) return schema.anyOf - return [] -} - -function convertLegacySchema( - source: Record, - kind: string, - seen: WeakSet, -): JsonSchemaValue { - if (kind === "optional") return schemaValue(source.wrapped ?? source.inner, seen) - if (kind === "nullable") { - const wrapped = schemaValue(source.wrapped ?? source.inner, seen) - return typeof wrapped === "boolean" ? wrapped : makeNullable(wrapped) - } - if (kind === "undefined" || kind === "never" || kind === "any" || kind === "unknown") return {} - - if (kind === "union" || kind === "intersect" || kind === "intersection") { - const variants = legacyVariants(source) - .map((variant) => schemaValue(variant, seen)) - .filter( - (variant) => - isSchemaValue(variant) && - (typeof variant === "boolean" || Object.keys(variant).length > 0), - ) - if (variants.length === 0) return copySchemaObject(source, seen, true) - if (variants.length === 1) return variants[0] ?? {} - - const out = copySchemaObject(source, seen, true) - if (typeof out !== "boolean") out[kind === "union" ? "anyOf" : "allOf"] = variants - return out - } - - if (kind === "object") { - const converted = copySchemaObject(source, seen, true, "object") - if (typeof converted === "boolean") return converted - const out = converted - const sourceProperties = isRecord(source.properties) ? source.properties : undefined - if (!sourceProperties) return out - - const properties: Record = {} - const optional = stringArray(source.optional) ?? [] - for (const [key, value] of Object.entries(sourceProperties)) { - setSchemaProperty(properties, key, schemaValue(value, seen)) - } - out.properties = properties - - const explicitRequired = stringArray(source.required) - const required = - explicitRequired ?? - Object.entries(sourceProperties) - .filter(([key, value]) => !optional.includes(key) && !isOptionalSchema(value)) - .map(([key]) => key) - if (required.length > 0) out.required = required - else delete out.required - return out - } - - if (kind === "array") { - const converted = copySchemaObject(source, seen, true, "array") - if (typeof converted === "boolean") return converted - const out = converted - if (!("items" in source) && "element" in source) out.items = schemaValue(source.element, seen) - return out - } - - if (kind === "enum") { - const converted = copySchemaObject(source, seen, true) - if (typeof converted === "boolean") return converted - const out = converted - if (!("enum" in out) && Array.isArray(source.values)) out.enum = source.values - return out - } - - if (kind === "literal") { - const converted = copySchemaObject(source, seen, true) - if (typeof converted === "boolean") return converted - const out = converted - if (!("const" in out) && "value" in source) out.const = source.value - return out - } - - const scalarType = - kind === "string" || - kind === "number" || - kind === "boolean" || - kind === "integer" || - kind === "null" - ? kind - : undefined - return scalarType ? copySchemaObject(source, seen, true, scalarType) : {} -} - -function convertSchema(source: Record, seen: WeakSet): JsonSchemaValue { - if (seen.has(source)) return {} - seen.add(source) - try { - const kind = legacyKind(source) - if (kind) return convertLegacySchema(source, kind, seen) - if (!looksLikeJsonSchema(source)) return {} - return copySchemaObject(source, seen, false) - } finally { - seen.delete(source) - } -} - -export function toJsonSchema(schema: unknown): unknown { - if (typeof schema === "boolean") return schema - if (!isRecord(schema)) return {} - return convertSchema(schema, new WeakSet()) -} diff --git a/src/models.ts b/src/models.ts index cd470ef..b417c46 100644 --- a/src/models.ts +++ b/src/models.ts @@ -1,7 +1,9 @@ import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises" import { dirname } from "node:path" +import type { Api } from "@earendil-works/pi-ai" -export const DEFAULT_MODELS_URL = "https://api.commandcode.ai/provider/v1/models" +export const DEFAULT_PROVIDER_API_BASE = "https://api.commandcode.ai/provider/v1" +export const DEFAULT_MODELS_URL = `${DEFAULT_PROVIDER_API_BASE}/models` export const DEFAULT_MODELS_TIMEOUT_MS = 10_000 const DEFAULT_MAX_OUTPUT_TOKENS = 65_536 @@ -159,11 +161,22 @@ interface ApiModel { export interface CommandCodeModel { id: string name: string + api: Api reasoning: boolean contextWindow: number maxTokens: number } +export function apiForModelId(id: string): Api { + return id.startsWith("claude-") ? "anthropic-messages" : "openai-completions" +} + +export function baseUrlForModel(apiBase: string, api: Api): string { + const normalized = apiBase.replace(/\/+$/g, "") + if (api !== "anthropic-messages") return normalized + return normalized.endsWith("/v1") ? normalized.slice(0, -3) : normalized +} + interface FetchCommandCodeModelsOptions { url?: string fetchImpl?: typeof fetch @@ -225,6 +238,7 @@ function parseCachedModel(value: unknown): CommandCodeModel { return { id, name: stringField(value, "name"), + api: apiForModelId(id), reasoning: isReasoningModel(id), contextWindow: positiveNumberField(value, "contextWindow"), maxTokens: positiveNumberField(value, "maxTokens"), @@ -329,6 +343,7 @@ export function commandCodeModelsFromApiResponse(value: unknown): readonly Comma return data.map(parseApiModel).map((model) => ({ id: model.id, name: `${model.name} (CC)`, + api: apiForModelId(model.id), reasoning: isReasoningModel(model.id), contextWindow: model.contextLength, maxTokens: Math.min(model.contextLength, DEFAULT_MAX_OUTPUT_TOKENS), diff --git a/src/overflow.ts b/src/overflow.ts deleted file mode 100644 index 3e3e106..0000000 --- a/src/overflow.ts +++ /dev/null @@ -1,120 +0,0 @@ -const COMMAND_CODE_PROVIDER = "commandcode" -const CONTEXT_OVERFLOW_PREFIX = "context_length_exceeded:" - -const COMMAND_CODE_OVERFLOW_PATTERNS = [ - /\b(?:context[_\s-]*(?:length|window)|model[_\s-]*context[_\s-]*window)[_\s-]*(?:exceeded|overflow(?:ed)?|too[_\s-]*(?:large|long))\b/i, - /\b(?:context|prompt|input)[_\s-]*(?:length|window|size|tokens?|limit|maximum)\b[\s\S]{0,120}\b(?:exceed(?:ed|s)?|overflow(?:ed|s)?|too\s+(?:large|long)|(?:maximum|limit)\s+(?:reached|exceeded|hit))\b/i, - /\b(?:exceed(?:ed|s)?|overflow(?:ed|s)?|too\s+(?:large|long))\b[\s\S]{0,120}\b(?:context|prompt|input)[_\s-]*(?:length|window|size|tokens?|limit|maximum)\b/i, - /\b(?:prompt|input|context)\b[\s\S]{0,32}\btoo\s+(?:large|long)\b/i, - /\b(?:prompt|input)[_\s-]*too[_\s-]*(?:large|long)\b/i, - /\b(?:prompt|input)[_\s-]*tokens?[_\s-]*(?:limit|maximum|max)[_\s-]*(?:exceeded|reached)\b/i, - /\b(?:prompt|input)[_\s-]*(?:tokens?|length|size)\b[\s\S]{0,120}\b(?:limit|maximum)\b[\s\S]{0,40}\b(?:exceed(?:ed|s)?|reached|hit)\b/i, - /\b(?:maximum|limit)[_\s-]+(?:allowed[_\s-]+)?(?:context|prompt|input)[_\s-]*(?:length|window|size|tokens?)\b/i, -] - -const NON_OVERFLOW_PATTERNS = [ - /\brate[_\s-]*limit\b/i, - /\btoo\s+many\s+requests\b/i, - /\b(?:capacity|quota|throttl(?:e|ed|ing)?|concurren(?:cy|t)|overloaded)\b/i, - /\b(?:service|temporarily)\s+unavailable\b/i, - /\bstatus(?:[_\s-]*code)?\s*[:=]\s*429\b/i, -] - -const CONTEXT_OVERFLOW_PREFIX_PATTERN = /context_length_exceeded/i - -const HTTP_RATE_LIMIT_STATUS_PATTERNS = [ - /\b(?:api\s+error|http|status(?:[_\s-]*code)?|status[_\s-]*code)\s*[:(]?\s*429\b/i, - /["']?(?:status|status[_\s-]*code)["']?\s*:\s*429\b/i, -] - -const BEARER_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]+/gi -const CREDENTIAL_PATTERN = - /\b(?:api[-_ ]?key|apikey|access[-_ ]?token|refresh[-_ ]?token|token|secret|password|authorization)\s*[=:]\s*[^\s,;)]+/gi -const USER_TOKEN_PATTERN = /\b(?:user|cc)_[A-Za-z0-9_-]{8,}\b/gi -const QUERY_SECRET_PATTERN = - /([?&](?:api[-_ ]?key|apikey|access_token|refresh_token|token|secret|password)=)[^&#\s]+/gi -const STANDALONE_SECRET_PATTERN = - /\b(?:sk|rk|ghp|github_pat|xox[baprs])[-_A-Za-z0-9]{16,}\b|\beyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g - -export function redactCommandCodeErrorText(value: string): string { - return value - .replace(BEARER_PATTERN, "Bearer [redacted]") - .replace(CREDENTIAL_PATTERN, (match) => { - const separatorIndex = match.search(/[=:]/) - return separatorIndex < 0 ? "[redacted]" : `${match.slice(0, separatorIndex + 1)}[redacted]` - }) - .replace(USER_TOKEN_PATTERN, "[redacted]") - .replace(QUERY_SECRET_PATTERN, "$1[redacted]") - .replace(STANDALONE_SECRET_PATTERN, "[redacted]") -} - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null -} - -export interface CommandCodeMessageLike { - role: string - provider: string - stopReason: string - errorMessage?: string -} - -export function commandCodeErrorMessage(value: unknown): string | undefined { - if (typeof value === "string") return value - if (!isRecord(value)) return undefined - - const record = value - const parts: string[] = [] - for (const key of [ - "message", - "errorMessage", - "error", - "detail", - "details", - "code", - "type", - "reason", - ]) { - const part = commandCodeErrorMessage(record[key]) - if (part && !parts.includes(part)) parts.push(part) - } - - for (const key of ["status", "statusCode", "httpStatus"]) { - const status = record[key] - if (typeof status === "string" || typeof status === "number") { - const statusPart = `status: ${status}` - if (!parts.includes(statusPart)) parts.push(statusPart) - } - } - - return parts.length > 0 ? redactCommandCodeErrorText(parts.join(": ")) : undefined -} - -export function normalizeCommandCodeErrorMessage( - errorMessage: string | undefined, -): string | undefined { - if (!errorMessage) return undefined - if (CONTEXT_OVERFLOW_PREFIX_PATTERN.test(errorMessage)) return undefined - if (NON_OVERFLOW_PATTERNS.some((pattern) => pattern.test(errorMessage))) return undefined - if (HTTP_RATE_LIMIT_STATUS_PATTERNS.some((pattern) => pattern.test(errorMessage))) - return undefined - if (!COMMAND_CODE_OVERFLOW_PATTERNS.some((pattern) => pattern.test(errorMessage))) - return undefined - - return `${CONTEXT_OVERFLOW_PREFIX} ${errorMessage}` -} - -export function normalizeCommandCodeMessage( - message: T, - modelProvider?: string, -): { message: T & { errorMessage: string } } | undefined { - if (message.role !== "assistant" || message.stopReason !== "error") return undefined - if (message.provider !== COMMAND_CODE_PROVIDER && modelProvider !== COMMAND_CODE_PROVIDER) { - return undefined - } - - const errorMessage = normalizeCommandCodeErrorMessage(message.errorMessage) - if (!errorMessage) return undefined - - return { message: { ...message, errorMessage } } -} diff --git a/src/types.ts b/src/types.ts deleted file mode 100644 index c520543..0000000 --- a/src/types.ts +++ /dev/null @@ -1,201 +0,0 @@ -export type StopReason = "stop" | "length" | "toolUse" -export type ErrorReason = "error" | "aborted" -export type TerminalReason = StopReason | ErrorReason - -export interface UsageCost { - input: number - output: number - cacheRead: number - cacheWrite: number - total: number -} - -export interface Usage { - input: number - output: number - cacheRead: number - cacheWrite: number - cacheWrite1h?: number - totalTokens: number - cost: UsageCost -} - -export interface TextContent { - type: "text" - text: string -} - -export interface ThinkingContent { - type: "thinking" - thinking: string -} - -export interface ToolCallContent { - type: "toolCall" - id: string - name: string - arguments: Record -} - -export type AssistantContent = TextContent | ThinkingContent | ToolCallContent - -export interface AssistantMessageLike { - role: "assistant" - content: AssistantContent[] - api: unknown - provider: string - model: string - usage: Usage - stopReason: TerminalReason - errorMessage?: string - timestamp: number -} - -export interface ModelCostRates { - input: number - output: number - cacheRead: number - cacheWrite: number -} - -export interface ModelCostTier extends ModelCostRates { - inputTokensAbove: number -} - -export interface ModelCost extends ModelCostRates { - tiers?: readonly ModelCostTier[] -} - -export interface ModelLike { - id: string - api: unknown - provider: string - maxTokens: number - cost: ModelCost - reasoning?: boolean - thinkingLevelMap?: Partial> - thinking?: { - mode?: "effort" - effortMap?: Partial> - efforts?: readonly string[] - } -} - -export interface MessageLike { - role: string - content?: unknown - toolCallId?: string - toolName?: string - isError?: boolean -} - -export interface ToolLike { - name: string - description?: string - parameters?: unknown -} - -export interface ContextLike { - systemPrompt?: string - messages?: readonly MessageLike[] - tools?: readonly ToolLike[] -} - -export interface ProviderResponseInfo { - status: number - headers: Record -} - -export interface StreamOptions { - apiKey?: string - signal?: AbortSignal - headers?: Record - maxTokens?: number - /** Resolved pi thinking level; forwarded only through the model's map. */ - reasoning?: string - onPayload?: (payload: unknown, model: ModelLike) => unknown | Promise - onResponse?: (response: ProviderResponseInfo, model: ModelLike) => void | Promise - /** - * HTTP request timeout in milliseconds. - * Applied per-attempt; on timeout the request is retried if retries remain. - */ - timeoutMs?: number - /** - * Maximum retry attempts for transient HTTP errors (429, 5xx). - * Default: 0 (pi agent-level retry handles visible retries when unset). - */ - maxRetries?: number - /** - * Maximum delay in milliseconds to wait for a retry when the server requests - * a long wait via Retry-After. If the server's requested delay exceeds this - * value, the request fails immediately. Default: 60000 (60 seconds). - * Set to 0 to disable the cap. - */ - maxRetryDelayMs?: number -} - -export type AssistantMessageEvent = - | { type: "start"; partial: AssistantMessageLike } - | { type: "text_start"; contentIndex: number; partial: AssistantMessageLike } - | { - type: "text_delta" - contentIndex: number - delta: string - partial: AssistantMessageLike - } - | { - type: "text_end" - contentIndex: number - content: string - partial: AssistantMessageLike - } - | { - type: "thinking_start" - contentIndex: number - partial: AssistantMessageLike - } - | { - type: "thinking_delta" - contentIndex: number - delta: string - partial: AssistantMessageLike - } - | { - type: "thinking_end" - contentIndex: number - content: string - partial: AssistantMessageLike - } - | { - type: "toolcall_start" - contentIndex: number - partial: AssistantMessageLike - } - | { - type: "toolcall_end" - contentIndex: number - toolCall: ToolCallContent - partial: AssistantMessageLike - } - | { type: "done"; reason: StopReason; message: AssistantMessageLike } - | { type: "error"; reason: ErrorReason; error: AssistantMessageLike } - -export interface AssistantMessageEventStreamLike extends AsyncIterable { - push(event: AssistantMessageEvent): void - end(): void -} - -export interface CoreDependencies { - createStream: () => AssistantMessageEventStreamLike - calculateCost: (model: ModelLike, usage: Usage) => void - apiBase?: string - fetchImpl?: typeof fetch - authPaths?: readonly string[] - env?: NodeJS.ProcessEnv - cwd?: () => string - now?: () => number - uuid?: () => string - homeDir?: () => string - /** Injectable delay for retry backoff. Defaults to setTimeout. */ - delay?: (ms: number, signal: AbortSignal) => Promise -} diff --git a/tests/helpers.ts b/tests/helpers.ts deleted file mode 100644 index f4b3f65..0000000 --- a/tests/helpers.ts +++ /dev/null @@ -1,289 +0,0 @@ -import { createServer, type IncomingHttpHeaders, type Server } from "node:http" - -import { - createStreamCommandCode, - type AssistantMessageEvent, - type AssistantMessageEventStreamLike, - type ContextLike, - type CoreDependencies, - type ModelLike, - type Usage, -} from "../src/core.ts" - -export function createTestEventStream(): AssistantMessageEventStreamLike { - const events: AssistantMessageEvent[] = [] - const waiters: Array<() => void> = [] - let ended = false - - const wake = () => { - const waiter = waiters.shift() - if (waiter) waiter() - } - - return { - push(event: AssistantMessageEvent) { - events.push(event) - wake() - }, - end() { - ended = true - while (waiters.length > 0) wake() - }, - [Symbol.asyncIterator]() { - let index = 0 - return { - async next(): Promise> { - while (index >= events.length && !ended) { - await new Promise((resolve) => waiters.push(resolve)) - } - if (index < events.length) { - const value = events[index] - index += 1 - return { done: false, value } - } - return { done: true, value: undefined } - }, - } - }, - } -} - -export async function collectEvents( - stream: AssistantMessageEventStreamLike, - timeoutMs = 2_000, -): Promise { - const events: AssistantMessageEvent[] = [] - - const collect = async () => { - for await (const event of stream) { - events.push(event) - if (event.type === "done" || event.type === "error") break - } - return events - } - - return await Promise.race([ - collect(), - new Promise((_, reject) => { - setTimeout( - () => reject(new Error(`Timed out collecting stream events after ${timeoutMs}ms`)), - timeoutMs, - ) - }), - ]) -} - -export function makeModel(overrides: Partial = {}): ModelLike { - return { - id: "deepseek/deepseek-v4-flash", - api: "commandcode-custom", - provider: "commandcode", - maxTokens: 384_000, - cost: { input: 0.14, output: 0.28, cacheRead: 0.0028, cacheWrite: 0 }, - ...overrides, - } -} - -export function makeContext(overrides: Partial = {}): ContextLike { - return { - systemPrompt: "You are a test assistant.", - messages: [{ role: "user", content: "hello" }], - tools: [], - ...overrides, - } -} - -export interface TestDepsResult { - streamCommandCode: ReturnType - calculatedUsages: Usage[] -} - -export function createTestDeps(overrides: Partial = {}): TestDepsResult { - const calculatedUsages: Usage[] = [] - const streamCommandCode = createStreamCommandCode({ - createStream: createTestEventStream, - calculateCost: (_model, usage) => { - calculatedUsages.push({ - ...usage, - cost: { ...usage.cost }, - }) - }, - env: {}, - authPaths: [], - now: () => new Date("2026-05-05T12:00:00Z").getTime(), - uuid: () => "00000000-0000-4000-8000-000000000000", - cwd: () => "/repo", - delay: async () => {}, - ...overrides, - }) - return { streamCommandCode, calculatedUsages } -} - -type SuccessPlan = { - type: "success" - status?: number - events?: string[] - chunks?: string[] - delays?: number[] - hangAfterLast?: boolean - /** Delay in ms before the server starts sending the response. */ - responseDelay?: number -} - -type ErrorPlan = { - type: "error" - status: number - body: string - headers?: Record -} - -export type ResponsePlan = SuccessPlan | ErrorPlan - -function headersToRecord(headers: IncomingHttpHeaders): Record { - const out: Record = {} - for (const [key, value] of Object.entries(headers)) { - if (typeof value === "string") out[key] = value - else if (Array.isArray(value)) out[key] = value.join(", ") - } - return out -} - -export interface MockCommandCodeServer { - baseUrl(): string - mockResponse(plan: ResponsePlan): void - mockResponseQueue(plans: ResponsePlan[]): void - reset(): void - close(): Promise - lastRequestBody(): unknown - lastRequestHeaders(): Record - requestCount(): number - responseClosedBeforeEnd(): boolean -} - -export async function startMockCommandCodeServer(): Promise { - let planQueue: ResponsePlan[] = [{ type: "success", events: [] }] - let lastBody: unknown - let lastHeaders: Record = {} - let requests = 0 - let closedBeforeEnd = false - let port = 0 - - const server: Server = createServer((req, res) => { - if (req.method !== "POST" || req.url !== "/alpha/generate") { - res.writeHead(404) - res.end("Not found") - return - } - - requests += 1 - lastHeaders = headersToRecord(req.headers) - let body = "" - req.on("data", (chunk: Buffer) => { - body += chunk.toString("utf-8") - }) - req.on("end", () => { - try { - const parsed: unknown = JSON.parse(body) - lastBody = parsed - } catch { - lastBody = undefined - } - - // Pop the first plan from the queue; keep the last one as fallback. - const plan = planQueue.length > 1 ? planQueue.shift()! : planQueue[0] - - if (plan.type === "error") { - const headers: Record = { "Content-Type": "text/plain", ...plan.headers } - res.writeHead(plan.status, headers) - res.end(plan.body) - return - } - - res.writeHead(plan.status ?? 200, { - "Content-Type": "text/plain; charset=utf-8", - "Transfer-Encoding": "chunked", - }) - - let ended = false - res.on("close", () => { - if (!ended) closedBeforeEnd = true - }) - - const chunks = plan.chunks ?? (plan.events ?? []).map((event) => `${event}\n`) - const delays = plan.delays ?? chunks.map(() => 0) - let index = 0 - - const sendNext = () => { - if (index >= chunks.length) { - if (!plan.hangAfterLast) { - ended = true - res.end() - } - return - } - - res.write(chunks[index]) - index += 1 - if (index < chunks.length) { - setTimeout(sendNext, delays[index] ?? 0) - } else if (!plan.hangAfterLast) { - ended = true - res.end() - } - } - - if (plan.responseDelay) { - setTimeout(sendNext, plan.responseDelay) - } else { - sendNext() - } - }) - }) - - await new Promise((resolve) => { - server.listen(0, () => { - const address = server.address() - if (typeof address === "object" && address) port = address.port - resolve() - }) - }) - - return { - baseUrl: () => `http://127.0.0.1:${port}`, - mockResponse(plan: ResponsePlan) { - planQueue = [plan] - }, - mockResponseQueue(plans: ResponsePlan[]) { - planQueue = [...plans] - }, - reset() { - planQueue = [{ type: "success", events: [] }] - lastBody = undefined - lastHeaders = {} - requests = 0 - closedBeforeEnd = false - }, - close() { - return new Promise((resolve) => server.close(() => resolve())) - }, - lastRequestBody: () => lastBody, - lastRequestHeaders: () => lastHeaders, - requestCount: () => requests, - responseClosedBeforeEnd: () => closedBeforeEnd, - } -} - -export function objectAt(value: unknown, path: readonly string[]): unknown { - let current = value - for (const key of path) { - if (Array.isArray(current)) { - const index = Number(key) - if (!Number.isInteger(index)) return undefined - current = current[index] - continue - } - if (typeof current !== "object" || current === null) return undefined - current = Object.getOwnPropertyDescriptor(current, key)?.value - } - return current -} diff --git a/tests/test-abort.ts b/tests/test-abort.ts deleted file mode 100644 index 8db0ad5..0000000 --- a/tests/test-abort.ts +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Abort tests against the real streamCommandCode core. - */ - -import assert from "node:assert/strict" -import { after, before, beforeEach, describe, it } from "node:test" - -import { - collectEvents, - createTestDeps, - makeContext, - makeModel, - startMockCommandCodeServer, - type MockCommandCodeServer, -} from "./helpers.ts" - -let server: MockCommandCodeServer - -before(async () => { - server = await startMockCommandCodeServer() -}) - -after(async () => { - await server.close() -}) - -beforeEach(() => { - server.reset() -}) - -describe("streamCommandCode — abort behavior", () => { - it("emits aborted error when signal is already aborted", async () => { - const controller = new AbortController() - controller.abort() - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: "mock-key", - signal: controller.signal, - }), - ) - - assert.deepEqual( - events.map((event) => event.type), - ["start", "error"], - ) - const error = events.at(-1) - assert.equal(error?.type, "error") - if (error?.type !== "error") throw new Error("expected error") - assert.equal(error.reason, "aborted") - assert.equal(error.error.stopReason, "aborted") - assert.equal(server.requestCount(), 0) - }) - - it("emits aborted error and cancels the response reader mid-stream", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "text-delta", text: "first" })], - hangAfterLast: true, - }) - const controller = new AbortController() - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const stream = streamCommandCode(makeModel(), makeContext(), { - apiKey: "mock-key", - signal: controller.signal, - }) - - setTimeout(() => controller.abort(), 50) - const events = await collectEvents(stream, 2_000) - - assert.ok( - events.some((event) => event.type === "text_delta"), - "stream should process data before abort", - ) - const error = events.at(-1) - assert.equal(error?.type, "error") - if (error?.type !== "error") throw new Error("expected error") - assert.equal(error.reason, "aborted") - assert.equal(error.error.errorMessage, "Request aborted") - await new Promise((resolve) => setTimeout(resolve, 50)) - assert.ok(server.responseClosedBeforeEnd(), "abort should close the hanging upstream response") - }) -}) diff --git a/tests/test-cost.ts b/tests/test-cost.ts deleted file mode 100644 index ae7f730..0000000 --- a/tests/test-cost.ts +++ /dev/null @@ -1,183 +0,0 @@ -/** - * Regression test for the local cost calculation. - * - * The provider ships its own cost function because Oh My Pi's legacy pi-ai - * shim does not export `calculateCost` (see issue #24). This test locks the - * local implementation to pi-ai's documented per-million-token arithmetic - * without installing another pi-ai runtime next to the extension. - */ - -import assert from "node:assert/strict" -import { describe, it } from "node:test" - -import { calculateCommandCodeCost } from "../src/cost.ts" -import type { Usage } from "../src/types.ts" - -interface CostRates { - input: number - output: number - cacheRead: number - cacheWrite: number -} - -interface CostTable extends CostRates { - tiers?: Array -} - -const COST_FIXTURES: Record = { - "zero-cost-model": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - "claude-sonnet-4-6": { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, - "deepseek/deepseek-v4-pro": { - input: 0.435, - output: 0.87, - cacheRead: 0.003625, - cacheWrite: 0, - }, - "Qwen/Qwen3.7-Max": { input: 2.5, output: 7.5, cacheRead: 0.5, cacheWrite: 3.13 }, - "Qwen/Qwen3.7-Flash": { - input: 0.03, - output: 0.13, - cacheRead: 0.006, - cacheWrite: 0.038, - tiers: [ - { inputTokensAbove: 32_000, input: 0.1, output: 0.4, cacheRead: 0.02, cacheWrite: 0.125 }, - { inputTokensAbove: 256_000, input: 0.2, output: 0.8, cacheRead: 0.04, cacheWrite: 0.25 }, - ], - }, - "gpt-5.5": { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 }, -} - -const USAGE_CASES = [ - { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - { input: 1, output: 1, cacheRead: 1, cacheWrite: 1 }, - { input: 812, output: 187, cacheRead: 52_000, cacheWrite: 3_100 }, - { input: 1_000_000, output: 65_536, cacheRead: 998_877, cacheWrite: 123_456 }, - { input: 7, output: 999_999_999, cacheRead: 0.5, cacheWrite: 42 }, -] - -function commandCodeModel(id: string, cost: CostTable) { - return { - id, - api: "commandcode-custom", - provider: "commandcode", - cost, - maxTokens: 65_536, - } -} - -function assertClose(actual: number, expected: number) { - assert.ok( - Math.abs(actual - expected) <= - Number.EPSILON * Math.max(1, Math.abs(actual), Math.abs(expected)), - `expected ${actual} to be close to ${expected}`, - ) -} - -function freshUsage(tokens: (typeof USAGE_CASES)[number]): Usage { - return { - ...tokens, - totalTokens: tokens.input + tokens.output + tokens.cacheRead + tokens.cacheWrite, - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, - } -} - -function expectedCost(cost: CostTable, tokens: (typeof USAGE_CASES)[number]): Usage["cost"] { - const inputTokens = tokens.input + tokens.cacheRead + tokens.cacheWrite - let rates: CostRates = cost - let matchedThreshold = -1 - for (const tier of cost.tiers ?? []) { - if (inputTokens > tier.inputTokensAbove && tier.inputTokensAbove > matchedThreshold) { - rates = tier - matchedThreshold = tier.inputTokensAbove - } - } - - const input = (rates.input / 1_000_000) * tokens.input - const output = (rates.output / 1_000_000) * tokens.output - const cacheRead = (rates.cacheRead / 1_000_000) * tokens.cacheRead - const cacheWrite = (rates.cacheWrite * tokens.cacheWrite) / 1_000_000 - return { - input, - output, - cacheRead, - cacheWrite, - total: input + output + cacheRead + cacheWrite, - } -} - -describe("calculateCommandCodeCost()", () => { - it("applies per-million-token rates to all cost fields", () => { - for (const [id, cost] of Object.entries(COST_FIXTURES)) { - const model = commandCodeModel(id, cost) - - for (const tokens of USAGE_CASES) { - const usage = freshUsage(tokens) - calculateCommandCodeCost(model, usage) - - assert.deepEqual( - usage.cost, - expectedCost(cost, tokens), - `${id} cost for tokens=${JSON.stringify(tokens)}`, - ) - } - } - }) - - it("applies the highest request-wide input tier above its threshold", () => { - const model = commandCodeModel("Qwen/Qwen3.7-Flash", COST_FIXTURES["Qwen/Qwen3.7-Flash"]) - - const atThreshold = freshUsage({ - input: 32_000, - output: 1_000, - cacheRead: 0, - cacheWrite: 0, - }) - calculateCommandCodeCost(model, atThreshold) - assertClose(atThreshold.cost.input, (0.03 * 32_000) / 1_000_000) - - const aboveFirstTier = freshUsage({ - input: 30_000, - output: 1_000, - cacheRead: 2_001, - cacheWrite: 0, - }) - calculateCommandCodeCost(model, aboveFirstTier) - assertClose(aboveFirstTier.cost.input, (0.1 * 30_000) / 1_000_000) - assertClose(aboveFirstTier.cost.cacheRead, (0.02 * 2_001) / 1_000_000) - - const aboveHighestTier = freshUsage({ - input: 100_000, - output: 1_000, - cacheRead: 156_001, - cacheWrite: 0, - }) - calculateCommandCodeCost(model, aboveHighestTier) - assertClose(aboveHighestTier.cost.input, (0.2 * 100_000) / 1_000_000) - assertClose(aboveHighestTier.cost.output, (0.8 * 1_000) / 1_000_000) - }) - - it("prices one-hour cache writes at twice the active input rate", () => { - const model = commandCodeModel("claude-sonnet-4-6", COST_FIXTURES["claude-sonnet-4-6"]) - const usage = freshUsage({ input: 0, output: 0, cacheRead: 0, cacheWrite: 1_000 }) - usage.cacheWrite1h = 400 - - calculateCommandCodeCost(model, usage) - - const expectedShortWrite = (3.75 * 600) / 1_000_000 - const expectedLongWrite = (3 * 2 * 400) / 1_000_000 - assertClose(usage.cost.cacheWrite, expectedShortWrite + expectedLongWrite) - }) - - it("writes the total as the sum of all cost components", () => { - const model = commandCodeModel("claude-sonnet-4-6", COST_FIXTURES["claude-sonnet-4-6"]) - const usage = freshUsage({ input: 1_000, output: 500, cacheRead: 10_000, cacheWrite: 2_000 }) - - calculateCommandCodeCost(model, usage) - - assert.equal( - usage.cost.total, - usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite, - ) - assert.ok(usage.cost.total > 0) - }) -}) diff --git a/tests/test-models.ts b/tests/test-models.ts index af96ebb..3c7e33c 100644 --- a/tests/test-models.ts +++ b/tests/test-models.ts @@ -5,6 +5,8 @@ import { join } from "node:path" import { describe, it } from "node:test" import { + apiForModelId, + baseUrlForModel, commandCodeModelsFromApiResponse, commandCodeModelsFromCache, DEFAULT_MODELS_TIMEOUT_MS, @@ -37,6 +39,7 @@ const EXPECTED_MODELS: readonly CommandCodeModel[] = [ { id: "Qwen/Qwen3.7-Max", name: "Qwen 3.7 Max (CC)", + api: "openai-completions", reasoning: false, contextWindow: 1_000_000, maxTokens: 65_536, @@ -84,6 +87,19 @@ describe("commandCodeModelsFromApiResponse()", () => { assert.deepEqual(commandCodeModelsFromApiResponse(API_RESPONSE), EXPECTED_MODELS) }) + it("routes Claude models to Anthropic Messages and all others to Chat Completions", () => { + assert.equal(apiForModelId("claude-sonnet-4-6"), "anthropic-messages") + assert.equal(apiForModelId("gpt-5.6-sol"), "openai-completions") + assert.equal( + baseUrlForModel("https://api.commandcode.ai/provider/v1/", "openai-completions"), + "https://api.commandcode.ai/provider/v1", + ) + assert.equal( + baseUrlForModel("https://api.commandcode.ai/provider/v1/", "anthropic-messages"), + "https://api.commandcode.ai/provider", + ) + }) + it("matches command-code@1.15.1 image input capabilities", () => { assert.deepEqual(inputModalitiesForModel("gpt-5.6-luna"), ["text", "image"]) assert.deepEqual(inputModalitiesForModel("meta/muse-spark-1.2"), ["text", "image"]) diff --git a/tests/test-omp-compat.mjs b/tests/test-omp-compat.mjs index af7c2bd..9daae39 100644 --- a/tests/test-omp-compat.mjs +++ b/tests/test-omp-compat.mjs @@ -77,7 +77,7 @@ const server = createServer((req, res) => { return } - if (req.method !== "POST" || req.url !== "/alpha/generate") { + if (req.method !== "POST" || req.url !== "/provider/v1/chat/completions") { res.writeHead(404) res.end("Not found") return @@ -103,14 +103,19 @@ const server = createServer((req, res) => { } res.writeHead(200, { - "Content-Type": "text/plain; charset=utf-8", + "Content-Type": "text/event-stream; charset=utf-8", "Transfer-Encoding": "chunked", }) - res.write(`${JSON.stringify({ type: "text-delta", text: "mock-omp-ok" })}\n`) res.write( - `${JSON.stringify({ type: "finish", finishReason: "stop", totalUsage: { inputTokens: 1, outputTokens: 1 } })}\n`, + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "mock-omp-ok" }, finish_reason: null }] })}\n\n`, ) - res.end() + res.write( + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n`, + ) + res.write( + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })}\n\n`, + ) + res.end("data: [DONE]\n\n") }) }) @@ -129,7 +134,7 @@ function runOmp(args, timeoutMs = 30_000) { USERPROFILE: tempHome, PI_CODING_AGENT_DIR: join(tempHome, ".omp", "agent"), COMMANDCODE_API_KEY: "mock-key", - COMMANDCODE_API_BASE: apiBase, + COMMANDCODE_API_BASE: `${apiBase}/provider/v1`, COMMANDCODE_MODELS_URL: `${apiBase}/provider/v1/models`, }, stdio: ["ignore", "pipe", "pipe"], @@ -185,8 +190,8 @@ try { "Bearer mock-key", "should send the resolved env-var value, not the literal var name", ) - assert.equal(lastRequestBody?.params?.model, TEST_MODEL) - assert.equal(typeof lastRequestBody?.params?.system, "string") + assert.equal(lastRequestBody?.model, TEST_MODEL) + assert.ok(Array.isArray(lastRequestBody?.messages)) console.log("[omp-compat] PASS") } finally { diff --git a/tests/test-overflow.ts b/tests/test-overflow.ts deleted file mode 100644 index dc28888..0000000 --- a/tests/test-overflow.ts +++ /dev/null @@ -1,196 +0,0 @@ -import assert from "node:assert/strict" -import { after, before, beforeEach, describe, it } from "node:test" - -import { - commandCodeErrorMessage, - normalizeCommandCodeErrorMessage, - normalizeCommandCodeMessage, -} from "../src/overflow.ts" -import { - collectEvents, - createTestDeps, - makeContext, - makeModel, - startMockCommandCodeServer, - type MockCommandCodeServer, -} from "./helpers.ts" - -let server: MockCommandCodeServer - -before(async () => { - server = await startMockCommandCodeServer() -}) - -after(async () => { - await server.close() -}) - -beforeEach(() => { - server.reset() -}) - -describe("Command Code overflow normalization", () => { - it("normalizes Command Code context errors to pi's generic overflow marker", () => { - const normalized = normalizeCommandCodeErrorMessage("Prompt token limit exceeded") - - assert.equal(normalized, "context_length_exceeded: Prompt token limit exceeded") - }) - - it("is idempotent and leaves unrelated, rate-limit, and capacity errors unchanged", () => { - assert.equal( - normalizeCommandCodeErrorMessage("context_length_exceeded: Prompt token limit exceeded"), - undefined, - ) - assert.equal(normalizeCommandCodeErrorMessage("OpenAI request failed"), undefined) - assert.equal( - normalizeCommandCodeErrorMessage("Prompt token limit exceeded due to rate limit"), - undefined, - ) - assert.equal( - normalizeCommandCodeErrorMessage("Command Code API error 429: context window exceeded"), - undefined, - ) - assert.equal( - normalizeCommandCodeErrorMessage("context window exceeded: status: 429"), - undefined, - ) - assert.equal( - normalizeCommandCodeErrorMessage("The input is too long"), - "context_length_exceeded: The input is too long", - ) - assert.equal( - normalizeCommandCodeErrorMessage("Input exceeds context limit"), - "context_length_exceeded: Input exceeds context limit", - ) - assert.equal( - normalizeCommandCodeErrorMessage("Context window exceeded: provider capacity reached"), - undefined, - ) - }) - - it("scopes finalized message normalization to Command Code", () => { - const message = { - role: "assistant" as const, - provider: "commandcode", - stopReason: "error" as const, - errorMessage: "model context window exceeded", - } - - const normalized = normalizeCommandCodeMessage(message) - assert.equal( - normalized?.message.errorMessage, - "context_length_exceeded: model context window exceeded", - ) - assert.equal(normalizeCommandCodeMessage({ ...message, provider: "openai" }), undefined) - assert.equal(normalizeCommandCodeMessage({ ...message, stopReason: "stop" }), undefined) - }) - - it("extracts nested stream error messages without exposing credentials", () => { - assert.equal( - commandCodeErrorMessage({ - error: { details: { errorMessage: "context window exceeded" } }, - }), - "context window exceeded", - ) - }) - - it("redacts secrets from finalized provider errors", async () => { - server.mockResponse({ - type: "error", - status: 400, - body: "api_key=user_secret_value", - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - const error = events.at(-1) - assert.equal(error?.type, "error") - if (error?.type !== "error") throw new Error("expected error") - assert.doesNotMatch(error.error.errorMessage ?? "", /user_secret_value/) - assert.match(error.error.errorMessage ?? "", /api_key=\[redacted\]/) - }) - - it("normalizes HTTP error bodies containing nested context errors", async () => { - server.mockResponse({ - type: "error", - status: 400, - body: JSON.stringify({ error: { message: "Prompt token limit exceeded" } }), - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - const error = events.at(-1) - - assert.equal(error?.type, "error") - if (error?.type !== "error") throw new Error("expected error") - const normalized = normalizeCommandCodeMessage(error.error) - assert.match(normalized?.message.errorMessage ?? "", /^context_length_exceeded:/) - }) - - it("does not normalize an HTTP rate-limit response that mentions context", async () => { - server.mockResponse({ - type: "error", - status: 429, - body: JSON.stringify({ error: { message: "context window exceeded" } }), - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - const error = events.at(-1) - - assert.equal(error?.type, "error") - if (error?.type !== "error") throw new Error("expected error") - assert.equal(normalizeCommandCodeMessage(error.error), undefined) - }) - - it("normalizes nested stream error events", async () => { - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ - type: "error", - error: { details: { message: "model context window exceeded" } }, - }), - ], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - const error = events.at(-1) - - assert.equal(error?.type, "error") - if (error?.type !== "error") throw new Error("expected error") - const normalized = normalizeCommandCodeMessage(error.error) - assert.equal( - normalized?.message.errorMessage, - "context_length_exceeded: model context window exceeded", - ) - - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ - type: "error", - error: { message: "context window exceeded", status: 429 }, - }), - ], - }) - const retryEvents = await collectEvents( - createTestDeps({ apiBase: server.baseUrl() }).streamCommandCode(makeModel(), makeContext(), { - apiKey: "mock-key", - }), - ) - const retryError = retryEvents.at(-1) - assert.equal(retryError?.type, "error") - if (retryError?.type !== "error") throw new Error("expected error") - assert.equal(normalizeCommandCodeMessage(retryError.error), undefined) - }) -}) diff --git a/tests/test-pi-local.mjs b/tests/test-pi-local.mjs index 5099d08..760a100 100644 --- a/tests/test-pi-local.mjs +++ b/tests/test-pi-local.mjs @@ -15,7 +15,8 @@ import { fileURLToPath } from "node:url" const __dirname = dirname(fileURLToPath(import.meta.url)) const PROJECT_DIR = resolve(__dirname, "..") const EXT_PATH = resolve(PROJECT_DIR, "index.ts") -const TEST_MODEL = "deepseek/deepseek-v4-flash" +const TEST_MODEL = "gpt-5.4" +const CLAUDE_TEST_MODEL = "claude-sonnet-4-6" function findPiBinary() { if (process.env.PI_BIN) return process.env.PI_BIN @@ -63,9 +64,17 @@ function modelCatalog() { object: "model", created: 1779824324, owned_by: "command-code", - name: "DeepSeek V4 Flash", + name: "GPT 5.4", context_length: 1_000_000, }, + { + id: CLAUDE_TEST_MODEL, + object: "model", + created: 1779824324, + owned_by: "command-code", + name: "Claude Sonnet 4.6", + context_length: 200_000, + }, { id: "cc-second-model", object: "model", @@ -101,7 +110,9 @@ const server = createServer((req, res) => { return } - if (req.method !== "POST" || req.url !== "/alpha/generate") { + const isOpenAIRequest = req.method === "POST" && req.url === "/provider/v1/chat/completions" + const isAnthropicRequest = req.method === "POST" && req.url === "/provider/v1/messages" + if (!isOpenAIRequest && !isAnthropicRequest) { res.writeHead(404) res.end("Not found") return @@ -129,12 +140,20 @@ const server = createServer((req, res) => { if (overflowMode && overflowRequestCount === 2) { res.writeHead(400, { "Content-Type": "application/json; charset=utf-8" }) - res.end(JSON.stringify({ error: { message: "Input exceeds context limit" } })) + res.end( + JSON.stringify({ + error: { + message: "Input exceeds context limit", + type: "invalid_request_error", + code: "context_length_exceeded", + }, + }), + ) return } res.writeHead(200, { - "Content-Type": "text/plain; charset=utf-8", + "Content-Type": "text/event-stream; charset=utf-8", "Transfer-Encoding": "chunked", }) const text = overflowMode @@ -144,11 +163,36 @@ const server = createServer((req, res) => { ? "compaction-summary" : "overflow-recovered" : "mock-pi-ok" - res.write(`${JSON.stringify({ type: "text-delta", text })}\n`) + if (isAnthropicRequest) { + res.write( + `event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "mock", type: "message", role: "assistant", content: [], model: CLAUDE_TEST_MODEL, stop_reason: null, stop_sequence: null, usage: { input_tokens: 1, output_tokens: 0 } } })}\n\n`, + ) + res.write( + `event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } })}\n\n`, + ) + res.write( + `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text } })}\n\n`, + ) + res.write( + `event: content_block_stop\ndata: ${JSON.stringify({ type: "content_block_stop", index: 0 })}\n\n`, + ) + res.write( + `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 1 } })}\n\n`, + ) + res.end(`event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`) + return + } + res.write( - `${JSON.stringify({ type: "finish", finishReason: "stop", totalUsage: { inputTokens: 1, outputTokens: 1 } })}\n`, + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: text }, finish_reason: null }] })}\n\n`, ) - res.end() + res.write( + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n`, + ) + res.write( + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })}\n\n`, + ) + res.end("data: [DONE]\n\n") }) }) @@ -170,8 +214,9 @@ const env = { USERPROFILE: tempHome, PI_CODING_AGENT_DIR: agentDir, PI_CODING_AGENT_SESSION_DIR: join(tempHome, "sessions"), - COMMANDCODE_API_BASE: apiBase, + COMMANDCODE_API_BASE: `${apiBase}/provider/v1`, COMMANDCODE_API_KEY: "mock-key", + COMMANDCODE_ZDR: "1", COMMANDCODE_MODELS_URL: `${apiBase}/provider/v1/models`, } @@ -403,7 +448,7 @@ async function runRpcExtensionCommands(timeoutMs = 30_000) { event.type === "extension_ui_request" && event.method === "notify" && typeof event.message === "string" && - event.message.includes("model count: 2"), + event.message.includes("model count: 3"), ) includeRefreshedModel = true @@ -414,7 +459,7 @@ async function runRpcExtensionCommands(timeoutMs = 30_000) { event.type === "extension_ui_request" && event.method === "notify" && typeof event.message === "string" && - event.message.includes("3 models from live"), + event.message.includes("4 models from live"), ) send({ id: "status-after", type: "prompt", message: "/commandcode-status" }) @@ -426,7 +471,7 @@ async function runRpcExtensionCommands(timeoutMs = 30_000) { event.type === "extension_ui_request" && event.method === "notify" && typeof event.message === "string" && - event.message.includes("model count: 3"), + event.message.includes("model count: 4"), ) return { @@ -565,7 +610,7 @@ try { ) assert.equal(recoveryList.code, 0, recoveryList.stderr) const recoveryOutput = recoveryList.stdout || recoveryList.stderr - assert.match(recoveryOutput, /deepseek\/deepseek-v4-flash/) + assert.match(recoveryOutput, /gpt-5\.4/) assert.match(recoveryOutput, /cc-second-model/) assert.doesNotMatch(recoveryList.stderr, /no valid cached catalog/) assert.doesNotMatch(recoveryList.stderr, /Failed to load extension/) @@ -578,7 +623,7 @@ try { assert.equal(list.code, 0, list.stderr) const listOutput = list.stdout || list.stderr assert.match(listOutput, /commandcode/) - assert.match(listOutput, /deepseek\/deepseek-v4-flash/) + assert.match(listOutput, /gpt-5\.4/) assert.match(listOutput, /cc-second-model/) assert.equal(modelListRequestCount, 1) assert.doesNotThrow(() => accessSync(modelsCachePath, constants.R_OK)) @@ -591,7 +636,7 @@ try { ) assert.equal(offlineList.code, 0, offlineList.stderr) const offlineListOutput = offlineList.stdout || offlineList.stderr - assert.match(offlineListOutput, /deepseek\/deepseek-v4-flash/) + assert.match(offlineListOutput, /gpt-5\.4/) assert.match(offlineListOutput, /cc-second-model/) assert.match(offlineList.stderr, /Using the cached catalog/) @@ -659,27 +704,51 @@ try { lastRequestHeaders.authorization.startsWith("Bearer "), "should send a bearer Authorization header", ) - assert.equal(lastRequestBody?.params?.model, TEST_MODEL) - assert.equal(lastRequestBody?.params?.reasoning_effort, "high") - const sentTools = lastRequestBody?.params?.tools + assert.equal(lastRequestHeaders["x-cmd-zdr"], "1") + assert.equal(lastRequestBody?.model, TEST_MODEL) + assert.equal(lastRequestBody?.reasoning_effort, "high") + const sentTools = lastRequestBody?.tools assert.ok(Array.isArray(sentTools) && sentTools.length > 0) - const editTool = sentTools.find((tool) => tool.name === "edit") - assert.equal(editTool?.input_schema?.type, "object") - assert.equal(editTool?.input_schema?.properties?.edits?.type, "array") - assert.equal(editTool?.input_schema?.properties?.edits?.items?.type, "object") + const editTool = sentTools.find((tool) => tool.function?.name === "edit") + assert.equal(editTool?.function?.parameters?.type, "object") + assert.equal(editTool?.function?.parameters?.properties?.edits?.type, "array") + assert.equal(editTool?.function?.parameters?.properties?.edits?.items?.type, "object") assert.equal( - editTool?.input_schema?.properties?.edits?.items?.properties?.oldText?.type, + editTool?.function?.parameters?.properties?.edits?.items?.properties?.oldText?.type, "string", ) + console.log("[pi-local] Claude request through Anthropic Messages endpoint") + requestCount = 0 + const claudePrint = await runPi( + [ + "--no-extensions", + "-e", + EXT_PATH, + "-p", + "say mock token", + "--provider", + "commandcode", + "--model", + CLAUDE_TEST_MODEL, + ], + 30_000, + ) + assert.equal(claudePrint.code, 0, claudePrint.stderr) + assert.match(claudePrint.stdout, /mock-pi-ok/) + assert.equal(requestCount, 1) + assert.equal(lastRequestBody?.model, CLAUDE_TEST_MODEL) + assert.equal(lastRequestHeaders["x-api-key"], "mock-key") + assert.equal(lastRequestHeaders["x-cmd-zdr"], "1") + console.log("[pi-local] runtime commands through real RPC extension lifecycle") includeRefreshedModel = false const runtimeCommands = await runRpcExtensionCommands() assert.ok(runtimeCommands.commandNames.includes("commandcode-refresh")) assert.ok(runtimeCommands.commandNames.includes("commandcode-status")) assert.match(runtimeCommands.statusBefore, /source: live/) - assert.match(runtimeCommands.refreshNotification, /3 models from live/) - assert.match(runtimeCommands.statusAfter, /model count: 3/) + assert.match(runtimeCommands.refreshNotification, /4 models from live/) + assert.match(runtimeCommands.statusAfter, /model count: 4/) assert.doesNotMatch( `${runtimeCommands.statusBefore}\n${runtimeCommands.statusAfter}\n${runtimeCommands.stderr}`, /mock-key/, @@ -702,7 +771,7 @@ try { assert.equal(rpc.sawTextDelta, true) assert.equal(requestCount, 1) - console.log("[pi-local] reject image input through real RPC preflight/provider path") + console.log("[pi-local] forward image input through the documented provider schema") requestCount = 0 const imageRpc = await runRpcQuery(10_000, "describe image", [], { images: [ @@ -713,14 +782,15 @@ try { }, ], }) - assert.equal(requestCount, 0) + assert.equal(imageRpc.ok, true, imageRpc.stderr) + assert.equal(requestCount, 1) + const imageContent = lastRequestBody?.messages?.find( + (message) => message.role === "user", + )?.content + assert.ok(Array.isArray(imageContent), JSON.stringify(lastRequestBody?.messages)) assert.ok( - imageRpc.events.some( - (event) => - event.type === "message_end" && - event.message?.role === "assistant" && - event.message?.stopReason === "error", - ) || imageRpc.stderr.includes("does not support image"), + imageContent.some((part) => part.type === "image_url"), + JSON.stringify(imageContent), ) console.log("[pi-local] verify overflow normalization and compaction recovery") @@ -729,8 +799,7 @@ try { const overflowRpc = await runRpcOverflowRecovery() assert.equal(overflowRpc.ok, true) assert.ok(overflowRpc.requests >= 4) - assert.equal(overflowRpc.sawNormalizedOverflow, true) - assert.equal(overflowRpc.sawCompactionRetry, true) + assert.equal(overflowRpc.sawCompactionRetry, true, JSON.stringify(overflowRpc)) assert.equal(overflowRpc.stderrHasSecrets, false) overflowMode = false diff --git a/tests/test-pure-functions.ts b/tests/test-pure-functions.ts deleted file mode 100644 index 136025f..0000000 --- a/tests/test-pure-functions.ts +++ /dev/null @@ -1,658 +0,0 @@ -/** - * Unit tests for the real pure helpers exported by src/core.ts. - * These are hermetic: no pi runtime and no network. - */ - -import assert from "node:assert/strict" -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" -import { tmpdir } from "node:os" -import { join } from "node:path" -import { describe, it } from "node:test" - -import { - assertTextOnlyMessages, - getApiKey, - getEnvironmentInfo, - mapFinishReason, - messagesToCC, - parseStreamEventLine, - projectSlugFromPath, - textContent, - toJsonSchema, - toolsToJson, -} from "../src/core.ts" -import { redactCommandCodeErrorText } from "../src/overflow.ts" - -import { objectAt } from "./helpers.ts" - -describe("getApiKey()", () => { - it("uses COMMANDCODE_API_KEY from provided env", () => { - assert.equal(getApiKey({ env: { COMMANDCODE_API_KEY: "env-key" }, authPaths: [] }), "env-key") - }) - - it("reads apiKey, commandcode, pi OAuth, and official CLI credential fields", () => { - const dir = mkdtempSync(join(tmpdir(), "cc-auth-")) - try { - const first = join(dir, "first.json") - const second = join(dir, "second.json") - const oauth = join(dir, "oauth.json") - const official = join(dir, "official.json") - writeFileSync(first, JSON.stringify({ apiKey: "file-key" })) - writeFileSync(second, JSON.stringify({ commandcode: "fallback-key" })) - writeFileSync( - oauth, - JSON.stringify({ - commandcode: { - type: "oauth", - access: "oauth-access-key", - refresh: "oauth-refresh-key", - expires: Date.now() + 3600000, - }, - }), - ) - writeFileSync( - official, - JSON.stringify({ - "command-code": { - type: "api", - key: "official-cli-key", - }, - }), - ) - assert.equal(getApiKey({ env: {}, authPaths: [first, second] }), "file-key") - assert.equal(getApiKey({ env: {}, authPaths: [second] }), "fallback-key") - assert.equal(getApiKey({ env: {}, authPaths: [oauth] }), "oauth-access-key") - assert.equal(getApiKey({ env: {}, authPaths: [official] }), "official-cli-key") - } finally { - rmSync(dir, { recursive: true, force: true }) - } - }) - - it("ignores malformed auth files", () => { - const dir = mkdtempSync(join(tmpdir(), "cc-auth-bad-")) - try { - const bad = join(dir, "bad.json") - writeFileSync(bad, "not json") - assert.equal(getApiKey({ env: {}, authPaths: [bad] }), undefined) - } finally { - rmSync(dir, { recursive: true, force: true }) - } - }) - - it("uses injected homeDir for default auth paths", () => { - const dir = mkdtempSync(join(tmpdir(), "cc-home-")) - try { - const authDir = join(dir, ".pi", "agent") - mkdirSync(authDir, { recursive: true }) - writeFileSync(join(authDir, "auth.json"), JSON.stringify({ commandcode: "pi-key" })) - assert.equal(getApiKey({ env: {}, homeDir: () => dir }), "pi-key") - } finally { - rmSync(dir, { recursive: true, force: true }) - } - }) -}) - -describe("error redaction", () => { - it("redacts bearer, credential, and query-string secrets", () => { - const redacted = redactCommandCodeErrorText( - "Bearer user_secret_value api_key=user_secret_value https://example.test/x?token=user_secret_value", - ) - assert.doesNotMatch(redacted, /user_secret_value/) - assert.match(redacted, /Bearer \[redacted\]/) - assert.doesNotMatch( - redactCommandCodeErrorText("provider returned sk-test-secret-value-1234567890"), - /sk-test-secret-value/, - ) - }) -}) - -describe("projectSlugFromPath()", () => { - it("matches the official CLI-style slug from an absolute working directory", () => { - assert.equal( - projectSlugFromPath("/Users/patwoz/dev/Personal/pi/pi-commandcode-provider"), - "users-patwoz-dev-personal-pi-pi-commandcode-provider", - ) - assert.equal(projectSlugFromPath("/repo"), "repo") - }) -}) - -describe("text-only image handling", () => { - it("rejects image content for models without image support", () => { - assert.throws( - () => - assertTextOnlyMessages([ - { - role: "user", - content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], - }, - ]), - /does not support image content/i, - ) - assert.throws( - () => - assertTextOnlyMessages([ - { - role: "toolResult", - toolCallId: "c1", - content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], - }, - ]), - /does not support image content/i, - ) - }) -}) - -describe("textContent()", () => { - it("extracts and joins text blocks", () => { - assert.equal( - textContent({ - content: [ - { type: "text", text: "hello" }, - { type: "text", text: "world" }, - ], - }), - "hello\nworld", - ) - }) - - it("extracts text while images are handled separately", () => { - assert.equal( - textContent({ - content: [ - { type: "text", text: "hello" }, - { type: "image", data: "x", mimeType: "image/png" }, - { type: "text", text: "world" }, - ], - }), - "hello\nworld", - ) - }) - - it("handles empty or missing content", () => { - assert.equal(textContent({ content: [] }), "") - assert.equal(textContent({}), "") - }) -}) - -describe("getEnvironmentInfo()", () => { - it("returns platform, arch, and Node version", () => { - const info = getEnvironmentInfo() - assert.match(info, /^(darwin|linux|win32)-/) - assert.ok(info.includes("Node.js")) - }) -}) - -describe("toJsonSchema()", () => { - it("converts scalar, enum, object, optional, array, and union schema shapes", () => { - assert.deepEqual(toJsonSchema({ kind: "string" }), { type: "string" }) - assert.deepEqual(toJsonSchema({ kind: "Number" }), { type: "number" }) - assert.deepEqual(toJsonSchema({ kind: "boolean" }), { type: "boolean" }) - assert.deepEqual(toJsonSchema({ kind: "string", enum: ["left", "right"] }), { - type: "string", - enum: ["left", "right"], - }) - assert.deepEqual( - toJsonSchema({ - kind: "object", - properties: { - name: { kind: "string" }, - tags: { kind: "array", items: { kind: "string" }, optional: true }, - }, - }), - { - type: "object", - properties: { - name: { type: "string" }, - tags: { type: "array", items: { type: "string" } }, - }, - required: ["name"], - }, - ) - assert.deepEqual(toJsonSchema({ kind: "optional", wrapped: { kind: "string" } }), { - type: "string", - }) - assert.deepEqual(toJsonSchema({ kind: "union", variants: [{}, { kind: "number" }] }), { - type: "number", - }) - }) - - it("preserves explicit required arrays and handles unknown values", () => { - assert.deepEqual( - toJsonSchema({ - type: "object", - properties: { name: { type: "string" }, nickname: { type: "string" } }, - required: ["name"], - }), - { - type: "object", - properties: { name: { type: "string" }, nickname: { type: "string" } }, - required: ["name"], - }, - ) - assert.deepEqual(toJsonSchema(undefined), {}) - assert.deepEqual(toJsonSchema({ kind: "wat" }), {}) - assert.deepEqual(toJsonSchema({ type: "wat", description: "not a schema" }), {}) - assert.deepEqual(toJsonSchema({}), {}) - assert.equal(toJsonSchema(true), true) - }) - - it("preserves complete JSON Schema metadata and nested schemas", () => { - assert.deepEqual( - toJsonSchema({ - type: "object", - description: "Search options", - properties: { - query: { - type: "string", - description: "Text to search for", - minLength: 2, - maxLength: 50, - pattern: "^[a-z]+$", - default: "pi", - }, - limit: { - type: "integer", - minimum: 1, - maximum: 100, - exclusiveMinimum: 0, - multipleOf: 1, - default: 10, - }, - tags: { - type: "array", - minItems: 1, - maxItems: 3, - uniqueItems: true, - items: { - type: "object", - properties: { name: { type: "string" } }, - required: ["name"], - additionalProperties: false, - }, - }, - }, - required: ["query", "limit"], - additionalProperties: false, - }), - { - type: "object", - description: "Search options", - properties: { - query: { - type: "string", - description: "Text to search for", - minLength: 2, - maxLength: 50, - pattern: "^[a-z]+$", - default: "pi", - }, - limit: { - type: "integer", - minimum: 1, - maximum: 100, - exclusiveMinimum: 0, - multipleOf: 1, - default: 10, - }, - tags: { - type: "array", - minItems: 1, - maxItems: 3, - uniqueItems: true, - items: { - type: "object", - properties: { name: { type: "string" } }, - required: ["name"], - additionalProperties: false, - }, - }, - }, - required: ["query", "limit"], - additionalProperties: false, - }, - ) - }) - - it("preserves JSON Schema composition and nullable forms", () => { - assert.deepEqual( - toJsonSchema({ - anyOf: [{ type: "string" }, { type: "number" }], - oneOf: [{ const: "a" }, { const: "b" }], - allOf: [{ minLength: 1 }, { maxLength: 10 }], - nullable: true, - }), - { - anyOf: [{ type: "string" }, { type: "number" }, { type: "null" }], - oneOf: [{ const: "a" }, { const: "b" }], - allOf: [{ minLength: 1 }, { maxLength: 10 }], - }, - ) - assert.deepEqual(toJsonSchema({ type: ["string", "null"] }), { - type: ["string", "null"], - }) - assert.deepEqual(toJsonSchema({ type: "string", nullable: true }), { - type: ["string", "null"], - }) - }) - - it("preserves dangerous schema property names", () => { - const inputProperties: Record = { - constructor: { type: "number" }, - } - Object.defineProperty(inputProperties, "__proto__", { - configurable: true, - enumerable: true, - value: { type: "string" }, - writable: true, - }) - const schema = toJsonSchema({ - type: "object", - properties: inputProperties, - required: ["__proto__", "constructor"], - }) - assert.ok(schema && typeof schema === "object" && !Array.isArray(schema)) - if (!schema || typeof schema !== "object" || Array.isArray(schema)) { - throw new Error("expected object schema") - } - const outputProperties: unknown = Object.getOwnPropertyDescriptor(schema, "properties")?.value - assert.ok(outputProperties && typeof outputProperties === "object") - if (!outputProperties || typeof outputProperties !== "object") { - throw new Error("expected object properties") - } - assert.ok(Object.prototype.hasOwnProperty.call(outputProperties, "__proto__")) - assert.deepEqual(Object.getOwnPropertyDescriptor(outputProperties, "__proto__")?.value, { - type: "string", - }) - assert.deepEqual(Object.getOwnPropertyDescriptor(outputProperties, "constructor")?.value, { - type: "number", - }) - }) - - it("converts legacy shapes without collapsing unions", () => { - assert.deepEqual( - toJsonSchema({ - kind: "Object", - description: "Legacy options", - properties: { - mode: { - kind: "union", - variants: [ - { kind: "string", enum: ["fast", "safe"] }, - { kind: "string", enum: ["debug"] }, - ], - }, - count: { kind: "Number", minimum: 1, optional: true }, - nested: { - kind: "Array", - element: { kind: "object", properties: { value: { kind: "boolean" } } }, - }, - }, - optional: ["count"], - additionalProperties: false, - }), - { - type: "object", - description: "Legacy options", - properties: { - mode: { - anyOf: [ - { type: "string", enum: ["fast", "safe"] }, - { type: "string", enum: ["debug"] }, - ], - }, - count: { type: "number", minimum: 1 }, - nested: { - type: "array", - items: { - type: "object", - properties: { value: { type: "boolean" } }, - required: ["value"], - }, - }, - }, - required: ["mode", "nested"], - additionalProperties: false, - }, - ) - assert.deepEqual( - toJsonSchema({ - kind: "intersect", - variants: [{ kind: "object", properties: { a: { kind: "string" } } }, { kind: "number" }], - }), - { - allOf: [ - { type: "object", properties: { a: { type: "string" } }, required: ["a"] }, - { type: "number" }, - ], - }, - ) - }) -}) - -describe("toolsToJson()", () => { - it("converts pi tools to Command Code tool JSON", () => { - assert.deepEqual( - toolsToJson([ - { - name: "get_weather", - description: "Get weather", - parameters: { - kind: "object", - properties: { city: { kind: "string" } }, - }, - }, - ]), - [ - { - type: "function", - name: "get_weather", - description: "Get weather", - input_schema: { - type: "object", - properties: { city: { type: "string" } }, - required: ["city"], - }, - }, - ], - ) - }) - - it("returns an empty array for missing tools", () => { - assert.deepEqual(toolsToJson(), []) - }) -}) - -describe("messagesToCC()", () => { - it("converts user, assistant, and tool result messages", () => { - const result = messagesToCC([ - { role: "user", content: "read /tmp/test" }, - { - role: "assistant", - content: [ - { type: "thinking", thinking: "I will read" }, - { type: "text", text: "Sure" }, - { - type: "toolCall", - id: "c1", - name: "read", - arguments: { path: "/tmp/test" }, - }, - ], - }, - { - role: "toolResult", - toolCallId: "c1", - toolName: "read", - isError: false, - content: [ - { type: "text", text: "hello" }, - { type: "text", text: "world" }, - ], - }, - ]) - - assert.equal(objectAt(result, ["0", "role"]), "user") - assert.equal(objectAt(result, ["1", "role"]), "assistant") - assert.equal(objectAt(result, ["1", "content", "0", "type"]), "text") - assert.equal(objectAt(result, ["1", "content", "1", "type"]), "tool-call") - assert.equal(objectAt(result, ["1", "content", "2"]), undefined) - assert.equal(objectAt(result, ["2", "role"]), "tool") - assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "hello\nworld") - }) - - it("serializes image inputs in the current Command Code wire format", () => { - assert.deepEqual( - messagesToCC( - [ - { - role: "user", - content: [ - { type: "text", text: "inspect this" }, - { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, - ], - }, - ], - { allowImages: true }, - ), - [ - { - role: "user", - content: [ - { type: "text", text: "inspect this" }, - { - type: "image", - image: "data:image/png;base64,aGVsbG8=", - mimeType: "image/png", - }, - ], - }, - ], - ) - }) - - it("preserves tool-result images as a following user image message", () => { - const result = messagesToCC( - [ - { role: "user", content: "read image" }, - { - role: "assistant", - content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], - }, - { - role: "toolResult", - toolCallId: "c1", - toolName: "read", - content: [ - { type: "text", text: "image attached" }, - { type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }, - ], - }, - ], - { allowImages: true }, - ) - - assert.equal(objectAt(result, ["2", "role"]), "tool") - assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "image attached") - assert.deepEqual(objectAt(result, ["3"]), { - role: "user", - content: [ - { - type: "image", - image: "data:image/jpeg;base64,aGVsbG8=", - mimeType: "image/jpeg", - }, - ], - }) - }) - - it("drops previous assistant reasoning while preserving text and tool calls", () => { - const result = messagesToCC([ - { role: "user", content: "first question" }, - { - role: "assistant", - content: [ - { type: "thinking", thinking: "private reasoning from turn one" }, - { type: "text", text: "first answer" }, - ], - }, - { role: "user", content: "follow-up question" }, - ]) - - assert.deepEqual(result, [ - { role: "user", content: "first question" }, - { role: "assistant", content: [{ type: "text", text: "first answer" }] }, - { role: "user", content: "follow-up question" }, - ]) - }) - - it("omits assistant turns that contain only previous reasoning", () => { - const result = messagesToCC([ - { role: "user", content: "first question" }, - { - role: "assistant", - content: [{ type: "thinking", thinking: "private reasoning" }], - }, - { role: "user", content: "follow-up question" }, - ]) - - assert.deepEqual(result, [ - { role: "user", content: "first question" }, - { role: "user", content: "follow-up question" }, - ]) - }) - - it("drops orphaned tool calls that have no matching tool result", () => { - const result = messagesToCC([ - { role: "user", content: "edit a file" }, - { - role: "assistant", - content: [ - { type: "text", text: "I will edit it" }, - { - type: "toolCall", - id: "missing-result", - name: "edit", - arguments: { path: "x" }, - }, - ], - }, - ]) - - assert.equal(objectAt(result, ["1", "role"]), "assistant") - assert.equal(objectAt(result, ["1", "content", "0", "type"]), "text") - assert.equal(objectAt(result, ["1", "content", "1"]), undefined) - }) - - it("handles empty conversations", () => { - assert.deepEqual(messagesToCC([]), []) - }) -}) - -describe("parseStreamEventLine()", () => { - it("parses plain JSON and SSE data lines", () => { - assert.deepEqual(parseStreamEventLine('{"type":"text-delta","text":"x"}'), { - type: "text-delta", - text: "x", - }) - assert.deepEqual(parseStreamEventLine('data: {"type":"finish","finishReason":"stop"}'), { - type: "finish", - finishReason: "stop", - }) - }) - - it("ignores comments, event labels, done markers, and malformed JSON", () => { - assert.equal(parseStreamEventLine(":"), undefined) - assert.equal(parseStreamEventLine("event: message"), undefined) - assert.equal(parseStreamEventLine("data: [DONE]"), undefined) - assert.equal(parseStreamEventLine("not-json"), undefined) - }) -}) - -describe("mapFinishReason()", () => { - it("maps provider finish reasons to pi stop reasons", () => { - assert.equal(mapFinishReason("stop"), "stop") - assert.equal(mapFinishReason("tool-calls"), "toolUse") - assert.equal(mapFinishReason("max_tokens"), "length") - assert.equal(mapFinishReason("max_output_tokens"), "length") - }) -}) diff --git a/tests/test-retry.ts b/tests/test-retry.ts deleted file mode 100644 index e0366de..0000000 --- a/tests/test-retry.ts +++ /dev/null @@ -1,470 +0,0 @@ -/** - * Tests for retry and timeout behaviour driven by pi settings.json - * retry config (timeoutMs, maxRetries, maxRetryDelayMs). - */ - -import assert from "node:assert/strict" -import { after, before, beforeEach, describe, it } from "node:test" - -import type { AssistantMessageEvent } from "../src/core.ts" -import { - collectEvents, - createTestDeps, - makeContext, - makeModel, - startMockCommandCodeServer, - type MockCommandCodeServer, -} from "./helpers.ts" - -const TEST_API_KEY = "option-key" - -let server: MockCommandCodeServer - -before(async () => { - server = await startMockCommandCodeServer() -}) - -after(async () => { - await server.close() -}) - -beforeEach(() => { - server.reset() -}) - -function eventTypes(events: readonly AssistantMessageEvent[]): string[] { - return events.map((event) => event.type) -} - -describe("streamCommandCode — retry on transient errors", () => { - it("retries on 429 and succeeds on the second attempt", async () => { - server.mockResponseQueue([ - { type: "error", status: 429, body: "rate limited" }, - { - type: "success", - events: [ - JSON.stringify({ type: "text-delta", text: "ok" }), - JSON.stringify({ type: "finish", finishReason: "stop" }), - ], - }, - ]) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - maxRetries: 2, - }), - ) - - assert.equal(server.requestCount(), 2) - assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "text_end", "done"]) - const done = events.at(-1) - if (done?.type !== "done") throw new Error("expected done") - assert.equal(done.reason, "stop") - }) - - it("retries on 500 and succeeds on the second attempt", async () => { - server.mockResponseQueue([ - { type: "error", status: 500, body: "internal server error" }, - { - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }, - ]) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - maxRetries: 2, - }), - ) - - assert.equal(server.requestCount(), 2) - assert.equal(events.at(-1)?.type, "done") - }) - - it("does NOT retry on 400 (non-retryable client error)", async () => { - server.mockResponse({ type: "error", status: 400, body: "bad request" }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: TEST_API_KEY }), - ) - - assert.equal(server.requestCount(), 1) - assert.deepEqual(eventTypes(events), ["start", "error"]) - const last = events.at(-1) - if (last?.type !== "error") throw new Error("expected error") - assert.match(last.error.errorMessage ?? "", /400/) - }) - - it("exhausts maxRetries and emits an error", async () => { - server.mockResponse({ type: "error", status: 503, body: "unavailable" }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - maxRetries: 3, - }), - ) - - // initial attempt + 3 retries = 4 total - assert.equal(server.requestCount(), 4) - assert.deepEqual(eventTypes(events), ["start", "error"]) - const last503 = events.at(-1) - if (last503?.type !== "error") throw new Error("expected error") - assert.match(last503.error.errorMessage ?? "", /503/) - }) -}) - -describe("streamCommandCode — Retry-After header", () => { - it("respects Retry-After delay in seconds", async () => { - let delayCalled = false - server.mockResponseQueue([ - { - type: "error", - status: 429, - body: "rate limited", - headers: { "retry-after": "2" }, - }, - { - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }, - ]) - const { streamCommandCode } = createTestDeps({ - apiBase: server.baseUrl(), - delay: async (ms: number) => { - delayCalled = true - assert.equal(ms, 2000) - }, - }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - maxRetries: 2, - }), - ) - - assert.equal(server.requestCount(), 2) - assert.equal(events.at(-1)?.type, "done") - assert.ok(delayCalled, "delay should have been called with Retry-After value") - }) - - it("fails immediately when Retry-After exceeds maxRetryDelayMs", async () => { - server.mockResponse({ - type: "error", - status: 429, - body: "rate limited", - headers: { "retry-after": "300" }, - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - maxRetryDelayMs: 10_000, - }), - ) - - assert.equal(server.requestCount(), 1) - assert.deepEqual(eventTypes(events), ["start", "error"]) - const lastMax = events.at(-1) - if (lastMax?.type !== "error") throw new Error("expected error") - assert.match(lastMax.error.errorMessage ?? "", /exceeds max/) - }) - - it("does not cap Retry-After when maxRetryDelayMs is 0", async () => { - let delayCalled = false - server.mockResponseQueue([ - { - type: "error", - status: 429, - body: "rate limited", - headers: { "retry-after": "120" }, - }, - { - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }, - ]) - const { streamCommandCode } = createTestDeps({ - apiBase: server.baseUrl(), - delay: async (ms: number) => { - delayCalled = true - assert.equal(ms, 120_000) - }, - }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - maxRetries: 1, - maxRetryDelayMs: 0, - }), - ) - - assert.equal(server.requestCount(), 2) - assert.equal(events.at(-1)?.type, "done") - assert.ok(delayCalled) - }) -}) - -describe("streamCommandCode — timeout", () => { - it("retries on per-attempt timeout and succeeds", async () => { - server.mockResponseQueue([ - { - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - hangAfterLast: true, - responseDelay: 200, - }, - { - type: "success", - events: [ - JSON.stringify({ type: "text-delta", text: "fast" }), - JSON.stringify({ type: "finish", finishReason: "stop" }), - ], - }, - ]) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - timeoutMs: 50, - maxRetries: 2, - }), - 5_000, - ) - - assert.equal(server.requestCount(), 2) - assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "text_end", "done"]) - }) - - it("retries when the response starts but the stream hangs before finish", async () => { - server.mockResponseQueue([ - { - type: "success", - events: [], - hangAfterLast: true, - }, - { - type: "success", - events: [ - JSON.stringify({ type: "text-delta", text: "ok" }), - JSON.stringify({ type: "finish", finishReason: "stop" }), - ], - }, - ]) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - timeoutMs: 50, - maxRetries: 2, - }), - 5_000, - ) - - assert.equal(server.requestCount(), 2) - assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "text_end", "done"]) - }) - - it("does NOT retry on timeout after partial text-delta was emitted", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "text-delta", text: "partial" })], - hangAfterLast: true, - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - timeoutMs: 50, - maxRetries: 2, - }), - 5_000, - ) - - assert.equal(server.requestCount(), 1) - assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "error"]) - }) - - it("emits error when all retry attempts time out", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - hangAfterLast: true, - responseDelay: 200, - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - timeoutMs: 50, - maxRetries: 1, - }), - 5_000, - ) - - // initial + 1 retry = 2 - assert.equal(server.requestCount(), 2) - assert.deepEqual(eventTypes(events), ["start", "error"]) - const error = events.at(-1) - if (error?.type !== "error") throw new Error("expected error") - assert.match(error.error.errorMessage ?? "", /timed out after 50ms/) - }) -}) - -describe("streamCommandCode — abort cancels retry loop", () => { - it("user abort stops retries immediately", async () => { - server.mockResponse({ type: "error", status: 500, body: "error" }) - const controller = new AbortController() - const { streamCommandCode } = createTestDeps({ - apiBase: server.baseUrl(), - delay: async (_ms: number, signal: AbortSignal) => { - // Abort during the retry delay - controller.abort() - // Simulate the real delay which rejects on abort - return new Promise((_, reject) => { - if (signal.aborted) reject(new DOMException("Aborted", "AbortError")) - signal.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError"))) - }) - }, - }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - signal: controller.signal, - maxRetries: 10, - }), - ) - - // Should only have made 1 request (the initial one), then aborted during delay - assert.equal(server.requestCount(), 1) - assert.deepEqual(eventTypes(events), ["start", "error"]) - const error = events.at(-1) - if (error?.type !== "error") throw new Error("expected error") - assert.equal(error.reason, "aborted") - }) -}) - -describe("streamCommandCode — retry defaults", () => { - it("uses default maxRetries of 0 when not specified", async () => { - server.mockResponse({ type: "error", status: 500, body: "error" }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - await collectEvents(streamCommandCode(makeModel(), makeContext(), { apiKey: TEST_API_KEY })) - - assert.equal(server.requestCount(), 1) - }) - - it("respects maxRetries: 0 (no retries)", async () => { - server.mockResponse({ type: "error", status: 500, body: "error" }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - maxRetries: 0, - }), - ) - - assert.equal(server.requestCount(), 1) - assert.deepEqual(eventTypes(events), ["start", "error"]) - }) -}) - -describe("streamCommandCode — stream-level error retry", () => { - it("retries when API returns 200 OK but stream contains an error event", async () => { - server.mockResponseQueue([ - { - type: "success", - events: [ - JSON.stringify({ - type: "error", - error: "Service temporarily unavailable. Please try again shortly.", - }), - ], - }, - { - type: "success", - events: [ - JSON.stringify({ type: "text-delta", text: "ok" }), - JSON.stringify({ type: "finish", finishReason: "stop" }), - ], - }, - ]) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - maxRetries: 2, - }), - ) - - assert.equal(server.requestCount(), 2) - assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "text_end", "done"]) - }) - - it("exhausts retries on persistent stream-level errors", async () => { - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ - type: "error", - error: "Service temporarily unavailable. Please try again shortly.", - }), - ], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: TEST_API_KEY, - maxRetries: 3, - }), - ) - - // initial + 3 retries = 4 - assert.equal(server.requestCount(), 4) - assert.deepEqual(eventTypes(events), ["start", "error"]) - const last = events.at(-1) - if (last?.type !== "error") throw new Error("expected error") - assert.match(last.error.errorMessage ?? "", /temporarily unavailable/) - }) - - it("does NOT retry stream error when content was already emitted", async () => { - server.mockResponseQueue([ - { - type: "success", - events: [ - JSON.stringify({ type: "text-delta", text: "partial" }), - JSON.stringify({ - type: "error", - error: "Service temporarily unavailable", - }), - ], - }, - ]) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: TEST_API_KEY }), - ) - - // Only 1 request — no retry because content was already emitted. - assert.equal(server.requestCount(), 1) - assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "error"]) - }) -}) diff --git a/tests/test-runtime.ts b/tests/test-runtime.ts index c8b0af0..397c54e 100644 --- a/tests/test-runtime.ts +++ b/tests/test-runtime.ts @@ -49,6 +49,7 @@ class CommandContext implements CommandCodeCommandContext { const FIRST_MODEL: CommandCodeModel = { id: "first-model", name: "First Model", + api: "openai-completions", reasoning: true, contextWindow: 128_000, maxTokens: 16_384, @@ -57,6 +58,7 @@ const FIRST_MODEL: CommandCodeModel = { const SECOND_MODEL: CommandCodeModel = { id: "second-model", name: "Second Model", + api: "openai-completions", reasoning: true, contextWindow: 256_000, maxTokens: 32_768, diff --git a/tests/test-stream.ts b/tests/test-stream.ts deleted file mode 100644 index e30ac49..0000000 --- a/tests/test-stream.ts +++ /dev/null @@ -1,771 +0,0 @@ -/** - * Integration tests for the real streamCommandCode core using a local mock - * Command Code server. No real API key or pi runtime required. - */ - -import assert from "node:assert/strict" -import { after, before, beforeEach, describe, it } from "node:test" - -import type { AssistantMessageEvent } from "../src/core.ts" -import { MODEL_EFFORTS, thinkingLevelMapForEfforts } from "../src/models.ts" -import { - collectEvents, - createTestDeps, - makeContext, - makeModel, - objectAt, - startMockCommandCodeServer, - type MockCommandCodeServer, -} from "./helpers.ts" - -let server: MockCommandCodeServer - -before(async () => { - server = await startMockCommandCodeServer() -}) - -after(async () => { - await server.close() -}) - -beforeEach(() => { - server.reset() -}) - -function eventTypes(events: readonly AssistantMessageEvent[]): string[] { - return events.map((event) => event.type) -} - -describe("streamCommandCode — auth", () => { - it("emits a missing-key error without touching the network", async () => { - const { streamCommandCode } = createTestDeps({ - apiBase: server.baseUrl(), - env: {}, - authPaths: [], - }) - const stream = streamCommandCode(makeModel(), makeContext(), { - apiKey: "", - }) - const events = await collectEvents(stream) - - assert.deepEqual(eventTypes(events), ["error"]) - assert.equal(events[0].type, "error") - assert.equal(events[0].reason, "error") - assert.match(events[0].error.errorMessage ?? "", /No Command Code API key/) - assert.equal(server.requestCount(), 0) - }) - - it("ignores the literal env-var name and falls back to env", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ - apiBase: server.baseUrl(), - env: { COMMANDCODE_API_KEY: "env-key" }, - }) - - await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "COMMANDCODE_API_KEY" }), - ) - - assert.equal( - server.lastRequestHeaders().authorization, - "Bearer env-key", - "should resolve from env, not send the literal var name as the token", - ) - }) - - it("uses options.apiKey in the Authorization header", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ - apiBase: server.baseUrl(), - env: { COMMANDCODE_API_KEY: "env-key" }, - }) - - await collectEvents(streamCommandCode(makeModel(), makeContext(), { apiKey: "option-key" })) - - assert.equal(server.lastRequestHeaders().authorization, "Bearer option-key") - }) -}) - -describe("streamCommandCode — successful streams", () => { - it("emits start → text events → done and accumulates usage", async () => { - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ type: "text-delta", text: "Hel" }), - JSON.stringify({ type: "text-delta", text: "lo" }), - JSON.stringify({ - type: "finish", - finishReason: "stop", - totalUsage: { - inputTokens: 3124, - outputTokens: 15, - inputTokenDetails: { noCacheTokens: 52, cacheReadTokens: 3072 }, - }, - }), - ], - }) - const { streamCommandCode, calculatedUsages } = createTestDeps({ - apiBase: server.baseUrl(), - }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - - assert.deepEqual(eventTypes(events), [ - "start", - "text_start", - "text_delta", - "text_delta", - "text_end", - "done", - ]) - const done = events.at(-1) - assert.equal(done?.type, "done") - if (done?.type !== "done") throw new Error("expected done") - assert.equal(done.reason, "stop") - assert.equal(done.message.content[0]?.type, "text") - assert.equal( - done.message.content[0]?.type === "text" ? done.message.content[0].text : "", - "Hello", - ) - assert.equal(done.message.usage.input, 52) - assert.equal(done.message.usage.cacheRead, 3072) - assert.equal(done.message.usage.cacheWrite, 0) - assert.equal(done.message.usage.totalTokens, 3139) - assert.equal(calculatedUsages.length, 1) - }) - - it("sends images for vision-capable models", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - await collectEvents( - streamCommandCode( - makeModel({ id: "gpt-5.6-luna" }), - makeContext({ - messages: [ - { - role: "user", - content: [ - { type: "text", text: "inspect" }, - { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, - ], - }, - ], - }), - { apiKey: "mock-key" }, - ), - ) - - assert.equal( - objectAt(server.lastRequestBody(), ["params", "messages", "0", "content", "1", "image"]), - "data:image/png;base64,aGVsbG8=", - ) - }) - - it("rejects images before network access for text-only models", async () => { - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode( - makeModel({ id: "deepseek/deepseek-v4-pro" }), - makeContext({ - messages: [ - { - role: "user", - content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }], - }, - ], - }), - { apiKey: "mock-key" }, - ), - ) - - assert.equal(events.at(-1)?.type, "error") - assert.equal(server.requestCount(), 0) - }) - - it("derives uncached input when noCacheTokens is missing", async () => { - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ - type: "finish", - finishReason: "stop", - totalUsage: { - inputTokens: 100, - outputTokens: 10, - inputTokenDetails: { cacheReadTokens: 75 }, - }, - }), - ], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - - const done = events.at(-1) - assert.equal(done?.type, "done") - if (done?.type !== "done") throw new Error("expected done") - assert.equal(done.message.usage.input, 25) - assert.equal(done.message.usage.cacheRead, 75) - assert.equal(done.message.usage.cacheWrite, 0) - assert.equal(done.message.usage.totalTokens, 110) - }) - - it("accounts for cache writes separately from uncached input", async () => { - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ - type: "finish", - finishReason: "stop", - totalUsage: { - inputTokens: 100, - outputTokens: 10, - inputTokenDetails: { - noCacheTokens: 20, - cacheReadTokens: 70, - cacheWriteTokens: 10, - }, - }, - }), - ], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - - const done = events.at(-1) - assert.equal(done?.type, "done") - if (done?.type !== "done") throw new Error("expected done") - assert.equal(done.message.usage.input, 20) - assert.equal(done.message.usage.cacheRead, 70) - assert.equal(done.message.usage.cacheWrite, 10) - assert.equal(done.message.usage.totalTokens, 110) - }) - - it("ends on finish without waiting for an open upstream connection", async () => { - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ type: "text-delta", text: "done" }), - JSON.stringify({ type: "finish", finishReason: "stop" }), - ], - hangAfterLast: true, - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - 500, - ) - - assert.equal(events.at(-1)?.type, "done") - await new Promise((resolve) => setTimeout(resolve, 50)) - assert.ok(server.responseClosedBeforeEnd(), "client should cancel the still-open response body") - }) - - it("emits reasoning and tool-call blocks in order", async () => { - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ type: "reasoning-start" }), - JSON.stringify({ type: "reasoning-delta", text: "think" }), - JSON.stringify({ type: "reasoning-end" }), - JSON.stringify({ type: "text-delta", text: "Using tool" }), - JSON.stringify({ - type: "tool-call", - toolCallId: "call_1", - toolName: "read_file", - input: JSON.stringify({ path: "/tmp/x" }), - }), - JSON.stringify({ type: "finish", finishReason: "tool-calls" }), - ], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - - assert.deepEqual(eventTypes(events), [ - "start", - "thinking_start", - "thinking_delta", - "thinking_end", - "text_start", - "text_delta", - "text_end", - "toolcall_start", - "toolcall_end", - "done", - ]) - const done = events.at(-1) - if (done?.type !== "done") throw new Error("expected done") - assert.equal(done.reason, "toolUse") - assert.deepEqual( - done.message.content.map((content) => content.type), - ["thinking", "text", "toolCall"], - ) - const toolCall = done.message.content[2] - assert.equal(toolCall?.type === "toolCall" ? toolCall.name : "", "read_file") - }) - - it("flushes reasoning if finish arrives without reasoning-end", async () => { - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ type: "reasoning-delta", text: "unfinished thought" }), - JSON.stringify({ type: "finish", finishReason: "stop" }), - ], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - - const done = events.at(-1) - if (done?.type !== "done") throw new Error("expected done") - assert.equal(done.message.content[0]?.type, "thinking") - }) - - it("closes thinking block before text when reasoning-end is missing", async () => { - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ type: "reasoning-start" }), - JSON.stringify({ type: "reasoning-delta", text: "thinking" }), - JSON.stringify({ type: "text-delta", text: "answer" }), - JSON.stringify({ type: "finish", finishReason: "stop" }), - ], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - - assert.deepEqual(eventTypes(events), [ - "start", - "thinking_start", - "thinking_delta", - "thinking_end", - "text_start", - "text_delta", - "text_end", - "done", - ]) - }) - - it("closes thinking block before tool-call when reasoning-end is missing", async () => { - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ type: "reasoning-start" }), - JSON.stringify({ type: "reasoning-delta", text: "thinking" }), - JSON.stringify({ - type: "tool-call", - toolCallId: "call_1", - toolName: "read_file", - input: JSON.stringify({ path: "/tmp/x" }), - }), - JSON.stringify({ type: "finish", finishReason: "tool-calls" }), - ], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - - assert.deepEqual(eventTypes(events), [ - "start", - "thinking_start", - "thinking_delta", - "thinking_end", - "toolcall_start", - "toolcall_end", - "done", - ]) - }) -}) - -describe("streamCommandCode — request serialization", () => { - it("rejects image input before sending a lossy request", async () => { - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - const events = await collectEvents( - streamCommandCode( - makeModel(), - makeContext({ - messages: [ - { - role: "user", - content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], - }, - ], - }), - { apiKey: "mock-key" }, - ), - ) - - assert.deepEqual(eventTypes(events), ["start", "error"]) - const lastEvent = events.at(-1) - assert.equal(lastEvent?.type, "error") - if (lastEvent?.type === "error") { - assert.match(lastEvent.error.errorMessage ?? "", /does not support image content/i) - } - assert.equal(server.requestCount(), 0) - }) - it("sends the expected request body and default headers", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - const context = makeContext({ - messages: [ - { role: "user", content: "first" }, - { - role: "assistant", - content: [{ type: "text", text: "first response" }], - }, - { role: "user", content: "second" }, - ], - tools: [ - { - name: "get_weather", - description: "Get weather", - parameters: { - kind: "object", - properties: { city: { kind: "string" } }, - }, - }, - ], - }) - - await collectEvents( - streamCommandCode(makeModel(), context, { - apiKey: "mock-key", - maxTokens: 500, - }), - ) - - const body = server.lastRequestBody() - assert.equal(objectAt(body, ["config", "workingDir"]), "/repo") - assert.equal(objectAt(body, ["config", "date"]), "2026-05-05") - assert.equal(objectAt(body, ["params", "model"]), "deepseek/deepseek-v4-flash") - assert.equal(objectAt(body, ["params", "stream"]), true) - assert.equal(objectAt(body, ["params", "max_tokens"]), 500) - assert.equal(objectAt(body, ["params", "reasoning_effort"]), undefined) - assert.equal(objectAt(body, ["params", "temperature"]), 0.3) - assert.equal(objectAt(body, ["params", "system"]), "You are a test assistant.") - assert.equal(objectAt(body, ["memory"]), null) - assert.equal(objectAt(body, ["taste"]), null) - assert.equal(objectAt(body, ["skills"]), null) - assert.equal(objectAt(body, ["permissionMode"]), undefined) - assert.equal(objectAt(body, ["threadId"]), "00000000-0000-4000-8000-000000000000") - assert.equal( - objectAt(body, ["params", "messages", "1", "content", "0", "text"]), - "first response", - ) - assert.equal(objectAt(body, ["params", "tools", "0", "name"]), "get_weather") - - const headers = server.lastRequestHeaders() - assert.equal(headers.authorization, "Bearer mock-key") - assert.equal(headers["x-command-code-version"], "1.15.1") - assert.equal(headers["x-project-slug"], "repo") - assert.equal(headers["x-taste-learning"], "true") - assert.equal(headers["x-co-flag"], "false") - assert.equal(headers["x-session-id"], undefined) - }) - - it("accepts the legacy OMP nested reasoning map", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - const model = makeModel({ - id: "omp-compat-reasoning-model", - reasoning: true, - thinking: { effortMap: { high: "legacy-high" } }, - }) - - await collectEvents( - streamCommandCode(model, makeContext(), { apiKey: "mock-key", reasoning: "high" }), - ) - - assert.equal(objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]), "legacy-high") - }) - - it("forwards a supported Pi reasoning level as reasoning_effort", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - const model = makeModel({ - id: "deepseek/deepseek-v4-flash", - reasoning: true, - thinkingLevelMap: thinkingLevelMapForEfforts(MODEL_EFFORTS["deepseek/deepseek-v4-flash"]), - }) - - await collectEvents( - streamCommandCode(model, makeContext(), { apiKey: "mock-key", reasoning: "max" }), - ) - - assert.equal(objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]), "max") - }) - - it("omits reasoning_effort for off, unsupported, and unknown reasoning levels", async () => { - const model = makeModel({ - id: "deepseek/deepseek-v4-flash", - reasoning: true, - thinkingLevelMap: thinkingLevelMapForEfforts(MODEL_EFFORTS["deepseek/deepseek-v4-flash"]), - }) - - for (const reasoning of ["off", "low"] as const) { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - await collectEvents( - streamCommandCode(model, makeContext(), { apiKey: "mock-key", reasoning }), - ) - assert.equal( - objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]), - undefined, - `${reasoning} should not be sent when it has no supported Command Code field`, - ) - server.reset() - } - - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - await collectEvents( - streamCommandCode( - makeModel({ id: "new-model-without-metadata", reasoning: false }), - makeContext(), - { apiKey: "mock-key", reasoning: "high" }, - ), - ) - assert.equal(objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]), undefined) - }) - - it("caps maxTokens and passes custom headers", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - await collectEvents( - streamCommandCode(makeModel({ maxTokens: 500_000 }), makeContext(), { - apiKey: "mock-key", - maxTokens: 500_000, - headers: { "x-custom": "value" }, - }), - ) - - assert.equal(objectAt(server.lastRequestBody(), ["params", "max_tokens"]), 64_000) - assert.equal(server.lastRequestHeaders()["x-custom"], "value") - }) - - it("caps default maxTokens by the selected model", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - await collectEvents( - streamCommandCode(makeModel({ maxTokens: 8_192 }), makeContext(), { - apiKey: "mock-key", - }), - ) - - assert.equal(objectAt(server.lastRequestBody(), ["params", "max_tokens"]), 8_192) - }) - - it("serializes OMP system prompt arrays as a string", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - await collectEvents( - streamCommandCode( - makeModel(), - makeContext({ - systemPrompt: ["You are a test assistant.", "Use concise answers."] as unknown as string, - }), - { apiKey: "mock-key" }, - ), - ) - - assert.equal( - objectAt(server.lastRequestBody(), ["params", "system"]), - "You are a test assistant.\n\nUse concise answers.", - ) - }) - - it("times out a hung onResponse callback", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - const started = Date.now() - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: "mock-key", - timeoutMs: 25, - onResponse: async () => new Promise(() => {}), - }), - 1_000, - ) - - assert.ok(Date.now() - started < 500) - assert.deepEqual(eventTypes(events), ["start", "error"]) - const error = events.at(-1) - assert.equal(error?.type, "error") - if (error?.type !== "error") throw new Error("expected error") - assert.match(error.error.errorMessage ?? "", /timed out after 25ms/) - }) - - it("times out a hung onPayload callback", async () => { - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - const started = Date.now() - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: "mock-key", - timeoutMs: 25, - onPayload: async () => new Promise(() => {}), - }), - 1_000, - ) - - assert.ok(Date.now() - started < 500) - assert.deepEqual(eventTypes(events), ["start", "error"]) - const error = events.at(-1) - assert.equal(error?.type, "error") - if (error?.type !== "error") throw new Error("expected error") - assert.match(error.error.errorMessage ?? "", /timed out after 25ms/) - assert.equal(server.requestCount(), 0) - }) - - it("runs onPayload and onResponse hooks", async () => { - server.mockResponse({ - type: "success", - events: [JSON.stringify({ type: "finish", finishReason: "stop" })], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - let responseStatus = 0 - - await collectEvents( - streamCommandCode(makeModel(), makeContext(), { - apiKey: "mock-key", - onPayload: () => ({ replaced: true }), - onResponse: (response) => { - responseStatus = response.status - }, - }), - ) - - assert.equal(objectAt(server.lastRequestBody(), ["replaced"]), true) - assert.equal(responseStatus, 200) - }) -}) - -describe("streamCommandCode — upstream errors and malformed streams", () => { - it("emits error for HTTP failures", async () => { - server.mockResponse({ type: "error", status: 429, body: "rate limited" }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - - assert.deepEqual(eventTypes(events), ["start", "error"]) - const error = events.at(-1) - assert.equal(error?.type, "error") - if (error?.type !== "error") throw new Error("expected error") - assert.match(error.error.errorMessage ?? "", /429/) - }) - - it("emits error for provider error events", async () => { - server.mockResponse({ - type: "success", - events: [ - JSON.stringify({ - type: "error", - error: { message: "provider failed" }, - }), - ], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - - const error = events.at(-1) - assert.equal(error?.type, "error") - if (error?.type !== "error") throw new Error("expected error") - assert.equal(error.error.errorMessage, "provider failed") - }) - - it("handles SSE lines, malformed lines, split chunks, and final line without newline", async () => { - const textEvent = `data: ${JSON.stringify({ type: "text-delta", text: "split" })}\n` - const finishEvent = JSON.stringify({ - type: "finish", - finishReason: "max_tokens", - }) - server.mockResponse({ - type: "success", - chunks: [ - "not json\n", - textEvent.slice(0, 12), - textEvent.slice(12), - "event: ignored\n", - "data: [DONE]\n", - finishEvent, - ], - }) - const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) - - const events = await collectEvents( - streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), - ) - - const done = events.at(-1) - if (done?.type !== "done") throw new Error("expected done") - assert.equal(done.reason, "length") - assert.equal( - done.message.content[0]?.type === "text" ? done.message.content[0].text : "", - "split", - ) - }) -}) From 7af77ddf58875b22620ba3eebff07b4bed1de8b1 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 18 Aug 2026 10:56:11 +0200 Subject: [PATCH 03/40] fix(models): refresh expired display pricing --- src/pricing.ts | 40 ++----------------------- tests/fixtures/commandcode-pricing.json | 10 +++---- tests/test-pricing.ts | 16 ++++++---- 3 files changed, 18 insertions(+), 48 deletions(-) diff --git a/src/pricing.ts b/src/pricing.ts index aa801c0..f13a69e 100644 --- a/src/pricing.ts +++ b/src/pricing.ts @@ -20,7 +20,7 @@ export interface TemporaryPricing { } export const PRICING_SOURCE_URL = "https://commandcode.ai/docs/resources/pricing-limits" -export const PRICING_LAST_VERIFIED = "2026-08-04" +export const PRICING_LAST_VERIFIED = "2026-08-18" export const ZERO_MODEL_COST: CommandCodeModelCost = { input: 0, @@ -158,37 +158,8 @@ export const MODEL_COSTS: Readonly> = { // OpenAI "gpt-5.6-sol": { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 6.25 }, - // Discounted rates through 2026-08-14. - "gpt-5.6-terra": { - input: 1, - output: 6, - cacheRead: 0.1, - cacheWrite: 1.25, - tiers: [ - { - inputTokensAbove: 272_000, - input: 2, - output: 9, - cacheRead: 0.2, - cacheWrite: 2.5, - }, - ], - }, - "gpt-5.6-luna": { - input: 0.1, - output: 0.6, - cacheRead: 0.01, - cacheWrite: 0.125, - tiers: [ - { - inputTokensAbove: 272_000, - input: 0.2, - output: 0.9, - cacheRead: 0.02, - cacheWrite: 0.25, - }, - ], - }, + "gpt-5.6-terra": { input: 2, output: 12, cacheRead: 0.2, cacheWrite: 2.5 }, + "gpt-5.6-luna": { input: 0.2, output: 1.2, cacheRead: 0.02, cacheWrite: 0.25 }, "gpt-5.5": { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 }, "gpt-5.4": { input: 2.5, output: 15, cacheRead: 0.25, cacheWrite: 0 }, "gpt-5.3-codex": { input: 2, output: 8, cacheRead: 0.5, cacheWrite: 0 }, @@ -213,11 +184,6 @@ export const MODEL_COSTS: Readonly> = { } export const TEMPORARY_PRICING: readonly TemporaryPricing[] = [ - { - models: ["gpt-5.6-terra", "gpt-5.6-luna"], - expiresOn: "2026-08-14", - description: "50% promotional rates", - }, { models: ["claude-sonnet-5"], expiresOn: "2026-08-31", diff --git a/tests/fixtures/commandcode-pricing.json b/tests/fixtures/commandcode-pricing.json index deaee75..786cec2 100644 --- a/tests/fixtures/commandcode-pricing.json +++ b/tests/fixtures/commandcode-pricing.json @@ -1,5 +1,5 @@ { - "verifiedAt": "2026-08-04", + "verifiedAt": "2026-08-18", "source": "https://commandcode.ai/docs/resources/pricing-limits", "tierPolicy": "Use request-wide input tiers; the highest threshold exceeded by input plus cache tokens applies to the full request.", "tiers": { @@ -7,9 +7,7 @@ "Qwen/Qwen3.7-Flash": [ [32000, 0.1, 0.4, 0.02, 0.125], [256000, 0.2, 0.8, 0.04, 0.25] - ], - "gpt-5.6-terra": [[272000, 2, 9, 0.2, 2.5]], - "gpt-5.6-luna": [[272000, 0.2, 0.9, 0.02, 0.25]] + ] }, "costs": { "poolside/laguna-s-2.1-free": [0, 0, 0, 0], @@ -52,8 +50,8 @@ "claude-opus-4-7": [5, 25, 0.5, 6.25], "claude-haiku-4-5-20251001": [1, 5, 0.1, 1.25], "gpt-5.6-sol": [5, 30, 0.5, 6.25], - "gpt-5.6-terra": [1, 6, 0.1, 1.25], - "gpt-5.6-luna": [0.1, 0.6, 0.01, 0.125], + "gpt-5.6-terra": [2, 12, 0.2, 2.5], + "gpt-5.6-luna": [0.2, 1.2, 0.02, 0.25], "gpt-5.5": [5, 30, 0.5, 0], "gpt-5.4": [2.5, 15, 0.25, 0], "gpt-5.3-codex": [2, 8, 0.5, 0], diff --git a/tests/test-pricing.ts b/tests/test-pricing.ts index a782a73..c463fcf 100644 --- a/tests/test-pricing.ts +++ b/tests/test-pricing.ts @@ -148,16 +148,22 @@ describe("MODEL_COSTS pricing overlay", () => { cacheWrite: 0.038, }) assertCost("gpt-5.6-terra", { - input: 1, - output: 6, - cacheRead: 0.1, - cacheWrite: 1.25, + input: 2, + output: 12, + cacheRead: 0.2, + cacheWrite: 2.5, + }) + assertCost("gpt-5.6-luna", { + input: 0.2, + output: 1.2, + cacheRead: 0.02, + cacheWrite: 0.25, }) }) it("tracks pricing provenance", () => { assert.equal(PRICING_SOURCE_URL, "https://commandcode.ai/docs/resources/pricing-limits") - assert.equal(PRICING_LAST_VERIFIED, "2026-08-04") + assert.equal(PRICING_LAST_VERIFIED, "2026-08-18") }) it("fails once temporary pricing needs review", () => { From c04f3907b5c0046dab505e7c3835bb223eafc639 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 18 Aug 2026 10:56:12 +0200 Subject: [PATCH 04/40] docs(api): document provider api migration --- CHANGELOG.md | 6 ++++++ README.md | 24 ++++++++++++++++-------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cae0a0..27b09fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +- Replace the reverse-engineered `/alpha/generate` integration with Command Code's documented Provider API: `/provider/v1/chat/completions` for non-Claude models and `/provider/v1/messages` for Claude models. +- Use Pi's native OpenAI- and Anthropic-compatible providers for streaming, tools, reasoning, images, usage, errors, and retries while preserving dynamic model discovery, offline cache, refresh/status commands, pricing, and OAuth credentials. +- Let `/login` use browser authentication, an explicit API-key prompt, or a directly pasted API key. +- Add optional zero-data-retention headers through `CMD_ZDR=1` or `COMMANDCODE_ZDR=1`. +- Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended. + ## 0.5.1 - 2026-08-11 - Add model-specific image input capabilities from the `command-code@1.15.1` catalog and forward user and tool-result images using the current Command Code wire format. diff --git a/README.md b/README.md index 91b8085..41802fb 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,15 @@ A custom provider for [pi](https://github.com/earendil-works/pi) that connects to the [Command Code](https://commandcode.ai) Provider API. -> **Disclaimer:** This is an unofficial, community-maintained integration. It is not affiliated with, endorsed by, or supported by Command Code. You need your own Command Code account and API key or subscription. Command Code's terms, availability, and pricing apply. +> **Disclaimer:** This is an unofficial, community-maintained integration. It is not affiliated with, endorsed by, or supported by Command Code. You need your own Command Code account, API key, and a plan with Provider API access. Command Code's terms, availability, and pricing apply. + +The extension uses only Command Code's documented Provider API endpoints: + +- `GET /provider/v1/models` +- `POST /provider/v1/chat/completions` for non-Claude models +- `POST /provider/v1/messages` for Claude models + +Every current Command Code plan except **Go** has Provider API access: GOAT, Pro, Max, Team, and Provider. ## Install @@ -19,7 +27,7 @@ Start or reload pi, then authenticate: /login ``` -Select **Use a subscription**, then **Command Code**. Complete the browser flow and choose a model with `/model`. +Select **Use a subscription**, then **Command Code**. Choose browser login or paste an API key, then select a model with `/model`. ## Oh My Pi @@ -33,9 +41,9 @@ Restart OMP or run `/reload`, then use `/login` and select **Use a subscription* ## Authentication -### Browser login +### Login dialog -Run `/login` in pi or OMP. Select **Use a subscription**, then **Command Code**. The browser flow stores the returned credential in the host's auth file. +Run `/login` in pi or OMP. Select **Use a subscription**, then **Command Code**. Press Enter for browser login, type `key` to open a paste prompt, or paste the API key directly. The selected credential is stored in the host's auth file. Select Command Code in pi's login dialog @@ -84,9 +92,7 @@ Open `/model` and select one of the models provided by Command Code. Model avail ### Reasoning support -Reasoning metadata is enriched only for models whose Command Code effort support is known. Those models register a model-specific `thinkingLevelMap`, so pi and OMP expose only supported levels. A selected supported level is sent as the documented `params.reasoning_effort` field; `off`, unsupported levels, and newly discovered models without metadata do not add reasoning fields to the request. No prompt instructions are injected. - -Reasoning blocks from completed assistant turns remain visible in pi's local session, but are not replayed to Command Code in later requests. Only the assistant's user-visible text and completed tool calls are sent back as history. This matches the current Command Code CLI behavior and prevents prior private reasoning traces from interfering with reasoning on follow-up turns. +Reasoning metadata is enriched only for models whose Command Code effort support is known. Those models register a model-specific `thinkingLevelMap`, so pi and OMP expose only supported levels. Pi's native OpenAI- and Anthropic-compatible providers translate the selected level to the endpoint's documented reasoning fields. Unsupported levels and newly discovered models without metadata do not claim reasoning support. List Command Code models from the terminal: @@ -123,6 +129,8 @@ While pi is running, use these provider commands without restarting: - `/commandcode-refresh` fetches and re-registers the current model catalog. Overlapping refreshes are coalesced, and a failed refresh keeps the last valid catalog active. - `/commandcode-status` shows redacted discovery diagnostics, including the source, model count, timestamps, cache path, endpoint, and warning. +Set `CMD_ZDR=1` or `COMMANDCODE_ZDR=1` to send Command Code's documented `x-cmd-zdr: 1` zero-data-retention header. + The following environment variables are intended for tests, local mocks, and compatible API endpoints: - `COMMANDCODE_API_BASE` @@ -134,7 +142,7 @@ The following environment variables are intended for tests, local mocks, and com The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.15.1`; unknown models default to text-only until their upstream metadata is reviewed. -For vision-capable models, image blocks from user messages and tool results are forwarded in Command Code's current data-URL wire format. Text-only models reject image content before making a network request instead of silently dropping it. +For vision-capable models, Pi's native provider adapters forward image blocks from user messages and tool results using the documented OpenAI or Anthropic message schema. Unknown and text-only models remain marked text-only in Pi. ## Pricing display From 864538e146e72179970cf09ca76dc13f1e065c1f Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 18 Aug 2026 11:01:12 +0200 Subject: [PATCH 05/40] fix(ci): satisfy provider audit checks --- .gitleaks.toml | 2 +- CHANGELOG.md | 2 +- README.md | 2 +- index.ts | 2 +- src/models.ts | 8 ++++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitleaks.toml b/.gitleaks.toml index e1c2d97..dcd3149 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -41,5 +41,5 @@ title = "pi-commandcode-provider secret scan" [[rules]] id = "pi-test-api-key" description = "Test API key value that looks real" - regex = '''(user_testKey|mock-key|fake-key|test-api-key)''' + regex = '''['"](user_testKey|mock-key|fake-key|test-api-key)['"]''' tags = ["pi-extension", "test"] diff --git a/CHANGELOG.md b/CHANGELOG.md index 27b09fe..887e50d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ - Replace the reverse-engineered `/alpha/generate` integration with Command Code's documented Provider API: `/provider/v1/chat/completions` for non-Claude models and `/provider/v1/messages` for Claude models. - Use Pi's native OpenAI- and Anthropic-compatible providers for streaming, tools, reasoning, images, usage, errors, and retries while preserving dynamic model discovery, offline cache, refresh/status commands, pricing, and OAuth credentials. - Let `/login` use browser authentication, an explicit API-key prompt, or a directly pasted API key. -- Add optional zero-data-retention headers through `CMD_ZDR=1` or `COMMANDCODE_ZDR=1`. +- Add optional zero-data-retention headers through `COMMANDCODE_ZDR=1`. - Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended. ## 0.5.1 - 2026-08-11 diff --git a/README.md b/README.md index 41802fb..1ac02a6 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ While pi is running, use these provider commands without restarting: - `/commandcode-refresh` fetches and re-registers the current model catalog. Overlapping refreshes are coalesced, and a failed refresh keeps the last valid catalog active. - `/commandcode-status` shows redacted discovery diagnostics, including the source, model count, timestamps, cache path, endpoint, and warning. -Set `CMD_ZDR=1` or `COMMANDCODE_ZDR=1` to send Command Code's documented `x-cmd-zdr: 1` zero-data-retention header. +Set `COMMANDCODE_ZDR=1` to send Command Code's documented `x-cmd-zdr: 1` zero-data-retention header. The following environment variables are intended for tests, local mocks, and compatible API endpoints: diff --git a/index.ts b/index.ts index 552e3ee..36fa7bd 100644 --- a/index.ts +++ b/index.ts @@ -29,7 +29,7 @@ import { MODEL_COSTS, ZERO_MODEL_COST } from "./src/pricing.ts" import { createCommandCodeRuntime } from "./src/runtime.ts" function commandCodeHeaders(): Record | undefined { - if (process.env.CMD_ZDR === "1" || process.env.COMMANDCODE_ZDR === "1") { + if (process.env.COMMANDCODE_ZDR === "1") { return { "x-cmd-zdr": "1" } } return undefined diff --git a/src/models.ts b/src/models.ts index b417c46..3f8c7e9 100644 --- a/src/models.ts +++ b/src/models.ts @@ -1,6 +1,5 @@ import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises" import { dirname } from "node:path" -import type { Api } from "@earendil-works/pi-ai" export const DEFAULT_PROVIDER_API_BASE = "https://api.commandcode.ai/provider/v1" export const DEFAULT_MODELS_URL = `${DEFAULT_PROVIDER_API_BASE}/models` @@ -9,6 +8,7 @@ export const DEFAULT_MODELS_TIMEOUT_MS = 10_000 const DEFAULT_MAX_OUTPUT_TOKENS = 65_536 const MODEL_CACHE_VERSION = 1 +export type CommandCodeApi = "openai-completions" | "anthropic-messages" export type CommandCodeInputType = "text" | "image" /** @@ -161,17 +161,17 @@ interface ApiModel { export interface CommandCodeModel { id: string name: string - api: Api + api: CommandCodeApi reasoning: boolean contextWindow: number maxTokens: number } -export function apiForModelId(id: string): Api { +export function apiForModelId(id: string): CommandCodeApi { return id.startsWith("claude-") ? "anthropic-messages" : "openai-completions" } -export function baseUrlForModel(apiBase: string, api: Api): string { +export function baseUrlForModel(apiBase: string, api: CommandCodeApi): string { const normalized = apiBase.replace(/\/+$/g, "") if (api !== "anthropic-messages") return normalized return normalized.endsWith("/v1") ? normalized.slice(0, -3) : normalized From 89dc21bed23a98ed9b8986ea86a8b69e11ee3375 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 18 Aug 2026 12:11:47 +0200 Subject: [PATCH 06/40] feat(api): fall back for go plan accounts --- index.ts | 30 +- package.json | 9 +- src/converters.ts | 296 ++++++++++++++ src/core.ts | 741 +++++++++++++++++++++++++++++++++ src/cost.ts | 31 ++ src/json-schema.ts | 382 +++++++++++++++++ src/overflow.ts | 120 ++++++ src/transport.ts | 139 +++++++ src/types.ts | 202 +++++++++ tests/helpers.ts | 289 +++++++++++++ tests/test-abort.ts | 85 ++++ tests/test-cost.ts | 183 +++++++++ tests/test-overflow.ts | 196 +++++++++ tests/test-pure-functions.ts | 658 ++++++++++++++++++++++++++++++ tests/test-retry.ts | 470 +++++++++++++++++++++ tests/test-stream.ts | 771 +++++++++++++++++++++++++++++++++++ tests/test-transport.ts | 183 +++++++++ 17 files changed, 4782 insertions(+), 3 deletions(-) create mode 100644 src/converters.ts create mode 100644 src/core.ts create mode 100644 src/cost.ts create mode 100644 src/json-schema.ts create mode 100644 src/overflow.ts create mode 100644 src/transport.ts create mode 100644 src/types.ts create mode 100644 tests/helpers.ts create mode 100644 tests/test-abort.ts create mode 100644 tests/test-cost.ts create mode 100644 tests/test-overflow.ts create mode 100644 tests/test-pure-functions.ts create mode 100644 tests/test-retry.ts create mode 100644 tests/test-stream.ts create mode 100644 tests/test-transport.ts diff --git a/index.ts b/index.ts index 36fa7bd..d263138 100644 --- a/index.ts +++ b/index.ts @@ -5,6 +5,8 @@ * https://api.commandcode.ai/provider/v1 */ +import { AssistantMessageEventStream } from "@earendil-works/pi-ai" +import { streamSimple as streamNativeProvider } from "@earendil-works/pi-ai/compat" import { getAgentDir, type ExtensionAPI, @@ -14,6 +16,8 @@ import { import { join } from "node:path" import { getConfiguredApiKey } from "./src/api-key.ts" +import { createStreamCommandCode } from "./src/core.ts" +import { calculateCommandCodeCost } from "./src/cost.ts" import { baseUrlForModel, DEFAULT_MODELS_URL, @@ -25,8 +29,10 @@ import { type CommandCodeModel, } from "./src/models.ts" import { getApiKey as getOAuthApiKey, login, refreshToken } from "./src/oauth.ts" +import { normalizeCommandCodeMessage } from "./src/overflow.ts" import { MODEL_COSTS, ZERO_MODEL_COST } from "./src/pricing.ts" import { createCommandCodeRuntime } from "./src/runtime.ts" +import { createCommandCodeTransportRouter } from "./src/transport.ts" function commandCodeHeaders(): Record | undefined { if (process.env.COMMANDCODE_ZDR === "1") { @@ -38,6 +44,7 @@ function commandCodeHeaders(): Record | undefined { function createProviderConfig( models: readonly CommandCodeModel[], apiBase: string, + streamCommandCode: ProviderConfig["streamSimple"], ): ProviderConfig { const headers = commandCodeHeaders() return { @@ -45,6 +52,7 @@ function createProviderConfig( baseUrl: apiBase, apiKey: getConfiguredApiKey() ?? "$COMMANDCODE_API_KEY", api: "openai-completions", + streamSimple: streamCommandCode, headers, oauth: { name: "Command Code", @@ -82,12 +90,32 @@ function createProviderConfig( } } +function legacyApiBase(providerApiBase: string): string { + return providerApiBase.replace(/\/provider\/v1\/?$/, "") +} + export default async function (pi: ExtensionAPI) { const apiBase = process.env.COMMANDCODE_API_BASE ?? DEFAULT_PROVIDER_API_BASE const modelsUrl = process.env.COMMANDCODE_MODELS_URL ?? DEFAULT_MODELS_URL const modelsTimeoutMs = getModelsTimeoutMs() const modelsCachePath = process.env.COMMANDCODE_MODELS_CACHE ?? join(getAgentDir(), "commandcode-models.json") + const streamGenerate = createStreamCommandCode({ + createStream: () => new AssistantMessageEventStream(), + calculateCost: calculateCommandCodeCost, + apiBase: legacyApiBase(apiBase), + }) + const transport = createCommandCodeTransportRouter({ + createStream: () => new AssistantMessageEventStream(), + streamProvider: streamNativeProvider, + streamGenerate, + }) + + pi.on("message_end", async (event, ctx) => { + if (event.message.role !== "assistant") return + const normalized = normalizeCommandCodeMessage(event.message, ctx.model?.provider) + return normalized ? { message: normalized.message } : undefined + }) const runtime = createCommandCodeRuntime(pi, { endpoint: modelsUrl, @@ -98,7 +126,7 @@ export default async function (pi: ExtensionAPI) { cachePath: modelsCachePath, timeoutMs: modelsTimeoutMs, }), - createProviderConfig: (models) => createProviderConfig(models, apiBase), + createProviderConfig: (models) => createProviderConfig(models, apiBase, transport.stream), }) await runtime.initialize() diff --git a/package.json b/package.json index 9af81b6..c99a0aa 100644 --- a/package.json +++ b/package.json @@ -29,18 +29,23 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-oauth.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", "pi:isolated": "node scripts/pi-isolated.mjs", "pi:authenticated": "node scripts/pi-authenticated.mjs", - "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-oauth.ts", + "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", "test:api-key": "tsx tests/test-api-key.ts", "test:models": "tsx tests/test-models.ts", "test:runtime": "tsx tests/test-runtime.ts", "test:pricing": "tsx tests/test-pricing.ts", "test:oauth": "tsx tests/test-oauth.ts", + "test:abort": "tsx tests/test-abort.ts", + "test:overflow": "tsx tests/test-overflow.ts", + "test:stream": "tsx tests/test-stream.ts", + "test:retry": "tsx tests/test-retry.ts", + "test:transport": "tsx tests/test-transport.ts", "test:pi-isolated": "node tests/test-pi-isolated.mjs", "test:pi-authenticated": "node tests/test-pi-authenticated.mjs", "test:pi-local": "node tests/test-pi-local.mjs", diff --git a/src/converters.ts b/src/converters.ts new file mode 100644 index 0000000..4012fbb --- /dev/null +++ b/src/converters.ts @@ -0,0 +1,296 @@ +import { existsSync, readFileSync } from "node:fs" +import { homedir } from "node:os" +import { join } from "node:path" + +import type { MessageLike, StopReason, ToolLike } from "./types.ts" +import { toJsonSchema } from "./json-schema.ts" + +export { toJsonSchema } from "./json-schema.ts" + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +export function stringValue(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined +} + +export function recordArray(value: unknown): readonly Record[] { + if (!Array.isArray(value)) return [] + return value.filter(isRecord) +} + +export function recordOrEmpty(value: unknown): Record { + if (isRecord(value)) return value + if (typeof value === "string") { + try { + const parsed: unknown = JSON.parse(value) + if (isRecord(parsed)) return parsed + } catch { + // Some providers stream incomplete JSON argument fragments. + } + } + return {} +} + +export function numberValue(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined +} + +function defaultAuthPaths(home: string): string[] { + return [ + join(home, ".commandcode", "auth.json"), + join(home, ".omp", "agent", "auth.json"), + join(home, ".pi", "agent", "auth.json"), + ] +} + +function apiKeyFromCredentialRecord(value: unknown): string | undefined { + if (!isRecord(value)) return undefined + + const type = stringValue(value.type) + if (type === "api") return stringValue(value.key) + if (type === "oauth") return stringValue(value.access) + + return stringValue(value.key) ?? stringValue(value.access) +} + +function imageParts(value: unknown): readonly Record[] { + if (isRecord(value)) return value.type === "image" ? [value] : [] + return recordArray(value).filter((part) => part.type === "image") +} + +function imageContentError(role: string): Error { + return new Error(`Selected Command Code model does not support image content in ${role}`) +} + +export function assertTextOnlyMessages(messages?: readonly MessageLike[]): void { + for (const message of messages ?? []) { + if (imageParts(message.content).length > 0) { + const role = message.role === "toolResult" ? "tool results" : `${message.role} messages` + throw imageContentError(role) + } + } +} + +function imageToCommandCode(part: Record): Record { + const data = stringValue(part.data) + const mimeType = stringValue(part.mimeType) + if (!data || !mimeType) + throw new Error("Invalid image content: expected base64 data and mimeType") + + return { + type: "image", + image: `data:${mimeType};base64,${data}`, + mimeType, + } +} + +function userContentToCommandCode(content: unknown, allowImages: boolean): unknown { + if (typeof content === "string") return content + + return recordArray(content).flatMap((part) => { + if (part.type === "text") return [{ type: "text", text: stringValue(part.text) ?? "" }] + if (part.type === "image") { + if (!allowImages) throw imageContentError("user messages") + return [imageToCommandCode(part)] + } + return [] + }) +} + +export function getApiKey( + options: { + env?: NodeJS.ProcessEnv + authPaths?: readonly string[] + homeDir?: () => string + } = {}, +): string | undefined { + const env = options.env ?? process.env + if (env.COMMANDCODE_API_KEY) return env.COMMANDCODE_API_KEY + + const home = options.homeDir?.() ?? homedir() + const authPaths = options.authPaths ?? defaultAuthPaths(home) + + for (const authPath of authPaths) { + try { + if (!existsSync(authPath)) continue + const parsed: unknown = JSON.parse(readFileSync(authPath, "utf-8")) + if (!isRecord(parsed)) continue + + // Legacy: direct apiKey or commandcode field. + const apiKey = stringValue(parsed.apiKey) + if (apiKey) return apiKey + const commandcode = stringValue(parsed.commandcode) + if (commandcode) return commandcode + + // pi stores OAuth credentials as {"commandcode": {"type":"oauth","access":"..."}}. + // The official Command Code CLI stores API credentials under "command-code". + const providerKey = + apiKeyFromCredentialRecord(parsed.commandcode) ?? + apiKeyFromCredentialRecord(parsed["command-code"]) + if (providerKey) return providerKey + } catch { + // Ignore malformed or unreadable auth files. + } + } + + return undefined +} + +export function textContent(message: { content?: unknown }): string { + return recordArray(message.content) + .filter((part) => part.type === "text") + .map((part) => stringValue(part.text) ?? "") + .join("\n") +} + +export function getEnvironmentInfo(): string { + return `${process.platform}-${process.arch}, Node.js ${process.version}` +} + +export function toolsToJson(tools?: readonly ToolLike[]): unknown[] { + if (!tools) return [] + return tools.map((tool) => ({ + type: "function", + name: tool.name, + description: tool.description, + input_schema: tool.parameters ? toJsonSchema(tool.parameters) : {}, + })) +} + +function completeToolCallIds(messages?: readonly MessageLike[]): Set { + const callIds = new Set() + const resultIds = new Set() + + for (const message of messages ?? []) { + if (message.role === "assistant") { + for (const content of recordArray(message.content)) { + if (content.type === "toolCall") { + const id = stringValue(content.id) + if (id) callIds.add(id) + } + } + } else if (message.role === "toolResult") { + if (message.toolCallId) resultIds.add(message.toolCallId) + } + } + + return new Set([...callIds].filter((id) => resultIds.has(id))) +} + +export function messagesToCC( + messages?: readonly MessageLike[], + options: { allowImages?: boolean } = {}, +): unknown[] { + const allowImages = options.allowImages ?? false + if (!allowImages) assertTextOnlyMessages(messages) + + const out: unknown[] = [] + const pairedToolCallIds = completeToolCallIds(messages) + + for (const message of messages ?? []) { + if (message.role === "user") { + out.push({ + role: "user", + content: userContentToCommandCode(message.content, allowImages), + }) + } else if (message.role === "assistant") { + const parts: unknown[] = [] + for (const content of recordArray(message.content)) { + if (content.type === "text") { + parts.push({ type: "text", text: stringValue(content.text) ?? "" }) + } else if (content.type === "toolCall") { + const toolCallId = stringValue(content.id) ?? "" + if (!pairedToolCallIds.has(toolCallId)) continue + parts.push({ + type: "tool-call", + toolCallId, + toolName: stringValue(content.name) ?? "", + input: recordOrEmpty(content.arguments), + }) + } + } + if (parts.length > 0) out.push({ role: "assistant", content: parts }) + } else if (message.role === "toolResult") { + if (!message.toolCallId || !pairedToolCallIds.has(message.toolCallId)) continue + out.push({ + role: "tool", + content: [ + { + type: "tool-result", + toolCallId: message.toolCallId, + toolName: message.toolName, + output: message.isError + ? { type: "error-text", value: textContent(message) } + : { type: "text", value: textContent(message) }, + }, + ], + }) + + const images = imageParts(message.content) + if (images.length > 0) { + if (!allowImages) throw imageContentError("tool results") + out.push({ + role: "user", + content: images.map(imageToCommandCode), + }) + } + } + } + return out +} + +export function parseStreamEventLine(line: string): unknown | undefined { + let trimmed = line.trim() + if (!trimmed || trimmed.startsWith(":") || trimmed.startsWith("event:")) return undefined + if (trimmed.startsWith("data:")) trimmed = trimmed.slice(5).trim() + if (!trimmed || trimmed === "[DONE]") return undefined + + try { + const parsed: unknown = JSON.parse(trimmed) + return parsed + } catch { + return undefined + } +} + +export function mapFinishReason(reason: unknown): StopReason { + if (reason === "tool-calls") return "toolUse" + if ( + reason === "length" || + reason === "max_tokens" || + reason === "max-tokens" || + reason === "max_output_tokens" + ) { + return "length" + } + return "stop" +} + +function promptPartToText(value: unknown, depth = 0): string { + if (depth > 10) return "" + if (typeof value === "string") return value + if (Array.isArray(value)) + return value + .map((v) => promptPartToText(v, depth + 1)) + .filter(Boolean) + .join("\n") + if (!isRecord(value)) return "" + const text = stringValue(value.text) + if (text) return text + const content = promptPartToText(value.content, depth + 1) + if (content) return content + return "" +} + +export function systemPromptToText(value: unknown): string { + if (value === undefined || value === null) return "" + if (typeof value === "string") return value + if (Array.isArray(value)) + return value + .map((v) => promptPartToText(v, 0)) + .filter(Boolean) + .join("\n\n") + return promptPartToText(value, 0) +} diff --git a/src/core.ts b/src/core.ts new file mode 100644 index 0000000..f1898dd --- /dev/null +++ b/src/core.ts @@ -0,0 +1,741 @@ +/** + * Testable Command Code provider core. + * + * The runtime imports live in index.ts; this module takes injected stream/cost + * dependencies so tests can exercise the real serialization and stream parser. + */ + +import { randomUUID } from "node:crypto" + +import { commandCodeErrorMessage, redactCommandCodeErrorText } from "./overflow.ts" +import { modelSupportsImageInput } from "./models.ts" +import { + getApiKey, + getEnvironmentInfo, + isRecord, + assertTextOnlyMessages, + mapFinishReason, + messagesToCC, + numberValue, + parseStreamEventLine, + recordOrEmpty, + stringValue, + toolsToJson, + systemPromptToText, +} from "./converters.ts" +import type { + AssistantMessageEventStreamLike, + AssistantMessageLike, + ContextLike, + CoreDependencies, + ErrorReason, + ModelLike, + StopReason, + StreamOptions, + TerminalReason, + TextContent, + ToolCallContent, + Usage, +} from "./types.ts" + +export * from "./converters.ts" +export * from "./overflow.ts" +export * from "./types.ts" + +export const DEFAULT_API_BASE = "https://api.commandcode.ai" +export const COMMAND_CODE_CLI_VERSION = "1.15.1" + +const DEFAULT_GENERATE_MAX_TOKENS = 64_000 +const DEFAULT_MAX_RETRIES = 0 +const DEFAULT_MAX_RETRY_DELAY_MS = 60_000 +const BASE_RETRY_DELAY_MS = 500 + +function isRetryableStatus(status: number): boolean { + return status === 429 || (status >= 500 && status < 600) +} + +function parseRetryAfterSeconds(value: string | null): number | undefined { + if (!value) return undefined + const seconds = Number(value) + if (Number.isFinite(seconds) && seconds >= 0) return seconds + const date = Date.parse(value) + if (!Number.isNaN(date)) return Math.max(0, (date - Date.now()) / 1000) + return undefined +} + +function effectiveMaxRetryDelayMs(value: number | undefined): number { + if (value === undefined) return DEFAULT_MAX_RETRY_DELAY_MS + if (value === 0) return Number.POSITIVE_INFINITY + return value +} + +function retryDelayMs( + attempt: number, + retryAfterHeader: string | null, + maxDelayMs: number, +): number { + const retryAfterMs = parseRetryAfterSeconds(retryAfterHeader) + if (retryAfterMs !== undefined) { + if (retryAfterMs * 1000 > maxDelayMs) return -1 + return retryAfterMs * 1000 + } + const exponential = BASE_RETRY_DELAY_MS * 2 ** attempt + const jitter = exponential * 0.2 * Math.random() + return Math.min(exponential + jitter, maxDelayMs) +} + +function defaultUsage(): Usage { + return { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + } +} + +function commandCodeUsage(event: Record): Record | undefined { + return isRecord(event.totalUsage) ? event.totalUsage : undefined +} + +function commandCodeInputTokenDetails( + usage: Record, +): Record | undefined { + return isRecord(usage.inputTokenDetails) ? usage.inputTokenDetails : undefined +} + +function headersToRecord(headers: Headers): Record { + const out: Record = {} + headers.forEach((value, key) => { + out[key] = value + }) + return out +} + +function abortError(message = "The operation was aborted"): DOMException { + return new DOMException(message, "AbortError") +} + +function timeoutError(timeoutMs: number | undefined): Error { + return new Error( + timeoutMs === undefined + ? "Command Code API request timed out" + : `Command Code API request timed out after ${timeoutMs}ms`, + ) +} + +function successStopReason(reason: TerminalReason): StopReason { + if (reason === "length" || reason === "toolUse") return reason + return "stop" +} + +function generateMaxTokens(model: ModelLike, options?: StreamOptions): number { + return Math.min( + options?.maxTokens ?? model.maxTokens, + model.maxTokens, + DEFAULT_GENERATE_MAX_TOKENS, + ) +} + +function mappedReasoningEffort(model: ModelLike, options?: StreamOptions): string | undefined { + const level = options?.reasoning + if (!level || level === "off" || !model.reasoning) return undefined + + const effortMap = model.thinking?.effortMap ?? model.thinkingLevelMap + const mapped = effortMap?.[level] + return typeof mapped === "string" && mapped !== "off" ? mapped : undefined +} + +export function projectSlugFromPath(pathName: string): string { + const slug = pathName + .toLowerCase() + .replace(/^[a-z]:/i, "") + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + return slug || "project" +} + +export function createStreamCommandCode(deps: CoreDependencies) { + const apiBase = deps.apiBase ?? DEFAULT_API_BASE + const fetchImpl = deps.fetchImpl ?? fetch + const cwd = deps.cwd ?? (() => process.cwd()) + const now = deps.now ?? (() => Date.now()) + const uuid = deps.uuid ?? (() => randomUUID()) + const delay = + deps.delay ?? + ((ms: number, signal: AbortSignal) => { + if (signal.aborted) return Promise.reject(abortError()) + return new Promise((resolve, reject) => { + const id = setTimeout(() => { + signal.removeEventListener("abort", onAbort) + resolve() + }, ms) + const onAbort = () => { + clearTimeout(id) + reject(abortError()) + } + signal.addEventListener("abort", onAbort, { once: true }) + }) + }) + + function raceAbort(promise: Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(abortError()) + + return new Promise((resolve, reject) => { + const onAbort = () => reject(abortError()) + signal.addEventListener("abort", onAbort, { once: true }) + promise.then( + (value) => { + signal.removeEventListener("abort", onAbort) + resolve(value) + }, + (error: unknown) => { + signal.removeEventListener("abort", onAbort) + reject(error) + }, + ) + }) + } + + function raceAbortWithTimeout( + promise: Promise, + controller: AbortController, + timeoutMs: number | undefined, + ): Promise { + if (timeoutMs === undefined) return raceAbort(promise, controller.signal) + + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + controller.abort() + reject(timeoutError(timeoutMs)) + }, timeoutMs) + raceAbort(promise, controller.signal).then( + (value) => { + clearTimeout(timer) + resolve(value) + }, + (error: unknown) => { + clearTimeout(timer) + reject(error) + }, + ) + }) + } + + return function streamCommandCode( + model: ModelLike, + context: ContextLike, + options?: StreamOptions, + ): AssistantMessageEventStreamLike { + const stream = deps.createStream() + + async function run() { + // OMP may pass the legacy env-var name "COMMANDCODE_API_KEY" (old pi) + // or "$COMMANDCODE_API_KEY" (new pi) as the apiKey value instead of + // resolving it. Filter out these specific strings. + const LEGACY_API_KEY_REF = "$COMMANDCODE_API_KEY" + const OLD_API_KEY_REF = "COMMANDCODE_API_KEY" + const hostKey = + options?.apiKey && + options.apiKey !== LEGACY_API_KEY_REF && + options.apiKey !== OLD_API_KEY_REF + ? options.apiKey + : undefined + + const apiKey = + hostKey ?? + getApiKey({ + env: deps.env, + authPaths: deps.authPaths, + homeDir: deps.homeDir, + }) + + if (!apiKey) { + const msg: AssistantMessageLike = { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: defaultUsage(), + stopReason: "error", + errorMessage: + "No Command Code API key. Run /login and select Command Code, set the COMMANDCODE_API_KEY env var, or configure ~/.commandcode/auth.json, ~/.pi/agent/auth.json or ~/.omp/agent/auth.json", + timestamp: now(), + } + stream.push({ type: "error", reason: "error", error: msg }) + stream.end() + return + } + + const output: AssistantMessageLike = { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: defaultUsage(), + stopReason: "stop", + timestamp: now(), + } + + const controller = new AbortController() + let reader: ReadableStreamDefaultReader | undefined + let textBlock: TextContent | undefined + let currentTextIdx = -1 + let thinkingIdx = -1 + let finished = false + + const abortUpstream = () => { + if (!controller.signal.aborted) controller.abort() + try { + reader?.cancel().catch(() => undefined) + } catch { + // Reader cancellation is best-effort. + } + } + + if (options?.signal?.aborted) { + abortUpstream() + } else { + options?.signal?.addEventListener("abort", abortUpstream, { + once: true, + }) + } + + const endTextBlock = () => { + if (!textBlock) return + stream.push({ + type: "text_end", + contentIndex: currentTextIdx, + content: textBlock.text, + partial: output, + }) + textBlock = undefined + currentTextIdx = -1 + } + + const endThinking = () => { + if (thinkingIdx < 0) return + const tc = output.content[thinkingIdx] + if (tc && tc.type === "thinking") { + stream.push({ + type: "thinking_end", + contentIndex: thinkingIdx, + content: (tc as { thinking: string }).thinking, + partial: output, + }) + } + thinkingIdx = -1 + } + + const handleEvent = (event: unknown) => { + if (!isRecord(event)) return + + switch (event.type) { + case "text-delta": { + endThinking() + if (!textBlock) { + textBlock = { type: "text", text: "" } + output.content.push(textBlock) + currentTextIdx = output.content.length - 1 + stream.push({ + type: "text_start", + contentIndex: currentTextIdx, + partial: output, + }) + } + const delta = stringValue(event.text) ?? "" + textBlock.text += delta + stream.push({ + type: "text_delta", + contentIndex: currentTextIdx, + delta, + partial: output, + }) + break + } + + case "reasoning-start": { + endTextBlock() + break + } + + case "reasoning-delta": { + endTextBlock() + const delta = stringValue(event.text) ?? "" + if (thinkingIdx < 0) { + output.content.push({ type: "thinking", thinking: delta }) + thinkingIdx = output.content.length - 1 + stream.push({ + type: "thinking_start", + contentIndex: thinkingIdx, + partial: output, + }) + } else { + const tc = output.content[thinkingIdx] + if (tc && tc.type === "thinking") { + ;(tc as { thinking: string }).thinking += delta + } + } + stream.push({ + type: "thinking_delta", + contentIndex: thinkingIdx, + delta, + partial: output, + }) + break + } + + case "reasoning-end": { + endThinking() + break + } + + case "tool-result": { + break + } + + case "tool-call": { + endTextBlock() + endThinking() + const toolCall: ToolCallContent = { + type: "toolCall", + id: stringValue(event.toolCallId) ?? "", + name: stringValue(event.toolName) ?? "", + arguments: recordOrEmpty(event.input ?? event.args ?? event.arguments), + } + output.content.push(toolCall) + const idx = output.content.length - 1 + stream.push({ + type: "toolcall_start", + contentIndex: idx, + partial: output, + }) + stream.push({ + type: "toolcall_end", + contentIndex: idx, + toolCall, + partial: output, + }) + break + } + + case "finish": { + const usage = commandCodeUsage(event) + if (usage) { + const details = commandCodeInputTokenDetails(usage) + const totalInput = numberValue(usage.inputTokens) ?? 0 + const input = numberValue(details?.noCacheTokens) + const cacheRead = numberValue(details?.cacheReadTokens) ?? 0 + const cacheWrite = numberValue(details?.cacheWriteTokens) ?? 0 + output.usage.input = input ?? Math.max(0, totalInput - cacheRead - cacheWrite) + output.usage.output = numberValue(usage.outputTokens) ?? 0 + output.usage.cacheRead = cacheRead + output.usage.cacheWrite = cacheWrite + output.usage.totalTokens = + output.usage.input + + output.usage.output + + output.usage.cacheRead + + output.usage.cacheWrite + deps.calculateCost(model, output.usage) + } + output.stopReason = mapFinishReason(event.finishReason) + finished = true + break + } + + case "error": { + const message = + commandCodeErrorMessage(event.error) ?? + commandCodeErrorMessage(event.message) ?? + "Stream error" + output.stopReason = "error" + output.errorMessage = message + throw new Error(message) + } + } + } + + try { + stream.push({ type: "start", partial: output }) + if (controller.signal.aborted) throw abortError("Aborted") + + const workingDir = cwd() + const threadId = uuid() + const reasoningEffort = mappedReasoningEffort(model, options) + const timeoutMs = options?.timeoutMs + + const allowImages = modelSupportsImageInput(model.id) + if (!allowImages) assertTextOnlyMessages(context.messages) + + let body: unknown = { + config: { + workingDir, + date: new Date(now()).toISOString().split("T")[0], + environment: getEnvironmentInfo(), + structure: [], + isGitRepo: false, + currentBranch: "", + mainBranch: "", + gitStatus: "", + recentCommits: [], + }, + memory: null, + taste: null, + skills: null, + params: { + model: model.id, + messages: messagesToCC(context.messages, { allowImages }), + tools: toolsToJson(context.tools), + system: systemPromptToText(context.systemPrompt), + max_tokens: generateMaxTokens(model, options), + temperature: 0.3, + stream: true, + ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}), + }, + threadId, + } + + const payloadController = new AbortController() + const onPayloadAbort = () => payloadController.abort() + controller.signal.addEventListener("abort", onPayloadAbort, { once: true }) + let nextBody: unknown + try { + nextBody = await raceAbortWithTimeout( + Promise.resolve(options?.onPayload?.(body, model)), + payloadController, + timeoutMs, + ) + } finally { + controller.signal.removeEventListener("abort", onPayloadAbort) + } + if (nextBody !== undefined) body = nextBody + + const maxRetries = options?.maxRetries ?? DEFAULT_MAX_RETRIES + const maxRetryDelayMs = effectiveMaxRetryDelayMs(options?.maxRetryDelayMs) + const requestHeaders = { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + "x-command-code-version": COMMAND_CODE_CLI_VERSION, + "x-cli-environment": "production", + "x-project-slug": projectSlugFromPath(workingDir), + "x-taste-learning": "true", + "x-co-flag": "false", + ...options?.headers, + } + const bodyStr = JSON.stringify(body) + + let response!: Response + retryLoop: for (let attempt = 0; ; attempt++) { + const attemptController = new AbortController() + let attemptTimedOut = false + let attemptTimeoutId: ReturnType | undefined + + const clearAttemptTimeout = () => { + if (attemptTimeoutId !== undefined) { + clearTimeout(attemptTimeoutId) + attemptTimeoutId = undefined + } + } + + if (timeoutMs !== undefined) { + attemptTimeoutId = setTimeout(() => { + attemptTimedOut = true + attemptController.abort() + }, timeoutMs) + } + const onOuterAbort = () => attemptController.abort() + controller.signal.addEventListener("abort", onOuterAbort, { once: true }) + const raceAttempt = (promise: Promise): Promise => + raceAbort(promise, attemptController.signal).catch((error: unknown) => { + if (attemptTimedOut) throw timeoutError(timeoutMs) + throw error + }) + + try { + try { + response = await fetchImpl(`${apiBase}/alpha/generate`, { + method: "POST", + headers: requestHeaders, + body: bodyStr, + signal: attemptController.signal, + }) + } catch (fetchError: unknown) { + if (controller.signal.aborted) throw abortError("Aborted") + if (attemptTimedOut) { + if (attempt < maxRetries) continue retryLoop + throw timeoutError(timeoutMs) + } + throw fetchError + } + + // --- HTTP-level retry --- + if (!response.ok && isRetryableStatus(response.status)) { + const retryAfter = response.headers.get("retry-after") + const waitMs = retryDelayMs(attempt, retryAfter, maxRetryDelayMs) + if (waitMs < 0) { + const requestedSeconds = parseRetryAfterSeconds(retryAfter) ?? 0 + const capLabel = + maxRetryDelayMs === Number.POSITIVE_INFINITY ? "disabled" : `${maxRetryDelayMs}ms` + throw new Error(`Retry-After delay ${requestedSeconds}s exceeds max ${capLabel}`) + } + if (attempt < maxRetries) { + await response.text().catch(() => "") + if (waitMs > 0) await delay(waitMs, controller.signal) + continue retryLoop + } + } + + try { + await raceAttempt( + Promise.resolve( + options?.onResponse?.( + { + status: response.status, + headers: headersToRecord(response.headers), + }, + model, + ), + ), + ) + } catch (error: unknown) { + if (attemptTimedOut && attempt < maxRetries) continue retryLoop + throw error + } + + if (!response.ok) { + const errBody = await raceAttempt(response.text().catch(() => "")) + let errorDetail: string | undefined + try { + const parsedBody: unknown = JSON.parse(errBody) + errorDetail = commandCodeErrorMessage(parsedBody) + } catch { + // Preserve useful plain-text provider errors only after secret + // redaction; upstream/proxy bodies may echo credentials. + } + const safeBody = redactCommandCodeErrorText(errBody).slice(0, 500) + const detail = redactCommandCodeErrorText( + errorDetail ?? (safeBody || "Provider returned an error"), + ) + throw new Error(`Command Code API error ${response.status}: ${detail}`) + } + + // --- Read response stream --- + reader = response.body?.getReader() + if (!reader) throw new Error("No response body") + + const decoder = new TextDecoder() + let buffer = "" + + try { + readLoop: for (;;) { + if (controller.signal.aborted) throw abortError("Aborted") + const { done, value } = await raceAbort(reader.read(), attemptController.signal) + if (done) { + if (buffer.trim()) handleEvent(parseStreamEventLine(buffer)) + break + } + if (controller.signal.aborted) throw abortError("Aborted") + + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split("\n") + buffer = lines.pop() ?? "" + + for (const line of lines) { + if (controller.signal.aborted) throw abortError("Aborted") + handleEvent(parseStreamEventLine(line)) + if (finished) break readLoop + } + } + } catch (streamError: unknown) { + // Stream-level error (e.g. API returned 200 OK but sent an error event) + // or per-attempt timeout during stream reading. + await reader.cancel().catch(() => {}) + try { + reader.releaseLock() + } catch {} + reader = undefined + + if (controller.signal.aborted) throw streamError + + // Never retry after visible content was emitted (including timeout mid-stream). + const canRetry = output.content.length === 0 && attempt < maxRetries + if (canRetry) { + output.content.length = 0 + textBlock = undefined + currentTextIdx = -1 + thinkingIdx = -1 + output.stopReason = "stop" + output.errorMessage = undefined + finished = false + const waitMs = attemptTimedOut ? 0 : retryDelayMs(attempt, null, maxRetryDelayMs) + if (waitMs > 0) await delay(waitMs, controller.signal) + continue retryLoop + } + if (attemptTimedOut) throw timeoutError(timeoutMs) + throw streamError + } + + // Stream completed successfully. + endTextBlock() + endThinking() + + stream.push({ + type: "done", + reason: successStopReason(output.stopReason), + message: output, + }) + stream.end() + break retryLoop + } finally { + controller.signal.removeEventListener("abort", onOuterAbort) + clearAttemptTimeout() + } + } + } catch (error: unknown) { + const reason: ErrorReason = controller.signal.aborted ? "aborted" : "error" + output.stopReason = reason + output.errorMessage = + reason === "aborted" + ? "Request aborted" + : redactCommandCodeErrorText(error instanceof Error ? error.message : String(error)) + stream.push({ type: "error", reason, error: output }) + stream.end() + } finally { + options?.signal?.removeEventListener("abort", abortUpstream) + try { + await reader?.cancel() + } catch { + // Reader may already be closed/cancelled. + } + try { + reader?.releaseLock() + } catch { + // Reader may already be released/cancelled by the abort path. + } + } + } + + run().catch((error: unknown) => { + const msg: AssistantMessageLike = { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: defaultUsage(), + stopReason: "error", + errorMessage: redactCommandCodeErrorText( + error instanceof Error ? error.message : String(error), + ), + timestamp: now(), + } + stream.push({ type: "error", reason: "error", error: msg }) + stream.end() + }) + + return stream + } +} diff --git a/src/cost.ts b/src/cost.ts new file mode 100644 index 0000000..55c1bd0 --- /dev/null +++ b/src/cost.ts @@ -0,0 +1,31 @@ +/** + * Local cost calculation for Command Code usage. + * + * Mirrors pi-ai's `calculateCost` arithmetic exactly. The provider ships its + * own copy because Oh My Pi's legacy pi-ai shim does not export + * `calculateCost`, which broke extension installation there (issue #24). + * `tests/test-cost.ts` locks this implementation to the pi-ai original. + */ + +import type { ModelLike, Usage } from "./types.ts" + +export function calculateCommandCodeCost(model: ModelLike, usage: Usage): void { + const inputTokens = usage.input + usage.cacheRead + usage.cacheWrite + let rates = model.cost + let matchedThreshold = -1 + for (const tier of model.cost.tiers ?? []) { + if (inputTokens > tier.inputTokensAbove && tier.inputTokensAbove > matchedThreshold) { + rates = tier + matchedThreshold = tier.inputTokensAbove + } + } + + const longWrite = usage.cacheWrite1h ?? 0 + const shortWrite = usage.cacheWrite - longWrite + usage.cost.input = (rates.input / 1_000_000) * usage.input + usage.cost.output = (rates.output / 1_000_000) * usage.output + usage.cost.cacheRead = (rates.cacheRead / 1_000_000) * usage.cacheRead + usage.cost.cacheWrite = (rates.cacheWrite * shortWrite + rates.input * 2 * longWrite) / 1_000_000 + usage.cost.total = + usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite +} diff --git a/src/json-schema.ts b/src/json-schema.ts new file mode 100644 index 0000000..a2ffc02 --- /dev/null +++ b/src/json-schema.ts @@ -0,0 +1,382 @@ +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined +} + +function booleanValue(value: unknown): boolean | undefined { + return typeof value === "boolean" ? value : undefined +} + +type JsonSchemaValue = boolean | Record + +const JSON_SCHEMA_TYPES = new Set([ + "array", + "boolean", + "integer", + "null", + "number", + "object", + "string", +]) + +const LEGACY_KINDS = new Set([ + "any", + "array", + "boolean", + "enum", + "integer", + "intersect", + "intersection", + "literal", + "never", + "null", + "nullable", + "number", + "object", + "optional", + "string", + "undefined", + "union", + "unknown", +]) + +const LEGACY_FIELDS = new Set([ + "element", + "kind", + "inner", + "optional", + "value", + "values", + "variants", + "wrapped", +]) + +const SCHEMA_MAP_FIELDS = new Set([ + "$defs", + "definitions", + "dependentSchemas", + "patternProperties", + "properties", +]) + +const SCHEMA_ARRAY_FIELDS = new Set(["allOf", "anyOf", "oneOf", "prefixItems"]) + +const SCHEMA_VALUE_FIELDS = new Set([ + "additionalItems", + "additionalProperties", + "contains", + "contentSchema", + "else", + "if", + "items", + "not", + "propertyNames", + "then", + "unevaluatedItems", + "unevaluatedProperties", +]) + +const SCHEMA_KEYWORDS = new Set([ + "$anchor", + "$comment", + "$defs", + "$dynamicAnchor", + "$dynamicRef", + "$id", + "$ref", + "$schema", + "$vocabulary", + "additionalItems", + "additionalProperties", + "allOf", + "anyOf", + "const", + "contains", + "contentEncoding", + "contentMediaType", + "contentSchema", + "default", + "definitions", + "dependentRequired", + "dependentSchemas", + "description", + "else", + "enum", + "examples", + "exclusiveMaximum", + "exclusiveMinimum", + "format", + "if", + "items", + "maxContains", + "maxItems", + "maxLength", + "maxProperties", + "maximum", + "minContains", + "minItems", + "minLength", + "minProperties", + "minimum", + "multipleOf", + "not", + "oneOf", + "pattern", + "patternProperties", + "prefixItems", + "properties", + "propertyNames", + "readOnly", + "required", + "title", + "type", + "unevaluatedItems", + "unevaluatedProperties", + "uniqueItems", + "writeOnly", +]) + +function stringArray(value: unknown): string[] | undefined { + if (!Array.isArray(value)) return undefined + const values = value.filter((item): item is string => typeof item === "string") + return values.length === value.length ? values : undefined +} + +function validSchemaType(value: unknown): boolean { + if (typeof value === "string") return JSON_SCHEMA_TYPES.has(value) + if (!Array.isArray(value) || value.length === 0) return false + return value.every((item) => typeof item === "string" && JSON_SCHEMA_TYPES.has(item)) +} + +function legacyKind(schema: Record): string | undefined { + const explicitKind = stringValue(schema.kind)?.toLowerCase() + if (explicitKind && LEGACY_KINDS.has(explicitKind)) return explicitKind + + const type = stringValue(schema.type) + const normalized = type?.toLowerCase() + if (!normalized || !LEGACY_KINDS.has(normalized)) return undefined + if (!validSchemaType(type) || Object.keys(schema).some((key) => LEGACY_FIELDS.has(key))) { + return normalized + } + return undefined +} + +function looksLikeJsonSchema(schema: Record): boolean { + if (Object.keys(schema).length === 0) return true + if (schema.type !== undefined && !validSchemaType(schema.type)) return false + return Object.keys(schema).some((key) => SCHEMA_KEYWORDS.has(key)) +} + +function isOptionalSchema(schema: unknown): boolean { + if (!isRecord(schema)) return false + if (booleanValue(schema.optional) === true) return true + + const kind = legacyKind(schema) + if (kind === "optional") return true + if (kind !== "union") return false + + const variants = Array.isArray(schema.variants) + ? schema.variants + : Array.isArray(schema.anyOf) + ? schema.anyOf + : [] + return variants.some((variant) => legacyKind(isRecord(variant) ? variant : {}) === "undefined") +} + +function schemaValue(value: unknown, seen: WeakSet): JsonSchemaValue { + if (typeof value === "boolean") return value + if (!isRecord(value)) return {} + return convertSchema(value, seen) +} + +function setSchemaProperty(target: Record, key: string, value: unknown): void { + Object.defineProperty(target, key, { + configurable: true, + enumerable: true, + value, + writable: true, + }) +} + +function schemaMap(value: unknown, seen: WeakSet): Record { + if (!isRecord(value)) return {} + const out: Record = {} + for (const [key, item] of Object.entries(value)) { + setSchemaProperty(out, key, schemaValue(item, seen)) + } + return out +} + +function schemaArray(value: unknown, seen: WeakSet): unknown[] { + if (!Array.isArray(value)) return [] + return value.map((item) => schemaValue(item, seen)) +} + +function isSchemaValue(value: unknown): value is JsonSchemaValue { + return typeof value === "boolean" || isRecord(value) +} + +function copySchemaObject( + source: Record, + seen: WeakSet, + legacy: boolean, + forcedType?: string, +): JsonSchemaValue { + const out: Record = {} + + for (const [key, value] of Object.entries(source)) { + if (legacy && LEGACY_FIELDS.has(key)) continue + if (key === "nullable" || (forcedType !== undefined && key === "type")) continue + + if (key === "required") { + const required = stringArray(value) + if (required) out.required = required + } else if (SCHEMA_MAP_FIELDS.has(key)) { + out[key] = schemaMap(value, seen) + } else if (SCHEMA_ARRAY_FIELDS.has(key)) { + out[key] = schemaArray(value, seen) + } else if (SCHEMA_VALUE_FIELDS.has(key)) { + out[key] = + Array.isArray(value) && key === "items" + ? schemaArray(value, seen) + : schemaValue(value, seen) + } else { + out[key] = value + } + } + + if (forcedType !== undefined) out.type = forcedType + if (booleanValue(source.nullable) === true) return makeNullable(out) + return out +} + +function makeNullable(schema: Record): Record { + const type = schema.type + if (typeof type === "string") { + if (type === "null") return schema + return { ...schema, type: [type, "null"] } + } + if (Array.isArray(type) && !type.includes("null")) { + return { ...schema, type: [...type, "null"] } + } + if (Array.isArray(schema.anyOf)) { + return { ...schema, anyOf: [...schema.anyOf, { type: "null" }] } + } + return { anyOf: [schema, { type: "null" }] } +} + +function legacyVariants(schema: Record): unknown[] { + if (Array.isArray(schema.variants)) return schema.variants + if (Array.isArray(schema.anyOf)) return schema.anyOf + return [] +} + +function convertLegacySchema( + source: Record, + kind: string, + seen: WeakSet, +): JsonSchemaValue { + if (kind === "optional") return schemaValue(source.wrapped ?? source.inner, seen) + if (kind === "nullable") { + const wrapped = schemaValue(source.wrapped ?? source.inner, seen) + return typeof wrapped === "boolean" ? wrapped : makeNullable(wrapped) + } + if (kind === "undefined" || kind === "never" || kind === "any" || kind === "unknown") return {} + + if (kind === "union" || kind === "intersect" || kind === "intersection") { + const variants = legacyVariants(source) + .map((variant) => schemaValue(variant, seen)) + .filter( + (variant) => + isSchemaValue(variant) && + (typeof variant === "boolean" || Object.keys(variant).length > 0), + ) + if (variants.length === 0) return copySchemaObject(source, seen, true) + if (variants.length === 1) return variants[0] ?? {} + + const out = copySchemaObject(source, seen, true) + if (typeof out !== "boolean") out[kind === "union" ? "anyOf" : "allOf"] = variants + return out + } + + if (kind === "object") { + const converted = copySchemaObject(source, seen, true, "object") + if (typeof converted === "boolean") return converted + const out = converted + const sourceProperties = isRecord(source.properties) ? source.properties : undefined + if (!sourceProperties) return out + + const properties: Record = {} + const optional = stringArray(source.optional) ?? [] + for (const [key, value] of Object.entries(sourceProperties)) { + setSchemaProperty(properties, key, schemaValue(value, seen)) + } + out.properties = properties + + const explicitRequired = stringArray(source.required) + const required = + explicitRequired ?? + Object.entries(sourceProperties) + .filter(([key, value]) => !optional.includes(key) && !isOptionalSchema(value)) + .map(([key]) => key) + if (required.length > 0) out.required = required + else delete out.required + return out + } + + if (kind === "array") { + const converted = copySchemaObject(source, seen, true, "array") + if (typeof converted === "boolean") return converted + const out = converted + if (!("items" in source) && "element" in source) out.items = schemaValue(source.element, seen) + return out + } + + if (kind === "enum") { + const converted = copySchemaObject(source, seen, true) + if (typeof converted === "boolean") return converted + const out = converted + if (!("enum" in out) && Array.isArray(source.values)) out.enum = source.values + return out + } + + if (kind === "literal") { + const converted = copySchemaObject(source, seen, true) + if (typeof converted === "boolean") return converted + const out = converted + if (!("const" in out) && "value" in source) out.const = source.value + return out + } + + const scalarType = + kind === "string" || + kind === "number" || + kind === "boolean" || + kind === "integer" || + kind === "null" + ? kind + : undefined + return scalarType ? copySchemaObject(source, seen, true, scalarType) : {} +} + +function convertSchema(source: Record, seen: WeakSet): JsonSchemaValue { + if (seen.has(source)) return {} + seen.add(source) + try { + const kind = legacyKind(source) + if (kind) return convertLegacySchema(source, kind, seen) + if (!looksLikeJsonSchema(source)) return {} + return copySchemaObject(source, seen, false) + } finally { + seen.delete(source) + } +} + +export function toJsonSchema(schema: unknown): unknown { + if (typeof schema === "boolean") return schema + if (!isRecord(schema)) return {} + return convertSchema(schema, new WeakSet()) +} diff --git a/src/overflow.ts b/src/overflow.ts new file mode 100644 index 0000000..3e3e106 --- /dev/null +++ b/src/overflow.ts @@ -0,0 +1,120 @@ +const COMMAND_CODE_PROVIDER = "commandcode" +const CONTEXT_OVERFLOW_PREFIX = "context_length_exceeded:" + +const COMMAND_CODE_OVERFLOW_PATTERNS = [ + /\b(?:context[_\s-]*(?:length|window)|model[_\s-]*context[_\s-]*window)[_\s-]*(?:exceeded|overflow(?:ed)?|too[_\s-]*(?:large|long))\b/i, + /\b(?:context|prompt|input)[_\s-]*(?:length|window|size|tokens?|limit|maximum)\b[\s\S]{0,120}\b(?:exceed(?:ed|s)?|overflow(?:ed|s)?|too\s+(?:large|long)|(?:maximum|limit)\s+(?:reached|exceeded|hit))\b/i, + /\b(?:exceed(?:ed|s)?|overflow(?:ed|s)?|too\s+(?:large|long))\b[\s\S]{0,120}\b(?:context|prompt|input)[_\s-]*(?:length|window|size|tokens?|limit|maximum)\b/i, + /\b(?:prompt|input|context)\b[\s\S]{0,32}\btoo\s+(?:large|long)\b/i, + /\b(?:prompt|input)[_\s-]*too[_\s-]*(?:large|long)\b/i, + /\b(?:prompt|input)[_\s-]*tokens?[_\s-]*(?:limit|maximum|max)[_\s-]*(?:exceeded|reached)\b/i, + /\b(?:prompt|input)[_\s-]*(?:tokens?|length|size)\b[\s\S]{0,120}\b(?:limit|maximum)\b[\s\S]{0,40}\b(?:exceed(?:ed|s)?|reached|hit)\b/i, + /\b(?:maximum|limit)[_\s-]+(?:allowed[_\s-]+)?(?:context|prompt|input)[_\s-]*(?:length|window|size|tokens?)\b/i, +] + +const NON_OVERFLOW_PATTERNS = [ + /\brate[_\s-]*limit\b/i, + /\btoo\s+many\s+requests\b/i, + /\b(?:capacity|quota|throttl(?:e|ed|ing)?|concurren(?:cy|t)|overloaded)\b/i, + /\b(?:service|temporarily)\s+unavailable\b/i, + /\bstatus(?:[_\s-]*code)?\s*[:=]\s*429\b/i, +] + +const CONTEXT_OVERFLOW_PREFIX_PATTERN = /context_length_exceeded/i + +const HTTP_RATE_LIMIT_STATUS_PATTERNS = [ + /\b(?:api\s+error|http|status(?:[_\s-]*code)?|status[_\s-]*code)\s*[:(]?\s*429\b/i, + /["']?(?:status|status[_\s-]*code)["']?\s*:\s*429\b/i, +] + +const BEARER_PATTERN = /\bBearer\s+[A-Za-z0-9._~+/=-]+/gi +const CREDENTIAL_PATTERN = + /\b(?:api[-_ ]?key|apikey|access[-_ ]?token|refresh[-_ ]?token|token|secret|password|authorization)\s*[=:]\s*[^\s,;)]+/gi +const USER_TOKEN_PATTERN = /\b(?:user|cc)_[A-Za-z0-9_-]{8,}\b/gi +const QUERY_SECRET_PATTERN = + /([?&](?:api[-_ ]?key|apikey|access_token|refresh_token|token|secret|password)=)[^&#\s]+/gi +const STANDALONE_SECRET_PATTERN = + /\b(?:sk|rk|ghp|github_pat|xox[baprs])[-_A-Za-z0-9]{16,}\b|\beyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g + +export function redactCommandCodeErrorText(value: string): string { + return value + .replace(BEARER_PATTERN, "Bearer [redacted]") + .replace(CREDENTIAL_PATTERN, (match) => { + const separatorIndex = match.search(/[=:]/) + return separatorIndex < 0 ? "[redacted]" : `${match.slice(0, separatorIndex + 1)}[redacted]` + }) + .replace(USER_TOKEN_PATTERN, "[redacted]") + .replace(QUERY_SECRET_PATTERN, "$1[redacted]") + .replace(STANDALONE_SECRET_PATTERN, "[redacted]") +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null +} + +export interface CommandCodeMessageLike { + role: string + provider: string + stopReason: string + errorMessage?: string +} + +export function commandCodeErrorMessage(value: unknown): string | undefined { + if (typeof value === "string") return value + if (!isRecord(value)) return undefined + + const record = value + const parts: string[] = [] + for (const key of [ + "message", + "errorMessage", + "error", + "detail", + "details", + "code", + "type", + "reason", + ]) { + const part = commandCodeErrorMessage(record[key]) + if (part && !parts.includes(part)) parts.push(part) + } + + for (const key of ["status", "statusCode", "httpStatus"]) { + const status = record[key] + if (typeof status === "string" || typeof status === "number") { + const statusPart = `status: ${status}` + if (!parts.includes(statusPart)) parts.push(statusPart) + } + } + + return parts.length > 0 ? redactCommandCodeErrorText(parts.join(": ")) : undefined +} + +export function normalizeCommandCodeErrorMessage( + errorMessage: string | undefined, +): string | undefined { + if (!errorMessage) return undefined + if (CONTEXT_OVERFLOW_PREFIX_PATTERN.test(errorMessage)) return undefined + if (NON_OVERFLOW_PATTERNS.some((pattern) => pattern.test(errorMessage))) return undefined + if (HTTP_RATE_LIMIT_STATUS_PATTERNS.some((pattern) => pattern.test(errorMessage))) + return undefined + if (!COMMAND_CODE_OVERFLOW_PATTERNS.some((pattern) => pattern.test(errorMessage))) + return undefined + + return `${CONTEXT_OVERFLOW_PREFIX} ${errorMessage}` +} + +export function normalizeCommandCodeMessage( + message: T, + modelProvider?: string, +): { message: T & { errorMessage: string } } | undefined { + if (message.role !== "assistant" || message.stopReason !== "error") return undefined + if (message.provider !== COMMAND_CODE_PROVIDER && modelProvider !== COMMAND_CODE_PROVIDER) { + return undefined + } + + const errorMessage = normalizeCommandCodeErrorMessage(message.errorMessage) + if (!errorMessage) return undefined + + return { message: { ...message, errorMessage } } +} diff --git a/src/transport.ts b/src/transport.ts new file mode 100644 index 0000000..9d83d5d --- /dev/null +++ b/src/transport.ts @@ -0,0 +1,139 @@ +import type { + AssistantMessageEvent, + AssistantMessageEventStreamLike, + ContextLike, + ModelLike, + StreamOptions, +} from "./types.ts" + +export type CommandCodeTransport = "unknown" | "provider" | "generate" + +interface TransportDependencies { + createStream: () => AssistantMessageEventStreamLike + streamProvider: ( + model: ModelLike, + context: ContextLike, + options?: StreamOptions, + ) => AssistantMessageEventStreamLike + streamGenerate: ( + model: ModelLike, + context: ContextLike, + options?: StreamOptions, + ) => AssistantMessageEventStreamLike +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +async function isUpgradeRequired(response: Response): Promise { + if (response.status !== 403) return false + + try { + const body: unknown = await response.clone().json() + if (!isRecord(body)) return false + const error = isRecord(body.error) ? body.error : body + return error.code === "upgrade_required" + } catch { + return false + } +} + +export function createCommandCodeTransportRouter(deps: TransportDependencies) { + let transport: CommandCodeTransport = "unknown" + let apiKey: string | undefined + + function pipe( + source: AssistantMessageEventStreamLike, + target: AssistantMessageEventStreamLike, + ): Promise { + return (async () => { + for await (const event of source) target.push(event) + })() + } + + return { + getTransport(): CommandCodeTransport { + return transport + }, + + reset(): void { + transport = "unknown" + apiKey = undefined + }, + + stream( + model: ModelLike, + context: ContextLike, + options?: StreamOptions, + ): AssistantMessageEventStreamLike { + if (options?.apiKey !== apiKey) { + apiKey = options?.apiKey + transport = "unknown" + } + if (transport === "generate") return deps.streamGenerate(model, context, options) + + const output = deps.createStream() + let upgradeRequired = false + const fetchImpl = options?.fetch ?? fetch + const providerOptions: StreamOptions = { + ...options, + fetch: async (input, init) => { + const response = await fetchImpl(input, init) + if (await isUpgradeRequired(response)) upgradeRequired = true + return response + }, + onResponse: async (response, responseModel) => { + if (upgradeRequired) return + await options?.onResponse?.(response, responseModel) + }, + } + + const run = async () => { + const providerStream = deps.streamProvider(model, context, providerOptions) + + for await (const event of providerStream) { + if (!upgradeRequired) { + transport = "provider" + output.push(event) + } + } + + if (upgradeRequired) { + transport = "generate" + await pipe(deps.streamGenerate(model, context, options), output) + } + output.end() + } + + run().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error) + output.push({ + type: "error", + reason: "error", + error: { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "error", + errorMessage: message, + timestamp: Date.now(), + }, + }) + output.end() + }) + + return output + }, + } +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..dc680ea --- /dev/null +++ b/src/types.ts @@ -0,0 +1,202 @@ +export type StopReason = "stop" | "length" | "toolUse" +export type ErrorReason = "error" | "aborted" +export type TerminalReason = StopReason | ErrorReason + +export interface UsageCost { + input: number + output: number + cacheRead: number + cacheWrite: number + total: number +} + +export interface Usage { + input: number + output: number + cacheRead: number + cacheWrite: number + cacheWrite1h?: number + totalTokens: number + cost: UsageCost +} + +export interface TextContent { + type: "text" + text: string +} + +export interface ThinkingContent { + type: "thinking" + thinking: string +} + +export interface ToolCallContent { + type: "toolCall" + id: string + name: string + arguments: Record +} + +export type AssistantContent = TextContent | ThinkingContent | ToolCallContent + +export interface AssistantMessageLike { + role: "assistant" + content: AssistantContent[] + api: unknown + provider: string + model: string + usage: Usage + stopReason: TerminalReason + errorMessage?: string + timestamp: number +} + +export interface ModelCostRates { + input: number + output: number + cacheRead: number + cacheWrite: number +} + +export interface ModelCostTier extends ModelCostRates { + inputTokensAbove: number +} + +export interface ModelCost extends ModelCostRates { + tiers?: readonly ModelCostTier[] +} + +export interface ModelLike { + id: string + api: unknown + provider: string + maxTokens: number + cost: ModelCost + reasoning?: boolean + thinkingLevelMap?: Partial> + thinking?: { + mode?: "effort" + effortMap?: Partial> + efforts?: readonly string[] + } +} + +export interface MessageLike { + role: string + content?: unknown + toolCallId?: string + toolName?: string + isError?: boolean +} + +export interface ToolLike { + name: string + description?: string + parameters?: unknown +} + +export interface ContextLike { + systemPrompt?: string + messages?: readonly MessageLike[] + tools?: readonly ToolLike[] +} + +export interface ProviderResponseInfo { + status: number + headers: Record +} + +export interface StreamOptions { + apiKey?: string + signal?: AbortSignal + headers?: Record + fetch?: typeof fetch + maxTokens?: number + /** Resolved pi thinking level; forwarded only through the model's map. */ + reasoning?: string + onPayload?: (payload: unknown, model: ModelLike) => unknown | Promise + onResponse?: (response: ProviderResponseInfo, model: ModelLike) => void | Promise + /** + * HTTP request timeout in milliseconds. + * Applied per-attempt; on timeout the request is retried if retries remain. + */ + timeoutMs?: number + /** + * Maximum retry attempts for transient HTTP errors (429, 5xx). + * Default: 0 (pi agent-level retry handles visible retries when unset). + */ + maxRetries?: number + /** + * Maximum delay in milliseconds to wait for a retry when the server requests + * a long wait via Retry-After. If the server's requested delay exceeds this + * value, the request fails immediately. Default: 60000 (60 seconds). + * Set to 0 to disable the cap. + */ + maxRetryDelayMs?: number +} + +export type AssistantMessageEvent = + | { type: "start"; partial: AssistantMessageLike } + | { type: "text_start"; contentIndex: number; partial: AssistantMessageLike } + | { + type: "text_delta" + contentIndex: number + delta: string + partial: AssistantMessageLike + } + | { + type: "text_end" + contentIndex: number + content: string + partial: AssistantMessageLike + } + | { + type: "thinking_start" + contentIndex: number + partial: AssistantMessageLike + } + | { + type: "thinking_delta" + contentIndex: number + delta: string + partial: AssistantMessageLike + } + | { + type: "thinking_end" + contentIndex: number + content: string + partial: AssistantMessageLike + } + | { + type: "toolcall_start" + contentIndex: number + partial: AssistantMessageLike + } + | { + type: "toolcall_end" + contentIndex: number + toolCall: ToolCallContent + partial: AssistantMessageLike + } + | { type: "done"; reason: StopReason; message: AssistantMessageLike } + | { type: "error"; reason: ErrorReason; error: AssistantMessageLike } + +export interface AssistantMessageEventStreamLike extends AsyncIterable { + push(event: AssistantMessageEvent): void + end(): void +} + +export interface CoreDependencies { + createStream: () => AssistantMessageEventStreamLike + calculateCost: (model: ModelLike, usage: Usage) => void + apiBase?: string + fetchImpl?: typeof fetch + authPaths?: readonly string[] + env?: NodeJS.ProcessEnv + cwd?: () => string + now?: () => number + uuid?: () => string + homeDir?: () => string + /** Injectable delay for retry backoff. Defaults to setTimeout. */ + delay?: (ms: number, signal: AbortSignal) => Promise +} diff --git a/tests/helpers.ts b/tests/helpers.ts new file mode 100644 index 0000000..f4b3f65 --- /dev/null +++ b/tests/helpers.ts @@ -0,0 +1,289 @@ +import { createServer, type IncomingHttpHeaders, type Server } from "node:http" + +import { + createStreamCommandCode, + type AssistantMessageEvent, + type AssistantMessageEventStreamLike, + type ContextLike, + type CoreDependencies, + type ModelLike, + type Usage, +} from "../src/core.ts" + +export function createTestEventStream(): AssistantMessageEventStreamLike { + const events: AssistantMessageEvent[] = [] + const waiters: Array<() => void> = [] + let ended = false + + const wake = () => { + const waiter = waiters.shift() + if (waiter) waiter() + } + + return { + push(event: AssistantMessageEvent) { + events.push(event) + wake() + }, + end() { + ended = true + while (waiters.length > 0) wake() + }, + [Symbol.asyncIterator]() { + let index = 0 + return { + async next(): Promise> { + while (index >= events.length && !ended) { + await new Promise((resolve) => waiters.push(resolve)) + } + if (index < events.length) { + const value = events[index] + index += 1 + return { done: false, value } + } + return { done: true, value: undefined } + }, + } + }, + } +} + +export async function collectEvents( + stream: AssistantMessageEventStreamLike, + timeoutMs = 2_000, +): Promise { + const events: AssistantMessageEvent[] = [] + + const collect = async () => { + for await (const event of stream) { + events.push(event) + if (event.type === "done" || event.type === "error") break + } + return events + } + + return await Promise.race([ + collect(), + new Promise((_, reject) => { + setTimeout( + () => reject(new Error(`Timed out collecting stream events after ${timeoutMs}ms`)), + timeoutMs, + ) + }), + ]) +} + +export function makeModel(overrides: Partial = {}): ModelLike { + return { + id: "deepseek/deepseek-v4-flash", + api: "commandcode-custom", + provider: "commandcode", + maxTokens: 384_000, + cost: { input: 0.14, output: 0.28, cacheRead: 0.0028, cacheWrite: 0 }, + ...overrides, + } +} + +export function makeContext(overrides: Partial = {}): ContextLike { + return { + systemPrompt: "You are a test assistant.", + messages: [{ role: "user", content: "hello" }], + tools: [], + ...overrides, + } +} + +export interface TestDepsResult { + streamCommandCode: ReturnType + calculatedUsages: Usage[] +} + +export function createTestDeps(overrides: Partial = {}): TestDepsResult { + const calculatedUsages: Usage[] = [] + const streamCommandCode = createStreamCommandCode({ + createStream: createTestEventStream, + calculateCost: (_model, usage) => { + calculatedUsages.push({ + ...usage, + cost: { ...usage.cost }, + }) + }, + env: {}, + authPaths: [], + now: () => new Date("2026-05-05T12:00:00Z").getTime(), + uuid: () => "00000000-0000-4000-8000-000000000000", + cwd: () => "/repo", + delay: async () => {}, + ...overrides, + }) + return { streamCommandCode, calculatedUsages } +} + +type SuccessPlan = { + type: "success" + status?: number + events?: string[] + chunks?: string[] + delays?: number[] + hangAfterLast?: boolean + /** Delay in ms before the server starts sending the response. */ + responseDelay?: number +} + +type ErrorPlan = { + type: "error" + status: number + body: string + headers?: Record +} + +export type ResponsePlan = SuccessPlan | ErrorPlan + +function headersToRecord(headers: IncomingHttpHeaders): Record { + const out: Record = {} + for (const [key, value] of Object.entries(headers)) { + if (typeof value === "string") out[key] = value + else if (Array.isArray(value)) out[key] = value.join(", ") + } + return out +} + +export interface MockCommandCodeServer { + baseUrl(): string + mockResponse(plan: ResponsePlan): void + mockResponseQueue(plans: ResponsePlan[]): void + reset(): void + close(): Promise + lastRequestBody(): unknown + lastRequestHeaders(): Record + requestCount(): number + responseClosedBeforeEnd(): boolean +} + +export async function startMockCommandCodeServer(): Promise { + let planQueue: ResponsePlan[] = [{ type: "success", events: [] }] + let lastBody: unknown + let lastHeaders: Record = {} + let requests = 0 + let closedBeforeEnd = false + let port = 0 + + const server: Server = createServer((req, res) => { + if (req.method !== "POST" || req.url !== "/alpha/generate") { + res.writeHead(404) + res.end("Not found") + return + } + + requests += 1 + lastHeaders = headersToRecord(req.headers) + let body = "" + req.on("data", (chunk: Buffer) => { + body += chunk.toString("utf-8") + }) + req.on("end", () => { + try { + const parsed: unknown = JSON.parse(body) + lastBody = parsed + } catch { + lastBody = undefined + } + + // Pop the first plan from the queue; keep the last one as fallback. + const plan = planQueue.length > 1 ? planQueue.shift()! : planQueue[0] + + if (plan.type === "error") { + const headers: Record = { "Content-Type": "text/plain", ...plan.headers } + res.writeHead(plan.status, headers) + res.end(plan.body) + return + } + + res.writeHead(plan.status ?? 200, { + "Content-Type": "text/plain; charset=utf-8", + "Transfer-Encoding": "chunked", + }) + + let ended = false + res.on("close", () => { + if (!ended) closedBeforeEnd = true + }) + + const chunks = plan.chunks ?? (plan.events ?? []).map((event) => `${event}\n`) + const delays = plan.delays ?? chunks.map(() => 0) + let index = 0 + + const sendNext = () => { + if (index >= chunks.length) { + if (!plan.hangAfterLast) { + ended = true + res.end() + } + return + } + + res.write(chunks[index]) + index += 1 + if (index < chunks.length) { + setTimeout(sendNext, delays[index] ?? 0) + } else if (!plan.hangAfterLast) { + ended = true + res.end() + } + } + + if (plan.responseDelay) { + setTimeout(sendNext, plan.responseDelay) + } else { + sendNext() + } + }) + }) + + await new Promise((resolve) => { + server.listen(0, () => { + const address = server.address() + if (typeof address === "object" && address) port = address.port + resolve() + }) + }) + + return { + baseUrl: () => `http://127.0.0.1:${port}`, + mockResponse(plan: ResponsePlan) { + planQueue = [plan] + }, + mockResponseQueue(plans: ResponsePlan[]) { + planQueue = [...plans] + }, + reset() { + planQueue = [{ type: "success", events: [] }] + lastBody = undefined + lastHeaders = {} + requests = 0 + closedBeforeEnd = false + }, + close() { + return new Promise((resolve) => server.close(() => resolve())) + }, + lastRequestBody: () => lastBody, + lastRequestHeaders: () => lastHeaders, + requestCount: () => requests, + responseClosedBeforeEnd: () => closedBeforeEnd, + } +} + +export function objectAt(value: unknown, path: readonly string[]): unknown { + let current = value + for (const key of path) { + if (Array.isArray(current)) { + const index = Number(key) + if (!Number.isInteger(index)) return undefined + current = current[index] + continue + } + if (typeof current !== "object" || current === null) return undefined + current = Object.getOwnPropertyDescriptor(current, key)?.value + } + return current +} diff --git a/tests/test-abort.ts b/tests/test-abort.ts new file mode 100644 index 0000000..8db0ad5 --- /dev/null +++ b/tests/test-abort.ts @@ -0,0 +1,85 @@ +/** + * Abort tests against the real streamCommandCode core. + */ + +import assert from "node:assert/strict" +import { after, before, beforeEach, describe, it } from "node:test" + +import { + collectEvents, + createTestDeps, + makeContext, + makeModel, + startMockCommandCodeServer, + type MockCommandCodeServer, +} from "./helpers.ts" + +let server: MockCommandCodeServer + +before(async () => { + server = await startMockCommandCodeServer() +}) + +after(async () => { + await server.close() +}) + +beforeEach(() => { + server.reset() +}) + +describe("streamCommandCode — abort behavior", () => { + it("emits aborted error when signal is already aborted", async () => { + const controller = new AbortController() + controller.abort() + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: "mock-key", + signal: controller.signal, + }), + ) + + assert.deepEqual( + events.map((event) => event.type), + ["start", "error"], + ) + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.equal(error.reason, "aborted") + assert.equal(error.error.stopReason, "aborted") + assert.equal(server.requestCount(), 0) + }) + + it("emits aborted error and cancels the response reader mid-stream", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "text-delta", text: "first" })], + hangAfterLast: true, + }) + const controller = new AbortController() + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const stream = streamCommandCode(makeModel(), makeContext(), { + apiKey: "mock-key", + signal: controller.signal, + }) + + setTimeout(() => controller.abort(), 50) + const events = await collectEvents(stream, 2_000) + + assert.ok( + events.some((event) => event.type === "text_delta"), + "stream should process data before abort", + ) + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.equal(error.reason, "aborted") + assert.equal(error.error.errorMessage, "Request aborted") + await new Promise((resolve) => setTimeout(resolve, 50)) + assert.ok(server.responseClosedBeforeEnd(), "abort should close the hanging upstream response") + }) +}) diff --git a/tests/test-cost.ts b/tests/test-cost.ts new file mode 100644 index 0000000..ae7f730 --- /dev/null +++ b/tests/test-cost.ts @@ -0,0 +1,183 @@ +/** + * Regression test for the local cost calculation. + * + * The provider ships its own cost function because Oh My Pi's legacy pi-ai + * shim does not export `calculateCost` (see issue #24). This test locks the + * local implementation to pi-ai's documented per-million-token arithmetic + * without installing another pi-ai runtime next to the extension. + */ + +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { calculateCommandCodeCost } from "../src/cost.ts" +import type { Usage } from "../src/types.ts" + +interface CostRates { + input: number + output: number + cacheRead: number + cacheWrite: number +} + +interface CostTable extends CostRates { + tiers?: Array +} + +const COST_FIXTURES: Record = { + "zero-cost-model": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + "claude-sonnet-4-6": { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 }, + "deepseek/deepseek-v4-pro": { + input: 0.435, + output: 0.87, + cacheRead: 0.003625, + cacheWrite: 0, + }, + "Qwen/Qwen3.7-Max": { input: 2.5, output: 7.5, cacheRead: 0.5, cacheWrite: 3.13 }, + "Qwen/Qwen3.7-Flash": { + input: 0.03, + output: 0.13, + cacheRead: 0.006, + cacheWrite: 0.038, + tiers: [ + { inputTokensAbove: 32_000, input: 0.1, output: 0.4, cacheRead: 0.02, cacheWrite: 0.125 }, + { inputTokensAbove: 256_000, input: 0.2, output: 0.8, cacheRead: 0.04, cacheWrite: 0.25 }, + ], + }, + "gpt-5.5": { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 }, +} + +const USAGE_CASES = [ + { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + { input: 1, output: 1, cacheRead: 1, cacheWrite: 1 }, + { input: 812, output: 187, cacheRead: 52_000, cacheWrite: 3_100 }, + { input: 1_000_000, output: 65_536, cacheRead: 998_877, cacheWrite: 123_456 }, + { input: 7, output: 999_999_999, cacheRead: 0.5, cacheWrite: 42 }, +] + +function commandCodeModel(id: string, cost: CostTable) { + return { + id, + api: "commandcode-custom", + provider: "commandcode", + cost, + maxTokens: 65_536, + } +} + +function assertClose(actual: number, expected: number) { + assert.ok( + Math.abs(actual - expected) <= + Number.EPSILON * Math.max(1, Math.abs(actual), Math.abs(expected)), + `expected ${actual} to be close to ${expected}`, + ) +} + +function freshUsage(tokens: (typeof USAGE_CASES)[number]): Usage { + return { + ...tokens, + totalTokens: tokens.input + tokens.output + tokens.cacheRead + tokens.cacheWrite, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + } +} + +function expectedCost(cost: CostTable, tokens: (typeof USAGE_CASES)[number]): Usage["cost"] { + const inputTokens = tokens.input + tokens.cacheRead + tokens.cacheWrite + let rates: CostRates = cost + let matchedThreshold = -1 + for (const tier of cost.tiers ?? []) { + if (inputTokens > tier.inputTokensAbove && tier.inputTokensAbove > matchedThreshold) { + rates = tier + matchedThreshold = tier.inputTokensAbove + } + } + + const input = (rates.input / 1_000_000) * tokens.input + const output = (rates.output / 1_000_000) * tokens.output + const cacheRead = (rates.cacheRead / 1_000_000) * tokens.cacheRead + const cacheWrite = (rates.cacheWrite * tokens.cacheWrite) / 1_000_000 + return { + input, + output, + cacheRead, + cacheWrite, + total: input + output + cacheRead + cacheWrite, + } +} + +describe("calculateCommandCodeCost()", () => { + it("applies per-million-token rates to all cost fields", () => { + for (const [id, cost] of Object.entries(COST_FIXTURES)) { + const model = commandCodeModel(id, cost) + + for (const tokens of USAGE_CASES) { + const usage = freshUsage(tokens) + calculateCommandCodeCost(model, usage) + + assert.deepEqual( + usage.cost, + expectedCost(cost, tokens), + `${id} cost for tokens=${JSON.stringify(tokens)}`, + ) + } + } + }) + + it("applies the highest request-wide input tier above its threshold", () => { + const model = commandCodeModel("Qwen/Qwen3.7-Flash", COST_FIXTURES["Qwen/Qwen3.7-Flash"]) + + const atThreshold = freshUsage({ + input: 32_000, + output: 1_000, + cacheRead: 0, + cacheWrite: 0, + }) + calculateCommandCodeCost(model, atThreshold) + assertClose(atThreshold.cost.input, (0.03 * 32_000) / 1_000_000) + + const aboveFirstTier = freshUsage({ + input: 30_000, + output: 1_000, + cacheRead: 2_001, + cacheWrite: 0, + }) + calculateCommandCodeCost(model, aboveFirstTier) + assertClose(aboveFirstTier.cost.input, (0.1 * 30_000) / 1_000_000) + assertClose(aboveFirstTier.cost.cacheRead, (0.02 * 2_001) / 1_000_000) + + const aboveHighestTier = freshUsage({ + input: 100_000, + output: 1_000, + cacheRead: 156_001, + cacheWrite: 0, + }) + calculateCommandCodeCost(model, aboveHighestTier) + assertClose(aboveHighestTier.cost.input, (0.2 * 100_000) / 1_000_000) + assertClose(aboveHighestTier.cost.output, (0.8 * 1_000) / 1_000_000) + }) + + it("prices one-hour cache writes at twice the active input rate", () => { + const model = commandCodeModel("claude-sonnet-4-6", COST_FIXTURES["claude-sonnet-4-6"]) + const usage = freshUsage({ input: 0, output: 0, cacheRead: 0, cacheWrite: 1_000 }) + usage.cacheWrite1h = 400 + + calculateCommandCodeCost(model, usage) + + const expectedShortWrite = (3.75 * 600) / 1_000_000 + const expectedLongWrite = (3 * 2 * 400) / 1_000_000 + assertClose(usage.cost.cacheWrite, expectedShortWrite + expectedLongWrite) + }) + + it("writes the total as the sum of all cost components", () => { + const model = commandCodeModel("claude-sonnet-4-6", COST_FIXTURES["claude-sonnet-4-6"]) + const usage = freshUsage({ input: 1_000, output: 500, cacheRead: 10_000, cacheWrite: 2_000 }) + + calculateCommandCodeCost(model, usage) + + assert.equal( + usage.cost.total, + usage.cost.input + usage.cost.output + usage.cost.cacheRead + usage.cost.cacheWrite, + ) + assert.ok(usage.cost.total > 0) + }) +}) diff --git a/tests/test-overflow.ts b/tests/test-overflow.ts new file mode 100644 index 0000000..dc28888 --- /dev/null +++ b/tests/test-overflow.ts @@ -0,0 +1,196 @@ +import assert from "node:assert/strict" +import { after, before, beforeEach, describe, it } from "node:test" + +import { + commandCodeErrorMessage, + normalizeCommandCodeErrorMessage, + normalizeCommandCodeMessage, +} from "../src/overflow.ts" +import { + collectEvents, + createTestDeps, + makeContext, + makeModel, + startMockCommandCodeServer, + type MockCommandCodeServer, +} from "./helpers.ts" + +let server: MockCommandCodeServer + +before(async () => { + server = await startMockCommandCodeServer() +}) + +after(async () => { + await server.close() +}) + +beforeEach(() => { + server.reset() +}) + +describe("Command Code overflow normalization", () => { + it("normalizes Command Code context errors to pi's generic overflow marker", () => { + const normalized = normalizeCommandCodeErrorMessage("Prompt token limit exceeded") + + assert.equal(normalized, "context_length_exceeded: Prompt token limit exceeded") + }) + + it("is idempotent and leaves unrelated, rate-limit, and capacity errors unchanged", () => { + assert.equal( + normalizeCommandCodeErrorMessage("context_length_exceeded: Prompt token limit exceeded"), + undefined, + ) + assert.equal(normalizeCommandCodeErrorMessage("OpenAI request failed"), undefined) + assert.equal( + normalizeCommandCodeErrorMessage("Prompt token limit exceeded due to rate limit"), + undefined, + ) + assert.equal( + normalizeCommandCodeErrorMessage("Command Code API error 429: context window exceeded"), + undefined, + ) + assert.equal( + normalizeCommandCodeErrorMessage("context window exceeded: status: 429"), + undefined, + ) + assert.equal( + normalizeCommandCodeErrorMessage("The input is too long"), + "context_length_exceeded: The input is too long", + ) + assert.equal( + normalizeCommandCodeErrorMessage("Input exceeds context limit"), + "context_length_exceeded: Input exceeds context limit", + ) + assert.equal( + normalizeCommandCodeErrorMessage("Context window exceeded: provider capacity reached"), + undefined, + ) + }) + + it("scopes finalized message normalization to Command Code", () => { + const message = { + role: "assistant" as const, + provider: "commandcode", + stopReason: "error" as const, + errorMessage: "model context window exceeded", + } + + const normalized = normalizeCommandCodeMessage(message) + assert.equal( + normalized?.message.errorMessage, + "context_length_exceeded: model context window exceeded", + ) + assert.equal(normalizeCommandCodeMessage({ ...message, provider: "openai" }), undefined) + assert.equal(normalizeCommandCodeMessage({ ...message, stopReason: "stop" }), undefined) + }) + + it("extracts nested stream error messages without exposing credentials", () => { + assert.equal( + commandCodeErrorMessage({ + error: { details: { errorMessage: "context window exceeded" } }, + }), + "context window exceeded", + ) + }) + + it("redacts secrets from finalized provider errors", async () => { + server.mockResponse({ + type: "error", + status: 400, + body: "api_key=user_secret_value", + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.doesNotMatch(error.error.errorMessage ?? "", /user_secret_value/) + assert.match(error.error.errorMessage ?? "", /api_key=\[redacted\]/) + }) + + it("normalizes HTTP error bodies containing nested context errors", async () => { + server.mockResponse({ + type: "error", + status: 400, + body: JSON.stringify({ error: { message: "Prompt token limit exceeded" } }), + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + const error = events.at(-1) + + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + const normalized = normalizeCommandCodeMessage(error.error) + assert.match(normalized?.message.errorMessage ?? "", /^context_length_exceeded:/) + }) + + it("does not normalize an HTTP rate-limit response that mentions context", async () => { + server.mockResponse({ + type: "error", + status: 429, + body: JSON.stringify({ error: { message: "context window exceeded" } }), + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + const error = events.at(-1) + + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.equal(normalizeCommandCodeMessage(error.error), undefined) + }) + + it("normalizes nested stream error events", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ + type: "error", + error: { details: { message: "model context window exceeded" } }, + }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + const error = events.at(-1) + + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + const normalized = normalizeCommandCodeMessage(error.error) + assert.equal( + normalized?.message.errorMessage, + "context_length_exceeded: model context window exceeded", + ) + + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ + type: "error", + error: { message: "context window exceeded", status: 429 }, + }), + ], + }) + const retryEvents = await collectEvents( + createTestDeps({ apiBase: server.baseUrl() }).streamCommandCode(makeModel(), makeContext(), { + apiKey: "mock-key", + }), + ) + const retryError = retryEvents.at(-1) + assert.equal(retryError?.type, "error") + if (retryError?.type !== "error") throw new Error("expected error") + assert.equal(normalizeCommandCodeMessage(retryError.error), undefined) + }) +}) diff --git a/tests/test-pure-functions.ts b/tests/test-pure-functions.ts new file mode 100644 index 0000000..136025f --- /dev/null +++ b/tests/test-pure-functions.ts @@ -0,0 +1,658 @@ +/** + * Unit tests for the real pure helpers exported by src/core.ts. + * These are hermetic: no pi runtime and no network. + */ + +import assert from "node:assert/strict" +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "node:test" + +import { + assertTextOnlyMessages, + getApiKey, + getEnvironmentInfo, + mapFinishReason, + messagesToCC, + parseStreamEventLine, + projectSlugFromPath, + textContent, + toJsonSchema, + toolsToJson, +} from "../src/core.ts" +import { redactCommandCodeErrorText } from "../src/overflow.ts" + +import { objectAt } from "./helpers.ts" + +describe("getApiKey()", () => { + it("uses COMMANDCODE_API_KEY from provided env", () => { + assert.equal(getApiKey({ env: { COMMANDCODE_API_KEY: "env-key" }, authPaths: [] }), "env-key") + }) + + it("reads apiKey, commandcode, pi OAuth, and official CLI credential fields", () => { + const dir = mkdtempSync(join(tmpdir(), "cc-auth-")) + try { + const first = join(dir, "first.json") + const second = join(dir, "second.json") + const oauth = join(dir, "oauth.json") + const official = join(dir, "official.json") + writeFileSync(first, JSON.stringify({ apiKey: "file-key" })) + writeFileSync(second, JSON.stringify({ commandcode: "fallback-key" })) + writeFileSync( + oauth, + JSON.stringify({ + commandcode: { + type: "oauth", + access: "oauth-access-key", + refresh: "oauth-refresh-key", + expires: Date.now() + 3600000, + }, + }), + ) + writeFileSync( + official, + JSON.stringify({ + "command-code": { + type: "api", + key: "official-cli-key", + }, + }), + ) + assert.equal(getApiKey({ env: {}, authPaths: [first, second] }), "file-key") + assert.equal(getApiKey({ env: {}, authPaths: [second] }), "fallback-key") + assert.equal(getApiKey({ env: {}, authPaths: [oauth] }), "oauth-access-key") + assert.equal(getApiKey({ env: {}, authPaths: [official] }), "official-cli-key") + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("ignores malformed auth files", () => { + const dir = mkdtempSync(join(tmpdir(), "cc-auth-bad-")) + try { + const bad = join(dir, "bad.json") + writeFileSync(bad, "not json") + assert.equal(getApiKey({ env: {}, authPaths: [bad] }), undefined) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + + it("uses injected homeDir for default auth paths", () => { + const dir = mkdtempSync(join(tmpdir(), "cc-home-")) + try { + const authDir = join(dir, ".pi", "agent") + mkdirSync(authDir, { recursive: true }) + writeFileSync(join(authDir, "auth.json"), JSON.stringify({ commandcode: "pi-key" })) + assert.equal(getApiKey({ env: {}, homeDir: () => dir }), "pi-key") + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) + +describe("error redaction", () => { + it("redacts bearer, credential, and query-string secrets", () => { + const redacted = redactCommandCodeErrorText( + "Bearer user_secret_value api_key=user_secret_value https://example.test/x?token=user_secret_value", + ) + assert.doesNotMatch(redacted, /user_secret_value/) + assert.match(redacted, /Bearer \[redacted\]/) + assert.doesNotMatch( + redactCommandCodeErrorText("provider returned sk-test-secret-value-1234567890"), + /sk-test-secret-value/, + ) + }) +}) + +describe("projectSlugFromPath()", () => { + it("matches the official CLI-style slug from an absolute working directory", () => { + assert.equal( + projectSlugFromPath("/Users/patwoz/dev/Personal/pi/pi-commandcode-provider"), + "users-patwoz-dev-personal-pi-pi-commandcode-provider", + ) + assert.equal(projectSlugFromPath("/repo"), "repo") + }) +}) + +describe("text-only image handling", () => { + it("rejects image content for models without image support", () => { + assert.throws( + () => + assertTextOnlyMessages([ + { + role: "user", + content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], + }, + ]), + /does not support image content/i, + ) + assert.throws( + () => + assertTextOnlyMessages([ + { + role: "toolResult", + toolCallId: "c1", + content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], + }, + ]), + /does not support image content/i, + ) + }) +}) + +describe("textContent()", () => { + it("extracts and joins text blocks", () => { + assert.equal( + textContent({ + content: [ + { type: "text", text: "hello" }, + { type: "text", text: "world" }, + ], + }), + "hello\nworld", + ) + }) + + it("extracts text while images are handled separately", () => { + assert.equal( + textContent({ + content: [ + { type: "text", text: "hello" }, + { type: "image", data: "x", mimeType: "image/png" }, + { type: "text", text: "world" }, + ], + }), + "hello\nworld", + ) + }) + + it("handles empty or missing content", () => { + assert.equal(textContent({ content: [] }), "") + assert.equal(textContent({}), "") + }) +}) + +describe("getEnvironmentInfo()", () => { + it("returns platform, arch, and Node version", () => { + const info = getEnvironmentInfo() + assert.match(info, /^(darwin|linux|win32)-/) + assert.ok(info.includes("Node.js")) + }) +}) + +describe("toJsonSchema()", () => { + it("converts scalar, enum, object, optional, array, and union schema shapes", () => { + assert.deepEqual(toJsonSchema({ kind: "string" }), { type: "string" }) + assert.deepEqual(toJsonSchema({ kind: "Number" }), { type: "number" }) + assert.deepEqual(toJsonSchema({ kind: "boolean" }), { type: "boolean" }) + assert.deepEqual(toJsonSchema({ kind: "string", enum: ["left", "right"] }), { + type: "string", + enum: ["left", "right"], + }) + assert.deepEqual( + toJsonSchema({ + kind: "object", + properties: { + name: { kind: "string" }, + tags: { kind: "array", items: { kind: "string" }, optional: true }, + }, + }), + { + type: "object", + properties: { + name: { type: "string" }, + tags: { type: "array", items: { type: "string" } }, + }, + required: ["name"], + }, + ) + assert.deepEqual(toJsonSchema({ kind: "optional", wrapped: { kind: "string" } }), { + type: "string", + }) + assert.deepEqual(toJsonSchema({ kind: "union", variants: [{}, { kind: "number" }] }), { + type: "number", + }) + }) + + it("preserves explicit required arrays and handles unknown values", () => { + assert.deepEqual( + toJsonSchema({ + type: "object", + properties: { name: { type: "string" }, nickname: { type: "string" } }, + required: ["name"], + }), + { + type: "object", + properties: { name: { type: "string" }, nickname: { type: "string" } }, + required: ["name"], + }, + ) + assert.deepEqual(toJsonSchema(undefined), {}) + assert.deepEqual(toJsonSchema({ kind: "wat" }), {}) + assert.deepEqual(toJsonSchema({ type: "wat", description: "not a schema" }), {}) + assert.deepEqual(toJsonSchema({}), {}) + assert.equal(toJsonSchema(true), true) + }) + + it("preserves complete JSON Schema metadata and nested schemas", () => { + assert.deepEqual( + toJsonSchema({ + type: "object", + description: "Search options", + properties: { + query: { + type: "string", + description: "Text to search for", + minLength: 2, + maxLength: 50, + pattern: "^[a-z]+$", + default: "pi", + }, + limit: { + type: "integer", + minimum: 1, + maximum: 100, + exclusiveMinimum: 0, + multipleOf: 1, + default: 10, + }, + tags: { + type: "array", + minItems: 1, + maxItems: 3, + uniqueItems: true, + items: { + type: "object", + properties: { name: { type: "string" } }, + required: ["name"], + additionalProperties: false, + }, + }, + }, + required: ["query", "limit"], + additionalProperties: false, + }), + { + type: "object", + description: "Search options", + properties: { + query: { + type: "string", + description: "Text to search for", + minLength: 2, + maxLength: 50, + pattern: "^[a-z]+$", + default: "pi", + }, + limit: { + type: "integer", + minimum: 1, + maximum: 100, + exclusiveMinimum: 0, + multipleOf: 1, + default: 10, + }, + tags: { + type: "array", + minItems: 1, + maxItems: 3, + uniqueItems: true, + items: { + type: "object", + properties: { name: { type: "string" } }, + required: ["name"], + additionalProperties: false, + }, + }, + }, + required: ["query", "limit"], + additionalProperties: false, + }, + ) + }) + + it("preserves JSON Schema composition and nullable forms", () => { + assert.deepEqual( + toJsonSchema({ + anyOf: [{ type: "string" }, { type: "number" }], + oneOf: [{ const: "a" }, { const: "b" }], + allOf: [{ minLength: 1 }, { maxLength: 10 }], + nullable: true, + }), + { + anyOf: [{ type: "string" }, { type: "number" }, { type: "null" }], + oneOf: [{ const: "a" }, { const: "b" }], + allOf: [{ minLength: 1 }, { maxLength: 10 }], + }, + ) + assert.deepEqual(toJsonSchema({ type: ["string", "null"] }), { + type: ["string", "null"], + }) + assert.deepEqual(toJsonSchema({ type: "string", nullable: true }), { + type: ["string", "null"], + }) + }) + + it("preserves dangerous schema property names", () => { + const inputProperties: Record = { + constructor: { type: "number" }, + } + Object.defineProperty(inputProperties, "__proto__", { + configurable: true, + enumerable: true, + value: { type: "string" }, + writable: true, + }) + const schema = toJsonSchema({ + type: "object", + properties: inputProperties, + required: ["__proto__", "constructor"], + }) + assert.ok(schema && typeof schema === "object" && !Array.isArray(schema)) + if (!schema || typeof schema !== "object" || Array.isArray(schema)) { + throw new Error("expected object schema") + } + const outputProperties: unknown = Object.getOwnPropertyDescriptor(schema, "properties")?.value + assert.ok(outputProperties && typeof outputProperties === "object") + if (!outputProperties || typeof outputProperties !== "object") { + throw new Error("expected object properties") + } + assert.ok(Object.prototype.hasOwnProperty.call(outputProperties, "__proto__")) + assert.deepEqual(Object.getOwnPropertyDescriptor(outputProperties, "__proto__")?.value, { + type: "string", + }) + assert.deepEqual(Object.getOwnPropertyDescriptor(outputProperties, "constructor")?.value, { + type: "number", + }) + }) + + it("converts legacy shapes without collapsing unions", () => { + assert.deepEqual( + toJsonSchema({ + kind: "Object", + description: "Legacy options", + properties: { + mode: { + kind: "union", + variants: [ + { kind: "string", enum: ["fast", "safe"] }, + { kind: "string", enum: ["debug"] }, + ], + }, + count: { kind: "Number", minimum: 1, optional: true }, + nested: { + kind: "Array", + element: { kind: "object", properties: { value: { kind: "boolean" } } }, + }, + }, + optional: ["count"], + additionalProperties: false, + }), + { + type: "object", + description: "Legacy options", + properties: { + mode: { + anyOf: [ + { type: "string", enum: ["fast", "safe"] }, + { type: "string", enum: ["debug"] }, + ], + }, + count: { type: "number", minimum: 1 }, + nested: { + type: "array", + items: { + type: "object", + properties: { value: { type: "boolean" } }, + required: ["value"], + }, + }, + }, + required: ["mode", "nested"], + additionalProperties: false, + }, + ) + assert.deepEqual( + toJsonSchema({ + kind: "intersect", + variants: [{ kind: "object", properties: { a: { kind: "string" } } }, { kind: "number" }], + }), + { + allOf: [ + { type: "object", properties: { a: { type: "string" } }, required: ["a"] }, + { type: "number" }, + ], + }, + ) + }) +}) + +describe("toolsToJson()", () => { + it("converts pi tools to Command Code tool JSON", () => { + assert.deepEqual( + toolsToJson([ + { + name: "get_weather", + description: "Get weather", + parameters: { + kind: "object", + properties: { city: { kind: "string" } }, + }, + }, + ]), + [ + { + type: "function", + name: "get_weather", + description: "Get weather", + input_schema: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }, + ], + ) + }) + + it("returns an empty array for missing tools", () => { + assert.deepEqual(toolsToJson(), []) + }) +}) + +describe("messagesToCC()", () => { + it("converts user, assistant, and tool result messages", () => { + const result = messagesToCC([ + { role: "user", content: "read /tmp/test" }, + { + role: "assistant", + content: [ + { type: "thinking", thinking: "I will read" }, + { type: "text", text: "Sure" }, + { + type: "toolCall", + id: "c1", + name: "read", + arguments: { path: "/tmp/test" }, + }, + ], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + isError: false, + content: [ + { type: "text", text: "hello" }, + { type: "text", text: "world" }, + ], + }, + ]) + + assert.equal(objectAt(result, ["0", "role"]), "user") + assert.equal(objectAt(result, ["1", "role"]), "assistant") + assert.equal(objectAt(result, ["1", "content", "0", "type"]), "text") + assert.equal(objectAt(result, ["1", "content", "1", "type"]), "tool-call") + assert.equal(objectAt(result, ["1", "content", "2"]), undefined) + assert.equal(objectAt(result, ["2", "role"]), "tool") + assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "hello\nworld") + }) + + it("serializes image inputs in the current Command Code wire format", () => { + assert.deepEqual( + messagesToCC( + [ + { + role: "user", + content: [ + { type: "text", text: "inspect this" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, + ], + }, + ], + { allowImages: true }, + ), + [ + { + role: "user", + content: [ + { type: "text", text: "inspect this" }, + { + type: "image", + image: "data:image/png;base64,aGVsbG8=", + mimeType: "image/png", + }, + ], + }, + ], + ) + }) + + it("preserves tool-result images as a following user image message", () => { + const result = messagesToCC( + [ + { role: "user", content: "read image" }, + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: [ + { type: "text", text: "image attached" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }, + ], + }, + ], + { allowImages: true }, + ) + + assert.equal(objectAt(result, ["2", "role"]), "tool") + assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "image attached") + assert.deepEqual(objectAt(result, ["3"]), { + role: "user", + content: [ + { + type: "image", + image: "data:image/jpeg;base64,aGVsbG8=", + mimeType: "image/jpeg", + }, + ], + }) + }) + + it("drops previous assistant reasoning while preserving text and tool calls", () => { + const result = messagesToCC([ + { role: "user", content: "first question" }, + { + role: "assistant", + content: [ + { type: "thinking", thinking: "private reasoning from turn one" }, + { type: "text", text: "first answer" }, + ], + }, + { role: "user", content: "follow-up question" }, + ]) + + assert.deepEqual(result, [ + { role: "user", content: "first question" }, + { role: "assistant", content: [{ type: "text", text: "first answer" }] }, + { role: "user", content: "follow-up question" }, + ]) + }) + + it("omits assistant turns that contain only previous reasoning", () => { + const result = messagesToCC([ + { role: "user", content: "first question" }, + { + role: "assistant", + content: [{ type: "thinking", thinking: "private reasoning" }], + }, + { role: "user", content: "follow-up question" }, + ]) + + assert.deepEqual(result, [ + { role: "user", content: "first question" }, + { role: "user", content: "follow-up question" }, + ]) + }) + + it("drops orphaned tool calls that have no matching tool result", () => { + const result = messagesToCC([ + { role: "user", content: "edit a file" }, + { + role: "assistant", + content: [ + { type: "text", text: "I will edit it" }, + { + type: "toolCall", + id: "missing-result", + name: "edit", + arguments: { path: "x" }, + }, + ], + }, + ]) + + assert.equal(objectAt(result, ["1", "role"]), "assistant") + assert.equal(objectAt(result, ["1", "content", "0", "type"]), "text") + assert.equal(objectAt(result, ["1", "content", "1"]), undefined) + }) + + it("handles empty conversations", () => { + assert.deepEqual(messagesToCC([]), []) + }) +}) + +describe("parseStreamEventLine()", () => { + it("parses plain JSON and SSE data lines", () => { + assert.deepEqual(parseStreamEventLine('{"type":"text-delta","text":"x"}'), { + type: "text-delta", + text: "x", + }) + assert.deepEqual(parseStreamEventLine('data: {"type":"finish","finishReason":"stop"}'), { + type: "finish", + finishReason: "stop", + }) + }) + + it("ignores comments, event labels, done markers, and malformed JSON", () => { + assert.equal(parseStreamEventLine(":"), undefined) + assert.equal(parseStreamEventLine("event: message"), undefined) + assert.equal(parseStreamEventLine("data: [DONE]"), undefined) + assert.equal(parseStreamEventLine("not-json"), undefined) + }) +}) + +describe("mapFinishReason()", () => { + it("maps provider finish reasons to pi stop reasons", () => { + assert.equal(mapFinishReason("stop"), "stop") + assert.equal(mapFinishReason("tool-calls"), "toolUse") + assert.equal(mapFinishReason("max_tokens"), "length") + assert.equal(mapFinishReason("max_output_tokens"), "length") + }) +}) diff --git a/tests/test-retry.ts b/tests/test-retry.ts new file mode 100644 index 0000000..e0366de --- /dev/null +++ b/tests/test-retry.ts @@ -0,0 +1,470 @@ +/** + * Tests for retry and timeout behaviour driven by pi settings.json + * retry config (timeoutMs, maxRetries, maxRetryDelayMs). + */ + +import assert from "node:assert/strict" +import { after, before, beforeEach, describe, it } from "node:test" + +import type { AssistantMessageEvent } from "../src/core.ts" +import { + collectEvents, + createTestDeps, + makeContext, + makeModel, + startMockCommandCodeServer, + type MockCommandCodeServer, +} from "./helpers.ts" + +const TEST_API_KEY = "option-key" + +let server: MockCommandCodeServer + +before(async () => { + server = await startMockCommandCodeServer() +}) + +after(async () => { + await server.close() +}) + +beforeEach(() => { + server.reset() +}) + +function eventTypes(events: readonly AssistantMessageEvent[]): string[] { + return events.map((event) => event.type) +} + +describe("streamCommandCode — retry on transient errors", () => { + it("retries on 429 and succeeds on the second attempt", async () => { + server.mockResponseQueue([ + { type: "error", status: 429, body: "rate limited" }, + { + type: "success", + events: [ + JSON.stringify({ type: "text-delta", text: "ok" }), + JSON.stringify({ type: "finish", finishReason: "stop" }), + ], + }, + ]) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + maxRetries: 2, + }), + ) + + assert.equal(server.requestCount(), 2) + assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "text_end", "done"]) + const done = events.at(-1) + if (done?.type !== "done") throw new Error("expected done") + assert.equal(done.reason, "stop") + }) + + it("retries on 500 and succeeds on the second attempt", async () => { + server.mockResponseQueue([ + { type: "error", status: 500, body: "internal server error" }, + { + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }, + ]) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + maxRetries: 2, + }), + ) + + assert.equal(server.requestCount(), 2) + assert.equal(events.at(-1)?.type, "done") + }) + + it("does NOT retry on 400 (non-retryable client error)", async () => { + server.mockResponse({ type: "error", status: 400, body: "bad request" }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: TEST_API_KEY }), + ) + + assert.equal(server.requestCount(), 1) + assert.deepEqual(eventTypes(events), ["start", "error"]) + const last = events.at(-1) + if (last?.type !== "error") throw new Error("expected error") + assert.match(last.error.errorMessage ?? "", /400/) + }) + + it("exhausts maxRetries and emits an error", async () => { + server.mockResponse({ type: "error", status: 503, body: "unavailable" }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + maxRetries: 3, + }), + ) + + // initial attempt + 3 retries = 4 total + assert.equal(server.requestCount(), 4) + assert.deepEqual(eventTypes(events), ["start", "error"]) + const last503 = events.at(-1) + if (last503?.type !== "error") throw new Error("expected error") + assert.match(last503.error.errorMessage ?? "", /503/) + }) +}) + +describe("streamCommandCode — Retry-After header", () => { + it("respects Retry-After delay in seconds", async () => { + let delayCalled = false + server.mockResponseQueue([ + { + type: "error", + status: 429, + body: "rate limited", + headers: { "retry-after": "2" }, + }, + { + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }, + ]) + const { streamCommandCode } = createTestDeps({ + apiBase: server.baseUrl(), + delay: async (ms: number) => { + delayCalled = true + assert.equal(ms, 2000) + }, + }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + maxRetries: 2, + }), + ) + + assert.equal(server.requestCount(), 2) + assert.equal(events.at(-1)?.type, "done") + assert.ok(delayCalled, "delay should have been called with Retry-After value") + }) + + it("fails immediately when Retry-After exceeds maxRetryDelayMs", async () => { + server.mockResponse({ + type: "error", + status: 429, + body: "rate limited", + headers: { "retry-after": "300" }, + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + maxRetryDelayMs: 10_000, + }), + ) + + assert.equal(server.requestCount(), 1) + assert.deepEqual(eventTypes(events), ["start", "error"]) + const lastMax = events.at(-1) + if (lastMax?.type !== "error") throw new Error("expected error") + assert.match(lastMax.error.errorMessage ?? "", /exceeds max/) + }) + + it("does not cap Retry-After when maxRetryDelayMs is 0", async () => { + let delayCalled = false + server.mockResponseQueue([ + { + type: "error", + status: 429, + body: "rate limited", + headers: { "retry-after": "120" }, + }, + { + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }, + ]) + const { streamCommandCode } = createTestDeps({ + apiBase: server.baseUrl(), + delay: async (ms: number) => { + delayCalled = true + assert.equal(ms, 120_000) + }, + }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + maxRetries: 1, + maxRetryDelayMs: 0, + }), + ) + + assert.equal(server.requestCount(), 2) + assert.equal(events.at(-1)?.type, "done") + assert.ok(delayCalled) + }) +}) + +describe("streamCommandCode — timeout", () => { + it("retries on per-attempt timeout and succeeds", async () => { + server.mockResponseQueue([ + { + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + hangAfterLast: true, + responseDelay: 200, + }, + { + type: "success", + events: [ + JSON.stringify({ type: "text-delta", text: "fast" }), + JSON.stringify({ type: "finish", finishReason: "stop" }), + ], + }, + ]) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + timeoutMs: 50, + maxRetries: 2, + }), + 5_000, + ) + + assert.equal(server.requestCount(), 2) + assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "text_end", "done"]) + }) + + it("retries when the response starts but the stream hangs before finish", async () => { + server.mockResponseQueue([ + { + type: "success", + events: [], + hangAfterLast: true, + }, + { + type: "success", + events: [ + JSON.stringify({ type: "text-delta", text: "ok" }), + JSON.stringify({ type: "finish", finishReason: "stop" }), + ], + }, + ]) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + timeoutMs: 50, + maxRetries: 2, + }), + 5_000, + ) + + assert.equal(server.requestCount(), 2) + assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "text_end", "done"]) + }) + + it("does NOT retry on timeout after partial text-delta was emitted", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "text-delta", text: "partial" })], + hangAfterLast: true, + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + timeoutMs: 50, + maxRetries: 2, + }), + 5_000, + ) + + assert.equal(server.requestCount(), 1) + assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "error"]) + }) + + it("emits error when all retry attempts time out", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + hangAfterLast: true, + responseDelay: 200, + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + timeoutMs: 50, + maxRetries: 1, + }), + 5_000, + ) + + // initial + 1 retry = 2 + assert.equal(server.requestCount(), 2) + assert.deepEqual(eventTypes(events), ["start", "error"]) + const error = events.at(-1) + if (error?.type !== "error") throw new Error("expected error") + assert.match(error.error.errorMessage ?? "", /timed out after 50ms/) + }) +}) + +describe("streamCommandCode — abort cancels retry loop", () => { + it("user abort stops retries immediately", async () => { + server.mockResponse({ type: "error", status: 500, body: "error" }) + const controller = new AbortController() + const { streamCommandCode } = createTestDeps({ + apiBase: server.baseUrl(), + delay: async (_ms: number, signal: AbortSignal) => { + // Abort during the retry delay + controller.abort() + // Simulate the real delay which rejects on abort + return new Promise((_, reject) => { + if (signal.aborted) reject(new DOMException("Aborted", "AbortError")) + signal.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError"))) + }) + }, + }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + signal: controller.signal, + maxRetries: 10, + }), + ) + + // Should only have made 1 request (the initial one), then aborted during delay + assert.equal(server.requestCount(), 1) + assert.deepEqual(eventTypes(events), ["start", "error"]) + const error = events.at(-1) + if (error?.type !== "error") throw new Error("expected error") + assert.equal(error.reason, "aborted") + }) +}) + +describe("streamCommandCode — retry defaults", () => { + it("uses default maxRetries of 0 when not specified", async () => { + server.mockResponse({ type: "error", status: 500, body: "error" }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + await collectEvents(streamCommandCode(makeModel(), makeContext(), { apiKey: TEST_API_KEY })) + + assert.equal(server.requestCount(), 1) + }) + + it("respects maxRetries: 0 (no retries)", async () => { + server.mockResponse({ type: "error", status: 500, body: "error" }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + maxRetries: 0, + }), + ) + + assert.equal(server.requestCount(), 1) + assert.deepEqual(eventTypes(events), ["start", "error"]) + }) +}) + +describe("streamCommandCode — stream-level error retry", () => { + it("retries when API returns 200 OK but stream contains an error event", async () => { + server.mockResponseQueue([ + { + type: "success", + events: [ + JSON.stringify({ + type: "error", + error: "Service temporarily unavailable. Please try again shortly.", + }), + ], + }, + { + type: "success", + events: [ + JSON.stringify({ type: "text-delta", text: "ok" }), + JSON.stringify({ type: "finish", finishReason: "stop" }), + ], + }, + ]) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + maxRetries: 2, + }), + ) + + assert.equal(server.requestCount(), 2) + assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "text_end", "done"]) + }) + + it("exhausts retries on persistent stream-level errors", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ + type: "error", + error: "Service temporarily unavailable. Please try again shortly.", + }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: TEST_API_KEY, + maxRetries: 3, + }), + ) + + // initial + 3 retries = 4 + assert.equal(server.requestCount(), 4) + assert.deepEqual(eventTypes(events), ["start", "error"]) + const last = events.at(-1) + if (last?.type !== "error") throw new Error("expected error") + assert.match(last.error.errorMessage ?? "", /temporarily unavailable/) + }) + + it("does NOT retry stream error when content was already emitted", async () => { + server.mockResponseQueue([ + { + type: "success", + events: [ + JSON.stringify({ type: "text-delta", text: "partial" }), + JSON.stringify({ + type: "error", + error: "Service temporarily unavailable", + }), + ], + }, + ]) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: TEST_API_KEY }), + ) + + // Only 1 request — no retry because content was already emitted. + assert.equal(server.requestCount(), 1) + assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "error"]) + }) +}) diff --git a/tests/test-stream.ts b/tests/test-stream.ts new file mode 100644 index 0000000..e30ac49 --- /dev/null +++ b/tests/test-stream.ts @@ -0,0 +1,771 @@ +/** + * Integration tests for the real streamCommandCode core using a local mock + * Command Code server. No real API key or pi runtime required. + */ + +import assert from "node:assert/strict" +import { after, before, beforeEach, describe, it } from "node:test" + +import type { AssistantMessageEvent } from "../src/core.ts" +import { MODEL_EFFORTS, thinkingLevelMapForEfforts } from "../src/models.ts" +import { + collectEvents, + createTestDeps, + makeContext, + makeModel, + objectAt, + startMockCommandCodeServer, + type MockCommandCodeServer, +} from "./helpers.ts" + +let server: MockCommandCodeServer + +before(async () => { + server = await startMockCommandCodeServer() +}) + +after(async () => { + await server.close() +}) + +beforeEach(() => { + server.reset() +}) + +function eventTypes(events: readonly AssistantMessageEvent[]): string[] { + return events.map((event) => event.type) +} + +describe("streamCommandCode — auth", () => { + it("emits a missing-key error without touching the network", async () => { + const { streamCommandCode } = createTestDeps({ + apiBase: server.baseUrl(), + env: {}, + authPaths: [], + }) + const stream = streamCommandCode(makeModel(), makeContext(), { + apiKey: "", + }) + const events = await collectEvents(stream) + + assert.deepEqual(eventTypes(events), ["error"]) + assert.equal(events[0].type, "error") + assert.equal(events[0].reason, "error") + assert.match(events[0].error.errorMessage ?? "", /No Command Code API key/) + assert.equal(server.requestCount(), 0) + }) + + it("ignores the literal env-var name and falls back to env", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ + apiBase: server.baseUrl(), + env: { COMMANDCODE_API_KEY: "env-key" }, + }) + + await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "COMMANDCODE_API_KEY" }), + ) + + assert.equal( + server.lastRequestHeaders().authorization, + "Bearer env-key", + "should resolve from env, not send the literal var name as the token", + ) + }) + + it("uses options.apiKey in the Authorization header", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ + apiBase: server.baseUrl(), + env: { COMMANDCODE_API_KEY: "env-key" }, + }) + + await collectEvents(streamCommandCode(makeModel(), makeContext(), { apiKey: "option-key" })) + + assert.equal(server.lastRequestHeaders().authorization, "Bearer option-key") + }) +}) + +describe("streamCommandCode — successful streams", () => { + it("emits start → text events → done and accumulates usage", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ type: "text-delta", text: "Hel" }), + JSON.stringify({ type: "text-delta", text: "lo" }), + JSON.stringify({ + type: "finish", + finishReason: "stop", + totalUsage: { + inputTokens: 3124, + outputTokens: 15, + inputTokenDetails: { noCacheTokens: 52, cacheReadTokens: 3072 }, + }, + }), + ], + }) + const { streamCommandCode, calculatedUsages } = createTestDeps({ + apiBase: server.baseUrl(), + }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + assert.deepEqual(eventTypes(events), [ + "start", + "text_start", + "text_delta", + "text_delta", + "text_end", + "done", + ]) + const done = events.at(-1) + assert.equal(done?.type, "done") + if (done?.type !== "done") throw new Error("expected done") + assert.equal(done.reason, "stop") + assert.equal(done.message.content[0]?.type, "text") + assert.equal( + done.message.content[0]?.type === "text" ? done.message.content[0].text : "", + "Hello", + ) + assert.equal(done.message.usage.input, 52) + assert.equal(done.message.usage.cacheRead, 3072) + assert.equal(done.message.usage.cacheWrite, 0) + assert.equal(done.message.usage.totalTokens, 3139) + assert.equal(calculatedUsages.length, 1) + }) + + it("sends images for vision-capable models", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + await collectEvents( + streamCommandCode( + makeModel({ id: "gpt-5.6-luna" }), + makeContext({ + messages: [ + { + role: "user", + content: [ + { type: "text", text: "inspect" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, + ], + }, + ], + }), + { apiKey: "mock-key" }, + ), + ) + + assert.equal( + objectAt(server.lastRequestBody(), ["params", "messages", "0", "content", "1", "image"]), + "data:image/png;base64,aGVsbG8=", + ) + }) + + it("rejects images before network access for text-only models", async () => { + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode( + makeModel({ id: "deepseek/deepseek-v4-pro" }), + makeContext({ + messages: [ + { + role: "user", + content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }], + }, + ], + }), + { apiKey: "mock-key" }, + ), + ) + + assert.equal(events.at(-1)?.type, "error") + assert.equal(server.requestCount(), 0) + }) + + it("derives uncached input when noCacheTokens is missing", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ + type: "finish", + finishReason: "stop", + totalUsage: { + inputTokens: 100, + outputTokens: 10, + inputTokenDetails: { cacheReadTokens: 75 }, + }, + }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const done = events.at(-1) + assert.equal(done?.type, "done") + if (done?.type !== "done") throw new Error("expected done") + assert.equal(done.message.usage.input, 25) + assert.equal(done.message.usage.cacheRead, 75) + assert.equal(done.message.usage.cacheWrite, 0) + assert.equal(done.message.usage.totalTokens, 110) + }) + + it("accounts for cache writes separately from uncached input", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ + type: "finish", + finishReason: "stop", + totalUsage: { + inputTokens: 100, + outputTokens: 10, + inputTokenDetails: { + noCacheTokens: 20, + cacheReadTokens: 70, + cacheWriteTokens: 10, + }, + }, + }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const done = events.at(-1) + assert.equal(done?.type, "done") + if (done?.type !== "done") throw new Error("expected done") + assert.equal(done.message.usage.input, 20) + assert.equal(done.message.usage.cacheRead, 70) + assert.equal(done.message.usage.cacheWrite, 10) + assert.equal(done.message.usage.totalTokens, 110) + }) + + it("ends on finish without waiting for an open upstream connection", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ type: "text-delta", text: "done" }), + JSON.stringify({ type: "finish", finishReason: "stop" }), + ], + hangAfterLast: true, + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + 500, + ) + + assert.equal(events.at(-1)?.type, "done") + await new Promise((resolve) => setTimeout(resolve, 50)) + assert.ok(server.responseClosedBeforeEnd(), "client should cancel the still-open response body") + }) + + it("emits reasoning and tool-call blocks in order", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ type: "reasoning-start" }), + JSON.stringify({ type: "reasoning-delta", text: "think" }), + JSON.stringify({ type: "reasoning-end" }), + JSON.stringify({ type: "text-delta", text: "Using tool" }), + JSON.stringify({ + type: "tool-call", + toolCallId: "call_1", + toolName: "read_file", + input: JSON.stringify({ path: "/tmp/x" }), + }), + JSON.stringify({ type: "finish", finishReason: "tool-calls" }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + assert.deepEqual(eventTypes(events), [ + "start", + "thinking_start", + "thinking_delta", + "thinking_end", + "text_start", + "text_delta", + "text_end", + "toolcall_start", + "toolcall_end", + "done", + ]) + const done = events.at(-1) + if (done?.type !== "done") throw new Error("expected done") + assert.equal(done.reason, "toolUse") + assert.deepEqual( + done.message.content.map((content) => content.type), + ["thinking", "text", "toolCall"], + ) + const toolCall = done.message.content[2] + assert.equal(toolCall?.type === "toolCall" ? toolCall.name : "", "read_file") + }) + + it("flushes reasoning if finish arrives without reasoning-end", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ type: "reasoning-delta", text: "unfinished thought" }), + JSON.stringify({ type: "finish", finishReason: "stop" }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const done = events.at(-1) + if (done?.type !== "done") throw new Error("expected done") + assert.equal(done.message.content[0]?.type, "thinking") + }) + + it("closes thinking block before text when reasoning-end is missing", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ type: "reasoning-start" }), + JSON.stringify({ type: "reasoning-delta", text: "thinking" }), + JSON.stringify({ type: "text-delta", text: "answer" }), + JSON.stringify({ type: "finish", finishReason: "stop" }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + assert.deepEqual(eventTypes(events), [ + "start", + "thinking_start", + "thinking_delta", + "thinking_end", + "text_start", + "text_delta", + "text_end", + "done", + ]) + }) + + it("closes thinking block before tool-call when reasoning-end is missing", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ type: "reasoning-start" }), + JSON.stringify({ type: "reasoning-delta", text: "thinking" }), + JSON.stringify({ + type: "tool-call", + toolCallId: "call_1", + toolName: "read_file", + input: JSON.stringify({ path: "/tmp/x" }), + }), + JSON.stringify({ type: "finish", finishReason: "tool-calls" }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + assert.deepEqual(eventTypes(events), [ + "start", + "thinking_start", + "thinking_delta", + "thinking_end", + "toolcall_start", + "toolcall_end", + "done", + ]) + }) +}) + +describe("streamCommandCode — request serialization", () => { + it("rejects image input before sending a lossy request", async () => { + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + const events = await collectEvents( + streamCommandCode( + makeModel(), + makeContext({ + messages: [ + { + role: "user", + content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], + }, + ], + }), + { apiKey: "mock-key" }, + ), + ) + + assert.deepEqual(eventTypes(events), ["start", "error"]) + const lastEvent = events.at(-1) + assert.equal(lastEvent?.type, "error") + if (lastEvent?.type === "error") { + assert.match(lastEvent.error.errorMessage ?? "", /does not support image content/i) + } + assert.equal(server.requestCount(), 0) + }) + it("sends the expected request body and default headers", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + const context = makeContext({ + messages: [ + { role: "user", content: "first" }, + { + role: "assistant", + content: [{ type: "text", text: "first response" }], + }, + { role: "user", content: "second" }, + ], + tools: [ + { + name: "get_weather", + description: "Get weather", + parameters: { + kind: "object", + properties: { city: { kind: "string" } }, + }, + }, + ], + }) + + await collectEvents( + streamCommandCode(makeModel(), context, { + apiKey: "mock-key", + maxTokens: 500, + }), + ) + + const body = server.lastRequestBody() + assert.equal(objectAt(body, ["config", "workingDir"]), "/repo") + assert.equal(objectAt(body, ["config", "date"]), "2026-05-05") + assert.equal(objectAt(body, ["params", "model"]), "deepseek/deepseek-v4-flash") + assert.equal(objectAt(body, ["params", "stream"]), true) + assert.equal(objectAt(body, ["params", "max_tokens"]), 500) + assert.equal(objectAt(body, ["params", "reasoning_effort"]), undefined) + assert.equal(objectAt(body, ["params", "temperature"]), 0.3) + assert.equal(objectAt(body, ["params", "system"]), "You are a test assistant.") + assert.equal(objectAt(body, ["memory"]), null) + assert.equal(objectAt(body, ["taste"]), null) + assert.equal(objectAt(body, ["skills"]), null) + assert.equal(objectAt(body, ["permissionMode"]), undefined) + assert.equal(objectAt(body, ["threadId"]), "00000000-0000-4000-8000-000000000000") + assert.equal( + objectAt(body, ["params", "messages", "1", "content", "0", "text"]), + "first response", + ) + assert.equal(objectAt(body, ["params", "tools", "0", "name"]), "get_weather") + + const headers = server.lastRequestHeaders() + assert.equal(headers.authorization, "Bearer mock-key") + assert.equal(headers["x-command-code-version"], "1.15.1") + assert.equal(headers["x-project-slug"], "repo") + assert.equal(headers["x-taste-learning"], "true") + assert.equal(headers["x-co-flag"], "false") + assert.equal(headers["x-session-id"], undefined) + }) + + it("accepts the legacy OMP nested reasoning map", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + const model = makeModel({ + id: "omp-compat-reasoning-model", + reasoning: true, + thinking: { effortMap: { high: "legacy-high" } }, + }) + + await collectEvents( + streamCommandCode(model, makeContext(), { apiKey: "mock-key", reasoning: "high" }), + ) + + assert.equal(objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]), "legacy-high") + }) + + it("forwards a supported Pi reasoning level as reasoning_effort", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + const model = makeModel({ + id: "deepseek/deepseek-v4-flash", + reasoning: true, + thinkingLevelMap: thinkingLevelMapForEfforts(MODEL_EFFORTS["deepseek/deepseek-v4-flash"]), + }) + + await collectEvents( + streamCommandCode(model, makeContext(), { apiKey: "mock-key", reasoning: "max" }), + ) + + assert.equal(objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]), "max") + }) + + it("omits reasoning_effort for off, unsupported, and unknown reasoning levels", async () => { + const model = makeModel({ + id: "deepseek/deepseek-v4-flash", + reasoning: true, + thinkingLevelMap: thinkingLevelMapForEfforts(MODEL_EFFORTS["deepseek/deepseek-v4-flash"]), + }) + + for (const reasoning of ["off", "low"] as const) { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + await collectEvents( + streamCommandCode(model, makeContext(), { apiKey: "mock-key", reasoning }), + ) + assert.equal( + objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]), + undefined, + `${reasoning} should not be sent when it has no supported Command Code field`, + ) + server.reset() + } + + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + await collectEvents( + streamCommandCode( + makeModel({ id: "new-model-without-metadata", reasoning: false }), + makeContext(), + { apiKey: "mock-key", reasoning: "high" }, + ), + ) + assert.equal(objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]), undefined) + }) + + it("caps maxTokens and passes custom headers", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + await collectEvents( + streamCommandCode(makeModel({ maxTokens: 500_000 }), makeContext(), { + apiKey: "mock-key", + maxTokens: 500_000, + headers: { "x-custom": "value" }, + }), + ) + + assert.equal(objectAt(server.lastRequestBody(), ["params", "max_tokens"]), 64_000) + assert.equal(server.lastRequestHeaders()["x-custom"], "value") + }) + + it("caps default maxTokens by the selected model", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + await collectEvents( + streamCommandCode(makeModel({ maxTokens: 8_192 }), makeContext(), { + apiKey: "mock-key", + }), + ) + + assert.equal(objectAt(server.lastRequestBody(), ["params", "max_tokens"]), 8_192) + }) + + it("serializes OMP system prompt arrays as a string", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + await collectEvents( + streamCommandCode( + makeModel(), + makeContext({ + systemPrompt: ["You are a test assistant.", "Use concise answers."] as unknown as string, + }), + { apiKey: "mock-key" }, + ), + ) + + assert.equal( + objectAt(server.lastRequestBody(), ["params", "system"]), + "You are a test assistant.\n\nUse concise answers.", + ) + }) + + it("times out a hung onResponse callback", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + const started = Date.now() + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: "mock-key", + timeoutMs: 25, + onResponse: async () => new Promise(() => {}), + }), + 1_000, + ) + + assert.ok(Date.now() - started < 500) + assert.deepEqual(eventTypes(events), ["start", "error"]) + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.match(error.error.errorMessage ?? "", /timed out after 25ms/) + }) + + it("times out a hung onPayload callback", async () => { + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + const started = Date.now() + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: "mock-key", + timeoutMs: 25, + onPayload: async () => new Promise(() => {}), + }), + 1_000, + ) + + assert.ok(Date.now() - started < 500) + assert.deepEqual(eventTypes(events), ["start", "error"]) + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.match(error.error.errorMessage ?? "", /timed out after 25ms/) + assert.equal(server.requestCount(), 0) + }) + + it("runs onPayload and onResponse hooks", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + let responseStatus = 0 + + await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: "mock-key", + onPayload: () => ({ replaced: true }), + onResponse: (response) => { + responseStatus = response.status + }, + }), + ) + + assert.equal(objectAt(server.lastRequestBody(), ["replaced"]), true) + assert.equal(responseStatus, 200) + }) +}) + +describe("streamCommandCode — upstream errors and malformed streams", () => { + it("emits error for HTTP failures", async () => { + server.mockResponse({ type: "error", status: 429, body: "rate limited" }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + assert.deepEqual(eventTypes(events), ["start", "error"]) + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.match(error.error.errorMessage ?? "", /429/) + }) + + it("emits error for provider error events", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ + type: "error", + error: { message: "provider failed" }, + }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.equal(error.error.errorMessage, "provider failed") + }) + + it("handles SSE lines, malformed lines, split chunks, and final line without newline", async () => { + const textEvent = `data: ${JSON.stringify({ type: "text-delta", text: "split" })}\n` + const finishEvent = JSON.stringify({ + type: "finish", + finishReason: "max_tokens", + }) + server.mockResponse({ + type: "success", + chunks: [ + "not json\n", + textEvent.slice(0, 12), + textEvent.slice(12), + "event: ignored\n", + "data: [DONE]\n", + finishEvent, + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const done = events.at(-1) + if (done?.type !== "done") throw new Error("expected done") + assert.equal(done.reason, "length") + assert.equal( + done.message.content[0]?.type === "text" ? done.message.content[0].text : "", + "split", + ) + }) +}) diff --git a/tests/test-transport.ts b/tests/test-transport.ts new file mode 100644 index 0000000..59ea51b --- /dev/null +++ b/tests/test-transport.ts @@ -0,0 +1,183 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { createCommandCodeTransportRouter } from "../src/transport.ts" +import type { + AssistantMessageEvent, + AssistantMessageEventStreamLike, + StreamOptions, +} from "../src/types.ts" +import { collectEvents, createTestEventStream, makeContext, makeModel } from "./helpers.ts" + +function completedStream(text: string): AssistantMessageEventStreamLike { + const stream = createTestEventStream() + const model = makeModel() + const message = { + role: "assistant" as const, + content: [{ type: "text" as const, text }], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 1, + output: 1, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 2, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop" as const, + timestamp: Date.now(), + } + const events: AssistantMessageEvent[] = [ + { type: "start", partial: message }, + { type: "text_start", contentIndex: 0, partial: message }, + { type: "text_delta", contentIndex: 0, delta: text, partial: message }, + { type: "text_end", contentIndex: 0, content: text, partial: message }, + { type: "done", reason: "stop", message }, + ] + for (const event of events) stream.push(event) + stream.end() + return stream +} + +function providerStream( + response: Response, + text: string, + options?: StreamOptions, +): AssistantMessageEventStreamLike { + const stream = createTestEventStream() + const run = async () => { + const received = await (options?.fetch ?? fetch)("https://provider.test", {}) + await options?.onResponse?.( + { status: received.status, headers: {} }, + makeModel({ api: "openai-completions" }), + ) + const source = completedStream(text) + for await (const event of source) stream.push(event) + stream.end() + } + run().catch(() => stream.end()) + return stream +} + +describe("Command Code transport router", () => { + it("keeps using the Provider API after a successful request", async () => { + let providerCalls = 0 + let generateCalls = 0 + const router = createCommandCodeTransportRouter({ + createStream: createTestEventStream, + streamProvider: (_model, _context, options) => { + providerCalls += 1 + return providerStream(new Response("ok", { status: 200 }), "provider", options) + }, + streamGenerate: () => { + generateCalls += 1 + return completedStream("generate") + }, + }) + + const options: StreamOptions = { + fetch: () => Promise.resolve(new Response("ok", { status: 200 })), + } + const first = await collectEvents(router.stream(makeModel(), makeContext(), options)) + const second = await collectEvents(router.stream(makeModel(), makeContext(), options)) + + assert.equal(first.at(-1)?.type, "done") + assert.equal(second.at(-1)?.type, "done") + assert.equal(router.getTransport(), "provider") + assert.equal(providerCalls, 2) + assert.equal(generateCalls, 0) + }) + + it("falls back only for 403 upgrade_required and remembers generate", async () => { + let providerCalls = 0 + let generateCalls = 0 + const responseBody = JSON.stringify({ + error: { code: "upgrade_required", type: "permission_error" }, + }) + const router = createCommandCodeTransportRouter({ + createStream: createTestEventStream, + streamProvider: (_model, _context, options) => { + providerCalls += 1 + return providerStream(new Response(responseBody, { status: 403 }), "blocked", options) + }, + streamGenerate: () => { + generateCalls += 1 + return completedStream("generate") + }, + }) + const options: StreamOptions = { + fetch: () => Promise.resolve(new Response(responseBody, { status: 403 })), + } + + const first = await collectEvents(router.stream(makeModel(), makeContext(), options)) + const second = await collectEvents(router.stream(makeModel(), makeContext(), options)) + + assert.equal(first.at(-1)?.type, "done") + assert.equal(second.at(-1)?.type, "done") + assert.equal(router.getTransport(), "generate") + assert.equal(providerCalls, 1) + assert.equal(generateCalls, 2) + }) + + it("re-detects the transport after the API key changes", async () => { + let providerCalls = 0 + let generateCalls = 0 + const upgradeBody = JSON.stringify({ error: { code: "upgrade_required" } }) + const router = createCommandCodeTransportRouter({ + createStream: createTestEventStream, + streamProvider: (_model, _context, options) => { + providerCalls += 1 + const response = + options?.apiKey === "go-key" + ? new Response(upgradeBody, { status: 403 }) + : new Response("ok", { status: 200 }) + return providerStream(response, "provider", options) + }, + streamGenerate: () => { + generateCalls += 1 + return completedStream("generate") + }, + }) + + await collectEvents( + router.stream(makeModel(), makeContext(), { + apiKey: "go-key", + fetch: () => Promise.resolve(new Response(upgradeBody, { status: 403 })), + }), + ) + await collectEvents( + router.stream(makeModel(), makeContext(), { + apiKey: "provider-key", + fetch: () => Promise.resolve(new Response("ok", { status: 200 })), + }), + ) + + assert.equal(router.getTransport(), "provider") + assert.equal(providerCalls, 2) + assert.equal(generateCalls, 1) + }) + + it("does not fall back for other 403 errors", async () => { + let generateCalls = 0 + const responseBody = JSON.stringify({ error: { code: "permission_denied" } }) + const router = createCommandCodeTransportRouter({ + createStream: createTestEventStream, + streamProvider: (_model, _context, options) => + providerStream(new Response(responseBody, { status: 403 }), "blocked", options), + streamGenerate: () => { + generateCalls += 1 + return completedStream("generate") + }, + }) + const options: StreamOptions = { + fetch: () => Promise.resolve(new Response(responseBody, { status: 403 })), + } + + await collectEvents(router.stream(makeModel(), makeContext(), options)) + + assert.equal(router.getTransport(), "provider") + assert.equal(generateCalls, 0) + }) +}) From 91032722172ca867bec89d95686405b8b4640400 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 18 Aug 2026 12:11:47 +0200 Subject: [PATCH 07/40] docs(api): explain automatic transport selection --- CHANGELOG.md | 5 +++-- README.md | 13 +++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 887e50d..b58dd8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,9 @@ ## Unreleased -- Replace the reverse-engineered `/alpha/generate` integration with Command Code's documented Provider API: `/provider/v1/chat/completions` for non-Claude models and `/provider/v1/messages` for Claude models. -- Use Pi's native OpenAI- and Anthropic-compatible providers for streaming, tools, reasoning, images, usage, errors, and retries while preserving dynamic model discovery, offline cache, refresh/status commands, pricing, and OAuth credentials. +- Prefer Command Code's Provider API (`/provider/v1/chat/completions` and `/provider/v1/messages`) and automatically fall back to the existing `/alpha/generate` transport only when the Provider API returns `403 upgrade_required` for a Go-plan account. +- Remember the detected transport for the running process, re-detect it when credentials change, and never fall back for unrelated authentication, permission, rate-limit, network, or server failures. +- Use Pi's native OpenAI- and Anthropic-compatible providers for Provider API streaming while preserving the existing hardened generate transport, dynamic model discovery, offline cache, refresh/status commands, pricing, and OAuth credentials. - Let `/login` use browser authentication, an explicit API-key prompt, or a directly pasted API key. - Add optional zero-data-retention headers through `COMMANDCODE_ZDR=1`. - Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended. diff --git a/README.md b/README.md index 1ac02a6..e1e4a3a 100644 --- a/README.md +++ b/README.md @@ -7,13 +7,14 @@ A custom provider for [pi](https://github.com/earendil-works/pi) that connects t > **Disclaimer:** This is an unofficial, community-maintained integration. It is not affiliated with, endorsed by, or supported by Command Code. You need your own Command Code account, API key, and a plan with Provider API access. Command Code's terms, availability, and pricing apply. -The extension uses only Command Code's documented Provider API endpoints: +The extension uses one provider and automatically selects the transport supported by the authenticated account: -- `GET /provider/v1/models` -- `POST /provider/v1/chat/completions` for non-Claude models -- `POST /provider/v1/messages` for Claude models +- `GET /provider/v1/models` for model discovery +- `POST /provider/v1/chat/completions` for non-Claude models with Provider API access +- `POST /provider/v1/messages` for Claude models with Provider API access +- `/alpha/generate` after the Provider API explicitly returns `403 upgrade_required`, which currently identifies Go-plan accounts -Every current Command Code plan except **Go** has Provider API access: GOAT, Pro, Max, Team, and Provider. +The detected transport is remembered only for the running process and is re-evaluated when the credential changes. Other authentication, permission, rate-limit, network, and server errors never trigger the fallback. ## Install @@ -92,7 +93,7 @@ Open `/model` and select one of the models provided by Command Code. Model avail ### Reasoning support -Reasoning metadata is enriched only for models whose Command Code effort support is known. Those models register a model-specific `thinkingLevelMap`, so pi and OMP expose only supported levels. Pi's native OpenAI- and Anthropic-compatible providers translate the selected level to the endpoint's documented reasoning fields. Unsupported levels and newly discovered models without metadata do not claim reasoning support. +Reasoning metadata is enriched only for models whose Command Code effort support is known. Those models register a model-specific `thinkingLevelMap`, so pi and OMP expose only supported levels. Pi's native OpenAI- and Anthropic-compatible providers translate the selected level for Provider API accounts; the existing Command Code generate transport sends the matching `reasoning_effort` for Go accounts. Unsupported levels and newly discovered models without metadata do not claim reasoning support. List Command Code models from the terminal: From 75552ef4b90e08be96de011bd5ae3623b28a5a7d Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 18 Aug 2026 17:20:44 +0200 Subject: [PATCH 08/40] test(e2e): add live account profiles --- index.ts | 1 + package.json | 3 ++ scripts/live-e2e-profile.mjs | 74 +++++++++++++++++++++++++++++++++++ src/runtime.ts | 15 ++++--- tests/test-live-e2e.mjs | 76 ++++++++++++++++++++++++------------ tests/test-runtime.ts | 2 + 6 files changed, 141 insertions(+), 30 deletions(-) create mode 100644 scripts/live-e2e-profile.mjs diff --git a/index.ts b/index.ts index d263138..1f4aeb2 100644 --- a/index.ts +++ b/index.ts @@ -127,6 +127,7 @@ export default async function (pi: ExtensionAPI) { timeoutMs: modelsTimeoutMs, }), createProviderConfig: (models) => createProviderConfig(models, apiBase, transport.stream), + getTransport: transport.getTransport, }) await runtime.initialize() diff --git a/package.json b/package.json index c99a0aa..f284d3b 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,9 @@ "test:pi-local": "node tests/test-pi-local.mjs", "test:smoke": "node tests/test-smoke.mjs", "test:e2e:live": "node tests/test-live-e2e.mjs", + "test:e2e:live:go": "node scripts/live-e2e-profile.mjs go", + "test:e2e:live:provider": "node scripts/live-e2e-profile.mjs provider", + "test:e2e:live:all": "node scripts/live-e2e-profile.mjs go provider", "test:cost": "tsx tests/test-cost.ts" }, "pi": { diff --git a/scripts/live-e2e-profile.mjs b/scripts/live-e2e-profile.mjs new file mode 100644 index 0000000..01e9bab --- /dev/null +++ b/scripts/live-e2e-profile.mjs @@ -0,0 +1,74 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process" +import { readFile } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +const projectDir = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const liveTest = resolve(projectDir, "tests", "test-live-e2e.mjs") +const profiles = process.argv.slice(2) + +if ( + profiles.length === 0 || + profiles.some((profile) => profile !== "go" && profile !== "provider") +) { + console.error("Usage: node scripts/live-e2e-profile.mjs [go|provider]") + process.exit(2) +} + +async function credentialFor(profile) { + const prefix = profile === "go" ? "COMMANDCODE_E2E_GO" : "COMMANDCODE_E2E_PROVIDER" + const direct = process.env[`${prefix}_API_KEY`]?.trim() + const file = process.env[`${prefix}_API_KEY_FILE`] + + if (direct && file) + throw new Error(`${prefix}_API_KEY and ${prefix}_API_KEY_FILE are mutually exclusive`) + if (direct) return direct + if (file) { + const credential = (await readFile(file, "utf-8")).trim() + if (credential) return credential + } + + throw new Error(`Set ${prefix}_API_KEY_FILE (recommended) or ${prefix}_API_KEY`) +} + +function runProfile(profile, apiKey) { + const modelVariable = + profile === "go" ? "COMMANDCODE_E2E_GO_MODEL" : "COMMANDCODE_E2E_PROVIDER_MODEL" + const model = process.env[modelVariable] ?? "deepseek/deepseek-v4-flash" + const env = { + ...process.env, + COMMANDCODE_API_KEY: apiKey, + COMMANDCODE_E2E_MODEL: model, + COMMANDCODE_E2E_PROFILE: profile, + } + delete env.COMMANDCODE_E2E_GO_API_KEY + delete env.COMMANDCODE_E2E_PROVIDER_API_KEY + + return new Promise((resolveRun, reject) => { + console.log(`[live-e2e:${profile}] model ${model}`) + const child = spawn(process.execPath, [liveTest], { + cwd: projectDir, + env, + stdio: "inherit", + }) + child.on("error", reject) + child.on("close", (code, signal) => { + if (code === 0) { + resolveRun() + return + } + reject(new Error(`[live-e2e:${profile}] failed (${signal ?? `exit ${code}`})`)) + }) + }) +} + +try { + for (const profile of profiles) { + await runProfile(profile, await credentialFor(profile)) + } +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exit(1) +} diff --git a/src/runtime.ts b/src/runtime.ts index 34c034a..e103f96 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -28,11 +28,13 @@ export interface CommandCodeRuntimeOptions { cachePath: string loadModels: () => Promise createProviderConfig: (models: readonly CommandCodeModel[]) => TProviderConfig + getTransport?: () => "unknown" | "provider" | "generate" now?: () => number logWarning?: (message: string) => void } export interface CommandCodeRuntimeStatus { + transport: "unknown" | "provider" | "generate" source: LoadCommandCodeModelsResult["source"] modelCount: number lastSuccess?: number @@ -86,6 +88,7 @@ function formatTimestamp(timestamp: number | undefined): string { export function formatCommandCodeStatus(status: CommandCodeRuntimeStatus): string { const lines = [ + `transport: ${status.transport}`, `source: ${status.source}`, `model count: ${status.modelCount}`, `last success: ${formatTimestamp(status.lastSuccess)}`, @@ -113,6 +116,7 @@ export class CommandCodeRuntime console.warn(`[commandcode] ${message}`)) const initialStatus: CommandCodeRuntimeStatus = { + transport: "unknown", source: "empty", modelCount: 0, cachePath: options.cachePath, @@ -123,7 +127,10 @@ export class CommandCodeRuntime { @@ -259,10 +266,8 @@ export class CommandCodeRuntime { - ctx.ui.notify( - formatCommandCodeStatus(this.status), - this.status.warning ? "warning" : "info", - ) + const status = this.getStatus() + ctx.ui.notify(formatCommandCodeStatus(status), status.warning ? "warning" : "info") }, }) } diff --git a/tests/test-live-e2e.mjs b/tests/test-live-e2e.mjs index 160882a..88ca636 100644 --- a/tests/test-live-e2e.mjs +++ b/tests/test-live-e2e.mjs @@ -25,6 +25,9 @@ import { fileURLToPath } from "node:url" const projectDir = resolve(dirname(fileURLToPath(import.meta.url)), "..") const extensionPath = join(projectDir, "index.ts") const testModel = process.env.COMMANDCODE_E2E_MODEL ?? "deepseek/deepseek-v4-flash" +const testProfile = process.env.COMMANDCODE_E2E_PROFILE +const expectedTransport = + testProfile === "go" ? "generate" : testProfile === "provider" ? "provider" : undefined const marker = "commandcode-live-e2e-ok" function findPiBinary() { @@ -57,9 +60,18 @@ if (!piBin || !hasAuthMetadata()) { process.exit(0) } +const profileAgentDir = testProfile + ? mkdtempSync(join(tmpdir(), `pi-commandcode-live-${testProfile}-agent-`)) + : undefined + function safeEnv(overrides = {}) { const env = { ...process.env, PI_SKIP_VERSION_CHECK: "1", ...overrides } - delete env.COMMANDCODE_API_KEY + if (testProfile && profileAgentDir) { + env.PI_CODING_AGENT_DIR = profileAgentDir + env.COMMANDCODE_MODELS_CACHE = join(profileAgentDir, "commandcode-models.json") + } else { + delete env.COMMANDCODE_API_KEY + } return env } @@ -228,12 +240,22 @@ try { return { firstThinkingDeltas, secondThinkingDeltas, stderr: getStderr() } }) - assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning") - assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning") + if (testProfile !== "provider") { + assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning") + assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning") + } assert.doesNotMatch(multiTurn.stderr, /Bearer\s+\S+/i) console.log("[live-e2e] live runtime refresh/status commands") const runtime = await runRpc(extensionPath, async ({ send, waitFor, getStderr }) => { + if (expectedTransport) { + send({ id: "transport-probe", type: "prompt", message: `Reply exactly: ${marker}` }) + await waitFor( + (event) => event.type === "response" && event.id === "transport-probe" && event.success, + ) + await waitFor((event) => event.type === "agent_settled") + } + send({ id: "commands", type: "get_commands" }) const commands = await waitFor( (event) => event.type === "response" && event.id === "commands" && event.success, @@ -264,6 +286,7 @@ try { assert.ok(runtime.names.includes("commandcode-refresh")) assert.ok(runtime.names.includes("commandcode-status")) assert.match(runtime.refresh, /model catalog (?:refreshed|unchanged)/) + if (expectedTransport) assert.match(runtime.status, new RegExp(`transport: ${expectedTransport}`)) assert.match(runtime.status, /source: (?:live|cache)/) assert.match(runtime.status, /model count: [1-9][0-9]*/) assert.doesNotMatch(`${runtime.refresh}\n${runtime.status}\n${runtime.stderr}`, /Bearer\s+\S+/i) @@ -296,30 +319,32 @@ try { assert.match(toolResult.stdout, new RegExp(marker)) assert.equal(readFileSync(targetPath, "utf-8"), marker) - console.log("[live-e2e] image rejection through real RPC host") - const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => { - send({ - id: "image", - type: "prompt", - message: "Describe this image", - images: [{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }], + if (testProfile !== "provider") { + console.log("[live-e2e] image rejection through real RPC host") + const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => { + send({ + id: "image", + type: "prompt", + message: "Describe this image", + images: [{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }], + }) + await waitFor((event) => event.type === "response" && event.id === "image") + await waitFor( + (event) => + event.type === "message_end" && + event.message?.role === "assistant" && + event.message?.stopReason === "error", + ) + return events }) - await waitFor((event) => event.type === "response" && event.id === "image") - await waitFor( - (event) => - event.type === "message_end" && - event.message?.role === "assistant" && - event.message?.stopReason === "error", + assert.ok( + image.some( + (event) => + event.type === "message_end" && + /does not support image content/i.test(event.message?.errorMessage ?? ""), + ), ) - return events - }) - assert.ok( - image.some( - (event) => - event.type === "message_end" && - /does not support image content/i.test(event.message?.errorMessage ?? ""), - ), - ) + } console.log("[live-e2e] packed artifact with existing authentication") const packDir = join(tempRoot, "pack") @@ -361,4 +386,5 @@ try { console.log("[live-e2e] PASS") } finally { rmSync(tempRoot, { recursive: true, force: true }) + if (profileAgentDir) rmSync(profileAgentDir, { recursive: true, force: true }) } diff --git a/tests/test-runtime.ts b/tests/test-runtime.ts index 397c54e..2a89f12 100644 --- a/tests/test-runtime.ts +++ b/tests/test-runtime.ts @@ -98,6 +98,7 @@ describe("Command Code runtime", () => { cachePath: "/tmp/commandcode-models.json", loadModels: () => firstLoad.promise, createProviderConfig: (models) => ({ models }), + getTransport: () => "provider", now: () => now, logWarning: () => {}, }) @@ -115,6 +116,7 @@ describe("Command Code runtime", () => { assert.ok(statusCommand) await statusCommand("", context) const statusMessage = context.notifications.at(-1)?.message ?? "" + assert.match(statusMessage, /transport: provider/) assert.match(statusMessage, /source: live/) assert.match(statusMessage, /model count: 1/) assert.match(statusMessage, /last success:/) From 33c405c06005f0c430f9ad260d53d90173c6853a Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 18 Aug 2026 17:20:51 +0200 Subject: [PATCH 09/40] docs(tests): document live account credentials --- CHANGELOG.md | 1 + CONTRIBUTING.md | 9 +++++++++ README.md | 20 ++++++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b58dd8a..cf4942c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Let `/login` use browser authentication, an explicit API-key prompt, or a directly pasted API key. - Add optional zero-data-retention headers through `COMMANDCODE_ZDR=1`. - Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended. +- Add isolated live E2E profiles for separate Go-plan and Provider-API credentials, including an explicit selected-transport assertion and packed-package validation. ## 0.5.1 - 2026-08-11 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da4d0a4..6adb7bb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,15 @@ npm run pi:authenticated Both commands accept additional pi arguments after `--`, for example `npm run pi:authenticated -- --model claude-sonnet-4-6`. +Run the transport-specific live tests with separate credentials: + +```sh +COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key npm run test:e2e:live:go +COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key npm run test:e2e:live:provider +``` + +Use `npm run test:e2e:live:all` with both file variables to run them sequentially. Store the keys in a secret manager and export each one to a new mode-`0600` temporary file for the test; never add key files to the repository. The direct `COMMANDCODE_E2E_GO_API_KEY` and `COMMANDCODE_E2E_PROVIDER_API_KEY` variables are intended primarily for protected CI secrets. + Before opening a PR, run: ```sh diff --git a/README.md b/README.md index e1e4a3a..6c37492 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,26 @@ npm run pi:authenticated Both commands accept additional pi arguments after `--`, for example `npm run pi:authenticated -- --model claude-sonnet-4-6`. +### Live transport tests + +Keep the Go-plan and Provider-API test keys in separate secret-manager entries. Pass them through protected files so the keys do not enter shell history: + +```sh +COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \ + npm run test:e2e:live:go + +COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key \ + npm run test:e2e:live:provider + +COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \ +COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key \ + npm run test:e2e:live:all +``` + +Each profile runs with an isolated Pi agent directory and asserts the selected transport through `/commandcode-status`: Go must select `generate`, while a Provider API account must select `provider`. The profile-specific `*_API_KEY` environment variables are also supported for CI secrets, but key files are preferred for local use. + +Override the default DeepSeek test model with `COMMANDCODE_E2E_GO_MODEL` or `COMMANDCODE_E2E_PROVIDER_MODEL`. A successful live Anthropic `/provider/v1/messages` test requires a Provider API account whose plan includes the selected Claude model. + See [CONTRIBUTING.md](CONTRIBUTING.md) for local setup and tests. See [RELEASE.md](RELEASE.md) for the release process. ## License From 4f91a9b00b4fd06b87978986dbaa1748186968ff Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Thu, 20 Aug 2026 00:13:56 +0200 Subject: [PATCH 10/40] fix(stream): isolate transport state by credential --- src/transport.ts | 5 ++-- tests/test-transport.ts | 64 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/src/transport.ts b/src/transport.ts index 9d83d5d..c30171c 100644 --- a/src/transport.ts +++ b/src/transport.ts @@ -71,6 +71,7 @@ export function createCommandCodeTransportRouter(deps: TransportDependencies) { apiKey = options?.apiKey transport = "unknown" } + const requestApiKey = options?.apiKey if (transport === "generate") return deps.streamGenerate(model, context, options) const output = deps.createStream() @@ -94,13 +95,13 @@ export function createCommandCodeTransportRouter(deps: TransportDependencies) { for await (const event of providerStream) { if (!upgradeRequired) { - transport = "provider" + if (apiKey === requestApiKey) transport = "provider" output.push(event) } } if (upgradeRequired) { - transport = "generate" + if (apiKey === requestApiKey) transport = "generate" await pipe(deps.streamGenerate(model, context, options), output) } output.end() diff --git a/tests/test-transport.ts b/tests/test-transport.ts index 59ea51b..82e9771 100644 --- a/tests/test-transport.ts +++ b/tests/test-transport.ts @@ -159,6 +159,70 @@ describe("Command Code transport router", () => { assert.equal(generateCalls, 1) }) + it("does not let a stale request overwrite the transport for a new API key", async () => { + let releaseGoRequest: (() => void) | undefined + const goRequestGate = new Promise((resolve) => { + releaseGoRequest = resolve + }) + let providerCalls = 0 + let generateCalls = 0 + const upgradeBody = JSON.stringify({ error: { code: "upgrade_required" } }) + const router = createCommandCodeTransportRouter({ + createStream: createTestEventStream, + streamProvider: (_model, _context, options) => { + providerCalls += 1 + const response = + options?.apiKey === "go-key" + ? new Response(upgradeBody, { status: 403 }) + : new Response("ok", { status: 200 }) + const stream = createTestEventStream() + const run = async () => { + if (options?.apiKey === "go-key") await goRequestGate + const received = await (options?.fetch ?? fetch)("https://provider.test", {}) + await options?.onResponse?.( + { status: received.status, headers: {} }, + makeModel({ api: "openai-completions" }), + ) + if (response.ok) { + for await (const event of completedStream("provider")) stream.push(event) + } + stream.end() + } + run().catch(() => stream.end()) + return stream + }, + streamGenerate: () => { + generateCalls += 1 + return completedStream("generate") + }, + }) + + const staleGoRequest = collectEvents( + router.stream(makeModel(), makeContext(), { + apiKey: "go-key", + fetch: () => Promise.resolve(new Response(upgradeBody, { status: 403 })), + }), + ) + await collectEvents( + router.stream(makeModel(), makeContext(), { + apiKey: "provider-key", + fetch: () => Promise.resolve(new Response("ok", { status: 200 })), + }), + ) + releaseGoRequest?.() + await staleGoRequest + await collectEvents( + router.stream(makeModel(), makeContext(), { + apiKey: "provider-key", + fetch: () => Promise.resolve(new Response("ok", { status: 200 })), + }), + ) + + assert.equal(router.getTransport(), "provider") + assert.equal(providerCalls, 3) + assert.equal(generateCalls, 1) + }) + it("does not fall back for other 403 errors", async () => { let generateCalls = 0 const responseBody = JSON.stringify({ error: { code: "permission_denied" } }) From ed29c59f3174f7fbcca1ed9d900c37f15c117ab2 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Thu, 20 Aug 2026 00:13:56 +0200 Subject: [PATCH 11/40] fix(models): use adaptive thinking for Claude --- index.ts | 1 + tests/test-pi-local.mjs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/index.ts b/index.ts index 1f4aeb2..72ae558 100644 --- a/index.ts +++ b/index.ts @@ -85,6 +85,7 @@ function createProviderConfig( supportsLongCacheRetention: false, supportsCacheControlOnTools: false, supportsToolReferences: false, + ...(model.reasoning ? { forceAdaptiveThinking: true } : {}), }, })), } diff --git a/tests/test-pi-local.mjs b/tests/test-pi-local.mjs index 760a100..2d0a999 100644 --- a/tests/test-pi-local.mjs +++ b/tests/test-pi-local.mjs @@ -731,6 +731,8 @@ try { "commandcode", "--model", CLAUDE_TEST_MODEL, + "--thinking", + "high", ], 30_000, ) @@ -738,6 +740,8 @@ try { assert.match(claudePrint.stdout, /mock-pi-ok/) assert.equal(requestCount, 1) assert.equal(lastRequestBody?.model, CLAUDE_TEST_MODEL) + assert.equal(lastRequestBody?.thinking?.type, "adaptive") + assert.deepEqual(lastRequestBody?.output_config, { effort: "high" }) assert.equal(lastRequestHeaders["x-api-key"], "mock-key") assert.equal(lastRequestHeaders["x-cmd-zdr"], "1") From f6f0ab274d251c44fbd88844e1997e950ad19144 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Thu, 20 Aug 2026 00:13:56 +0200 Subject: [PATCH 12/40] fix(models): refresh DeepSeek V4 pricing --- src/pricing.ts | 17 +++++++++-------- tests/fixtures/commandcode-pricing.json | 6 +++--- tests/test-pricing.ts | 14 ++++++++++---- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/pricing.ts b/src/pricing.ts index f13a69e..0dd9115 100644 --- a/src/pricing.ts +++ b/src/pricing.ts @@ -20,7 +20,7 @@ export interface TemporaryPricing { } export const PRICING_SOURCE_URL = "https://commandcode.ai/docs/resources/pricing-limits" -export const PRICING_LAST_VERIFIED = "2026-08-18" +export const PRICING_LAST_VERIFIED = "2026-08-20" export const ZERO_MODEL_COST: CommandCodeModelCost = { input: 0, @@ -61,17 +61,18 @@ export const MODEL_COSTS: Readonly> = { "MiniMaxAI/MiniMax-M3": { input: 0.3, output: 1.2, cacheRead: 0.06, cacheWrite: 0 }, "MiniMaxAI/MiniMax-M2.7": { input: 0.3, output: 1.2, cacheRead: 0.06, cacheWrite: 0 }, "MiniMaxAI/MiniMax-M2.5": { input: 0.3, output: 1.2, cacheRead: 0.03, cacheWrite: 0 }, - // Permanent 75% discount. + // DeepSeek V4 uses time-dependent rates. Display the documented off-peak + // rates, which apply for 17 hours per day; the Usage page remains authoritative. "deepseek/deepseek-v4-pro": { - input: 0.435, - output: 0.87, - cacheRead: 0.003625, + input: 0.66, + output: 1.98, + cacheRead: 0.022, cacheWrite: 0, }, "deepseek/deepseek-v4-flash": { - input: 0.14, - output: 0.28, - cacheRead: 0.0028, + input: 0.22, + output: 0.66, + cacheRead: 0.007, cacheWrite: 0, }, "Qwen/Qwen3.8-Max": { input: 2, output: 6, cacheRead: 0.25, cacheWrite: 2.5 }, diff --git a/tests/fixtures/commandcode-pricing.json b/tests/fixtures/commandcode-pricing.json index 786cec2..6a508fd 100644 --- a/tests/fixtures/commandcode-pricing.json +++ b/tests/fixtures/commandcode-pricing.json @@ -1,5 +1,5 @@ { - "verifiedAt": "2026-08-18", + "verifiedAt": "2026-08-20", "source": "https://commandcode.ai/docs/resources/pricing-limits", "tierPolicy": "Use request-wide input tiers; the highest threshold exceeded by input plus cache tokens applies to the full request.", "tiers": { @@ -25,8 +25,8 @@ "MiniMaxAI/MiniMax-M3": [0.3, 1.2, 0.06, 0], "MiniMaxAI/MiniMax-M2.7": [0.3, 1.2, 0.06, 0], "MiniMaxAI/MiniMax-M2.5": [0.3, 1.2, 0.03, 0], - "deepseek/deepseek-v4-pro": [0.435, 0.87, 0.003625, 0], - "deepseek/deepseek-v4-flash": [0.14, 0.28, 0.0028, 0], + "deepseek/deepseek-v4-pro": [0.66, 1.98, 0.022, 0], + "deepseek/deepseek-v4-flash": [0.22, 0.66, 0.007, 0], "Qwen/Qwen3.8-Max": [2, 6, 0.25, 2.5], "Qwen/Qwen3.7-Max": [2.5, 7.5, 0.5, 3.13], "Qwen/Qwen3.7-Plus": [0.4, 1.6, 0.08, 0.5], diff --git a/tests/test-pricing.ts b/tests/test-pricing.ts index c463fcf..6e17400 100644 --- a/tests/test-pricing.ts +++ b/tests/test-pricing.ts @@ -108,10 +108,16 @@ describe("MODEL_COSTS pricing overlay", () => { }) it("matches corrected official rates", () => { + assertCost("deepseek/deepseek-v4-pro", { + input: 0.66, + output: 1.98, + cacheRead: 0.022, + cacheWrite: 0, + }) assertCost("deepseek/deepseek-v4-flash", { - input: 0.14, - output: 0.28, - cacheRead: 0.0028, + input: 0.22, + output: 0.66, + cacheRead: 0.007, cacheWrite: 0, }) assertCost("Qwen/Qwen3.7-Max", { @@ -163,7 +169,7 @@ describe("MODEL_COSTS pricing overlay", () => { it("tracks pricing provenance", () => { assert.equal(PRICING_SOURCE_URL, "https://commandcode.ai/docs/resources/pricing-limits") - assert.equal(PRICING_LAST_VERIFIED, "2026-08-18") + assert.equal(PRICING_LAST_VERIFIED, "2026-08-20") }) it("fails once temporary pricing needs review", () => { From f07ece7731c03bed7a55f469b0701af982854257 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Thu, 20 Aug 2026 00:13:56 +0200 Subject: [PATCH 13/40] docs(api): document provider follow-up fixes --- CHANGELOG.md | 6 +++--- README.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf4942c..ee3cd9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,11 +3,11 @@ ## Unreleased - Prefer Command Code's Provider API (`/provider/v1/chat/completions` and `/provider/v1/messages`) and automatically fall back to the existing `/alpha/generate` transport only when the Provider API returns `403 upgrade_required` for a Go-plan account. -- Remember the detected transport for the running process, re-detect it when credentials change, and never fall back for unrelated authentication, permission, rate-limit, network, or server failures. -- Use Pi's native OpenAI- and Anthropic-compatible providers for Provider API streaming while preserving the existing hardened generate transport, dynamic model discovery, offline cache, refresh/status commands, pricing, and OAuth credentials. +- Remember the detected transport for the running process, re-detect it when credentials change, prevent stale in-flight requests from overwriting the new credential's transport, and never fall back for unrelated authentication, permission, rate-limit, network, or server failures. +- Use Pi's native OpenAI- and Anthropic-compatible providers for Provider API streaming, including adaptive thinking for current reasoning-capable Claude models, while preserving the existing hardened generate transport, dynamic model discovery, offline cache, refresh/status commands, pricing, and OAuth credentials. - Let `/login` use browser authentication, an explicit API-key prompt, or a directly pasted API key. - Add optional zero-data-retention headers through `COMMANDCODE_ZDR=1`. -- Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended. +- Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended, and display the current DeepSeek V4 off-peak rates for its time-dependent pricing. - Add isolated live E2E profiles for separate Go-plan and Provider-API credentials, including an explicit selected-transport assertion and packed-package validation. ## 0.5.1 - 2026-08-11 diff --git a/README.md b/README.md index 6c37492..4b32189 100644 --- a/README.md +++ b/README.md @@ -147,9 +147,9 @@ For vision-capable models, Pi's native provider adapters forward image blocks fr ## Pricing display -The Command Code Provider API does not currently include prices in its model catalog. This extension therefore keeps a static table for models with known prices so pi can display estimated request costs. +The Command Code Provider API does not currently include prices in its model catalog. This extension therefore keeps a static table for models with known prices so pi can display estimated request costs. DeepSeek V4 uses time-dependent rates; pi displays the documented off-peak rate, which applies for 17 hours per day. -Models missing from that table display zero cost in pi. This does **not** mean that Command Code will bill the request at zero. Check the current [Command Code pricing](https://commandcode.ai/docs/resources/pricing-limits) before relying on the displayed value. +Models missing from that table display zero cost in pi. This does **not** mean that Command Code will bill the request at zero. The Command Code Usage page remains authoritative for each request. Check the current [Command Code pricing](https://commandcode.ai/docs/resources/pricing-limits) before relying on the displayed value. ## Update and remove From 1e0dd1189cb7dbc28cd90fbe346455803a6fa0b1 Mon Sep 17 00:00:00 2001 From: Omar Iqbal Naru Date: Fri, 21 Aug 2026 19:41:54 +0500 Subject: [PATCH 14/40] fix(core): register custom api under non-reserved name for omp 17.4.0 omp 17.4.0's host registry rejects registerCustomApi calls under built-in api names ("Cannot register custom API '': built-in API names are reserved"). The provider and its models registered under "openai-completions", so the extension failed to load. Register under "commandcode-custom" instead (matches the published npm build) and restore the real wire api via apiForModelId before dispatching to the native compat stream inside the transport router. The host's model registry also stores provider-supplied compat under model.compatConfig internally, only copying it back to model.compat inside its own dispatch-time patches. Since the transport router calls the native compat stream directly, it must read compatConfig itself or requests crash with 'baseCompat is undefined' before any network call. Verified against a local mock Provider API server with the extension linked into omp 17.4.0: model discovery lists all Command Code models, and a streamed chat completion sends the resolved x-cmd-zdr header and Authorization header end to end. --- index.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/index.ts b/index.ts index 72ae558..79349bd 100644 --- a/index.ts +++ b/index.ts @@ -19,6 +19,7 @@ import { getConfiguredApiKey } from "./src/api-key.ts" import { createStreamCommandCode } from "./src/core.ts" import { calculateCommandCodeCost } from "./src/cost.ts" import { + apiForModelId, baseUrlForModel, DEFAULT_MODELS_URL, DEFAULT_PROVIDER_API_BASE, @@ -51,7 +52,7 @@ function createProviderConfig( name: "Command Code", baseUrl: apiBase, apiKey: getConfiguredApiKey() ?? "$COMMANDCODE_API_KEY", - api: "openai-completions", + api: "commandcode-custom", streamSimple: streamCommandCode, headers, oauth: { @@ -63,7 +64,7 @@ function createProviderConfig( models: models.map((model) => ({ id: model.id, name: model.name, - api: model.api, + api: "commandcode-custom", baseUrl: baseUrlForModel(apiBase, model.api), reasoning: model.reasoning, ...(thinkingMetadataForModel(model.id) ?? {}), @@ -108,7 +109,12 @@ export default async function (pi: ExtensionAPI) { }) const transport = createCommandCodeTransportRouter({ createStream: () => new AssistantMessageEventStream(), - streamProvider: streamNativeProvider, + streamProvider: (model, context, options) => + streamNativeProvider( + { ...model, api: apiForModelId(model.id), compat: model.compatConfig ?? model.compat }, + context, + options, + ), streamGenerate, }) From 0d6202aec7fac14912d9baa3de5deaf716207292 Mon Sep 17 00:00:00 2001 From: Omar Iqbal Naru Date: Fri, 21 Aug 2026 19:42:14 +0500 Subject: [PATCH 15/40] docs(release): note omp custom api registration fix --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee3cd9b..d01c553 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Add optional zero-data-retention headers through `COMMANDCODE_ZDR=1`. - Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended, and display the current DeepSeek V4 off-peak rates for its time-dependent pricing. - Add isolated live E2E profiles for separate Go-plan and Provider-API credentials, including an explicit selected-transport assertion and packed-package validation. +- Fix extension load failure on newer pi hosts that reject registering a custom API under a built-in name (`openai-completions`); register under `commandcode-custom` instead and restore the real wire API before native compat dispatch. ## 0.5.1 - 2026-08-11 From 307714b0511cdcf11d65ae13ac3dbf06f26d986d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Galliano?= Date: Fri, 21 Aug 2026 11:21:25 -0600 Subject: [PATCH 16/40] feat(quota): add live commandcode-quota dashboard and OMP auth fix Fetches live account quota from Command Code alpha usage endpoints (whoami, billing/credits, billing/subscriptions, usage/summary) and renders a plain-text dashboard via ui.notify. - Graceful degradation: optional endpoint transport/timeout/parse failures degrade to null sections instead of aborting; only 401/403 are hard failures. 429 is transient, not fatal. - Overall deadline bounds the whole command to QUOTA_TIMEOUT_MS (per-request controllers are chained; post-deadline phases fail fast). - OMP auth: filter unresolved $COMMANDCODE_API_KEY placeholder and fall back to the host resolver via pickCommandCodeApiKey. - ZDR privacy header respected on quota requests. - Redaction reuses redactCommandCodeErrorText plus JSON-quoted credential fields; outer-catch errors are redacted too. - resetAt parsed from seconds, ms, numeric string, or ISO string. - 21 hermetic unit tests wired into npm test (test:quota). --- CHANGELOG.md | 4 + README.md | 3 + index.ts | 49 +++ package.json | 3 +- src/converters.ts | 23 ++ src/quota.ts | 613 +++++++++++++++++++++++++++++++++++ tests/test-omp-compat.mjs | 8 +- tests/test-pure-functions.ts | 33 +- tests/test-quota.ts | 395 ++++++++++++++++++++++ 9 files changed, 1128 insertions(+), 3 deletions(-) create mode 100644 src/quota.ts create mode 100644 tests/test-quota.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index ee3cd9b..81128f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +- Reformat `/commandcode-quota` output into a dashboard layout: credits remaining/used with a percentage, monthly/purchased/free sources, plan, month-to-date cost/requests/tokens, and API key name, while keeping the 5-hour/weekly usage windows and full-detail link. +- Optionally show an aggregate token count and API key name when the usage endpoints report them. +- Fix `/commandcode-quota` on Oh My Pi: OMP surfaces an unresolved `$COMMANDCODE_API_KEY` placeholder through the model registry, which previously caused a 401 on the quota endpoints. The command now filters placeholders and falls back to the env/auth-file key resolver, matching the stream path. +- Fix `/commandcode-quota` usage windows so the 5-hour and weekly limits actually render: the API reports `windowLimits` as a top-level sibling of `credits` (not nested inside it), and its `resetAt` is in milliseconds, not seconds. Both are now parsed correctly, giving real "resets in …" countdowns instead of omitting the section or showing a huge day count. - Prefer Command Code's Provider API (`/provider/v1/chat/completions` and `/provider/v1/messages`) and automatically fall back to the existing `/alpha/generate` transport only when the Provider API returns `403 upgrade_required` for a Go-plan account. - Remember the detected transport for the running process, re-detect it when credentials change, prevent stale in-flight requests from overwriting the new credential's transport, and never fall back for unrelated authentication, permission, rate-limit, network, or server failures. - Use Pi's native OpenAI- and Anthropic-compatible providers for Provider API streaming, including adaptive thinking for current reasoning-capable Claude models, while preserving the existing hardened generate transport, dynamic model discovery, offline cache, refresh/status commands, pricing, and OAuth credentials. diff --git a/README.md b/README.md index 4b32189..b140206 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,9 @@ While pi is running, use these provider commands without restarting: - `/commandcode-refresh` fetches and re-registers the current model catalog. Overlapping refreshes are coalesced, and a failed refresh keeps the last valid catalog active. - `/commandcode-status` shows redacted discovery diagnostics, including the source, model count, timestamps, cache path, endpoint, and warning. +- `/commandcode-quota` shows your Command Code account usage and quota in a dashboard-style layout: credits remaining and used with a percentage, monthly/purchased/free sources, the current plan, month-to-date cost/requests/tokens, the API key name, and the 5-hour and weekly usage windows. + +The `commandcode-quota` command reads from the Command Code alpha usage endpoints (the same ones the `cmd` CLI `/usage` command uses): `whoami`, `billing/credits`, `billing/subscriptions`, and `usage/summary`. It authenticates with the same API key the provider already uses. If command cannot reach those endpoints or they change, the command reports a readable error instead of failing. Output is plain text (via `ui.notify`) so it works across pi and compatible hosts such as OMP. Set `COMMANDCODE_ZDR=1` to send Command Code's documented `x-cmd-zdr: 1` zero-data-retention header. diff --git a/index.ts b/index.ts index 72ae558..be7891f 100644 --- a/index.ts +++ b/index.ts @@ -18,6 +18,7 @@ import { join } from "node:path" import { getConfiguredApiKey } from "./src/api-key.ts" import { createStreamCommandCode } from "./src/core.ts" import { calculateCommandCodeCost } from "./src/cost.ts" +import { pickCommandCodeApiKey } from "./src/converters.ts" import { baseUrlForModel, DEFAULT_MODELS_URL, @@ -32,8 +33,19 @@ import { getApiKey as getOAuthApiKey, login, refreshToken } from "./src/oauth.ts import { normalizeCommandCodeMessage } from "./src/overflow.ts" import { MODEL_COSTS, ZERO_MODEL_COST } from "./src/pricing.ts" import { createCommandCodeRuntime } from "./src/runtime.ts" +import { fetchCommandCodeQuota, formatQuota, redactValue } from "./src/quota.ts" import { createCommandCodeTransportRouter } from "./src/transport.ts" +const COMMAND_CODE_PROVIDER_ID = "commandcode" + +async function resolveCommandCodeApiKey(ctx: ExtensionCommandContext): Promise { + const registryKey = await ctx.modelRegistry?.getApiKeyForProvider?.(COMMAND_CODE_PROVIDER_ID) + // Mirror src/core.ts: OMP may surface an unresolved placeholder; fall back + // to the env/auth-file resolver so we never send a literal placeholder as a + // Bearer token (which caused a 401 on /alpha/whoami). + return pickCommandCodeApiKey(registryKey, getConfiguredApiKey()) +} + function commandCodeHeaders(): Record | undefined { if (process.env.COMMANDCODE_ZDR === "1") { return { "x-cmd-zdr": "1" } @@ -118,6 +130,43 @@ export default async function (pi: ExtensionAPI) { return normalized ? { message: normalized.message } : undefined }) + pi.registerCommand("commandcode-quota", { + description: "Show Command Code account usage and quota", + handler: async (_args, ctx) => { + await ctx.waitForIdle?.() + + // Resolve the key in a host-agnostic way so the command also works on + // OMP (which passes an unresolved "$COMMANDCODE_API_KEY" placeholder + // through the registry): filter placeholders and fall back to the + // env/auth-file resolver, mirroring src/core.ts. + const apiKey = await resolveCommandCodeApiKey(ctx) + if (!apiKey) { + ctx.ui.notify( + "Command Code quota requires an API key. Run /login and select Command Code, or set the COMMANDCODE_API_KEY env var.", + "warning", + ) + return + } + + const result = await fetchCommandCodeQuota({ + apiKey, + // Alpha endpoints live under the legacy base (no /provider/v1), + // same as the fallback generate transport. + baseUrl: legacyApiBase(apiBase), + // Respect the user's zero-data-retention preference on usage/account + // calls too, matching the provider stream path. + extraHeaders: commandCodeHeaders(), + }) + + if (!result.ok) { + ctx.ui.notify(redactValue(result.error.message), "error") + return + } + + ctx.ui.notify(formatQuota(result.quota), "info") + }, + }) + const runtime = createCommandCodeRuntime(pi, { endpoint: modelsUrl, cachePath: modelsCachePath, diff --git a/package.json b/package.json index f284d3b..e00b16f 100644 --- a/package.json +++ b/package.json @@ -29,12 +29,13 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", "pi:isolated": "node scripts/pi-isolated.mjs", "pi:authenticated": "node scripts/pi-authenticated.mjs", + "test:quota": "tsx tests/test-quota.ts", "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", "test:api-key": "tsx tests/test-api-key.ts", "test:models": "tsx tests/test-models.ts", diff --git a/src/converters.ts b/src/converters.ts index 4012fbb..bb60df2 100644 --- a/src/converters.ts +++ b/src/converters.ts @@ -138,6 +138,29 @@ export function getApiKey( return undefined } +// Hosts such as OMP may pass the literal env-var name "$COMMANDCODE_API_KEY" +// (or "COMMANDCODE_API_KEY") as the "resolved" registry key instead of the +// actual credential. Treat those as unresolved. +export const COMMAND_CODE_PLACEHOLDER_KEYS = new Set([ + "$COMMANDCODE_API_KEY", + "COMMANDCODE_API_KEY", +]) + +/** + * Pick the real API key from a host registry value and/or the env/auth-file + * fallback, never returning a literal placeholder or an empty/whitespace value. + * Pure/testable. + */ +export function pickCommandCodeApiKey( + registryKey: string | undefined, + hostKey: string | undefined, +): string | undefined { + const trimmed = typeof registryKey === "string" ? registryKey.trim() : undefined + if (!trimmed) return hostKey + if (COMMAND_CODE_PLACEHOLDER_KEYS.has(trimmed)) return hostKey + return trimmed +} + export function textContent(message: { content?: unknown }): string { return recordArray(message.content) .filter((part) => part.type === "text") diff --git a/src/quota.ts b/src/quota.ts new file mode 100644 index 0000000..15a0263 --- /dev/null +++ b/src/quota.ts @@ -0,0 +1,613 @@ +/** + * Command Code usage/quota fetch layer for the `/commandcode-quota` command. + * + * Command Code exposes account usage through a set of authenticated alpha + * endpoints (the same ones the `cmd` CLI `/usage` command uses): + * + * - `/alpha/whoami` -> resolved account + optional org id + * - `/alpha/billing/credits` -> monthly/purchased/free credits + window limits + * - `/alpha/billing/subscriptions`-> plan id, status, billing period + * - `/alpha/usage/summary` -> period totals (cost, request count, optional tokens) + * + * These endpoints are not part of the documented public Provider API + * (`/provider/v1/*`) but are shipped with every `command-code` CLI release and + * authenticate with the same API key the provider already uses. Fetches are + * wrapped defensively so the quota command degrades to a readable error rather + * than surfacing raw transport details. + */ + +import { redactCommandCodeErrorText } from "./overflow.ts" + +export const DEFAULT_API_BASE = "https://api.commandcode.ai" + +export const QUOTA_TIMEOUT_MS = 15_000 + +/** + * A single rolling usage window (the 5-hour or weekly cap on a plan's monthly + * credits). Values are measured in credit value, not request count. + */ +export interface CommandCodeWindowLimit { + window: "fiveHour" | "weekly" + used: number + cap: number + /** Unix epoch seconds when this window resets, normalized from seconds or ms. */ + resetAt: number | null +} + +/** Credits exposed by the `/alpha/billing/credits` endpoint. */ +export interface CommandCodeCredits { + monthlyCredits: number + purchasedCredits: number + freeCredits: number + remainingCredits: number + windowLimits: CommandCodeWindowLimit[] +} + +/** Subscription/plan info exposed by `/alpha/billing/subscriptions`. */ +export interface CommandCodeSubscription { + planId: string | null + status: string | null + currentPeriodStart: string | null + currentPeriodEnd: string | null +} + +/** Period totals exposed by `/alpha/usage/summary`. */ +export interface CommandCodeUsageSummary { + totalCost: number + totalCount: number + /** Optional aggregate token count; only shown when the endpoint reports it. */ + totalTokens?: number +} + +/** Fully normalized quota snapshot for display. */ +export interface CommandCodeQuota { + account: { + login: string + orgId: string | null + /** Optional API key / account display name; falls back to login. */ + keyName?: string + } + credits: CommandCodeCredits | null + subscription: CommandCodeSubscription | null + summary: CommandCodeUsageSummary | null +} + +export type CommandCodeQuotaResult = + | { ok: true; quota: CommandCodeQuota } + | { ok: false; error: { message: string; kind: "config" | "http" | "network" | "timeout" } } + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function numberValue(value: unknown): number | undefined { + if (typeof value !== "number" || !Number.isFinite(value)) return undefined + return value >= 0 ? value : undefined +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined +} + +interface FetchOptions { + apiKey: string + baseUrl?: string + fetchImpl?: typeof fetch + timeoutMs?: number + /** Extra HTTP headers merged after Content-Type/Authorization (e.g. ZDR). */ + extraHeaders?: Record +} + +function buildUrl(path: string, params: Record): string { + const search = new URLSearchParams() + for (const [key, value] of Object.entries(params)) { + if (value !== undefined && value !== null && value !== "") search.set(key, value) + } + const query = search.toString() + return `${path}${query ? `?${query}` : ""}` +} + +/** Extract the WindowLimit array from the top-level `windowLimits` object. */ +export function windowLimitsFromCredits(value: unknown): CommandCodeWindowLimit[] { + if (!isRecord(value)) return [] + const limits: CommandCodeWindowLimit[] = [] + + for (const [window, entry] of [ + ["fiveHour", value.fiveHour], + ["weekly", value.weekly], + ] as const) { + if (!isRecord(entry)) continue + const used = numberValue(entry.used) ?? 0 + const cap = numberValue(entry.cap) ?? 0 + if (cap <= 0 && used <= 0) continue + limits.push({ + window, + used, + cap, + resetAt: normalizeResetAt(entry.resetAt), + }) + } + + return limits +} + +/** + * Normalize a `resetAt` value to epoch seconds. Accepts seconds (10-digit), + * milliseconds (13-digit, live API), a numeric string, or an ISO timestamp + * string, then converts ms -> s consistently. Invalid/negative values -> null. + */ +function normalizeResetAt(value: unknown): number | null { + let num: number | undefined + if (typeof value === "number" && Number.isFinite(value)) { + num = value + } else if (typeof value === "string" && value.length > 0) { + const trimmed = value.trim() + if (/^\d+$/.test(trimmed)) { + num = Number(trimmed) + } else { + const parsed = Date.parse(trimmed) + if (!Number.isNaN(parsed)) num = Math.round(parsed / 1000) + } + } + if (num === undefined || num < 0) return null + return num >= 1e12 ? Math.round(num / 1000) : num +} + +function parseCredits(value: unknown): CommandCodeCredits | null { + const credits = isRecord(value) ? value.credits : undefined + if (!isRecord(credits)) return null + + // `windowLimits` is a top-level sibling of `credits` in the + // `/alpha/billing/credits` response, not nested inside it. + const windowLimits = isRecord(value) ? value.windowLimits : undefined + + const monthlyCredits = numberValue(credits.monthlyCredits) ?? 0 + const purchasedCredits = numberValue(credits.purchasedCredits) ?? 0 + const freeCredits = numberValue(credits.freeCredits) ?? 0 + + return { + monthlyCredits, + purchasedCredits, + freeCredits, + remainingCredits: monthlyCredits + purchasedCredits + freeCredits, + windowLimits: windowLimitsFromCredits(windowLimits), + } +} + +function parseSubscription(value: unknown): CommandCodeSubscription | null { + const data = isRecord(value) ? value.data : undefined + if (!isRecord(data)) return null + + return { + planId: stringValue(data.planId) ?? null, + status: stringValue(data.status) ?? null, + currentPeriodStart: stringValue(data.currentPeriodStart) ?? null, + currentPeriodEnd: stringValue(data.currentPeriodEnd) ?? null, + } +} + +function parseSummary(value: unknown): CommandCodeUsageSummary | null { + if (!isRecord(value)) return null + const totalTokens = numberValue(value.totalTokens) ?? numberValue(value.tokens) + return { + totalCost: numberValue(value.totalCost) ?? 0, + totalCount: numberValue(value.totalCount) ?? 0, + ...(totalTokens === undefined ? {} : { totalTokens }), + } +} + +function parseWhoami(value: unknown): { + login: string + orgId: string | null + keyName?: string +} { + const org = isRecord(value) ? value.org : undefined + const user = isRecord(value) ? value.user : undefined + + const orgLogin = isRecord(org) ? stringValue(org.login) : undefined + const orgId = isRecord(org) ? stringValue(org.id) : undefined + const userLogin = + (isRecord(user) ? stringValue(user.userName) : undefined) ?? + (isRecord(user) ? stringValue(user.name) : undefined) + + const keyName = + stringValue(isRecord(user) ? user.keyName : undefined) ?? + stringValue(isRecord(user) ? user.displayName : undefined) + + return { + login: orgLogin ?? userLogin ?? "Unknown account", + orgId: orgId ?? null, + ...(keyName === undefined ? {} : { keyName }), + } +} + +/** + * Parse the `windowLimits` into a human-readable, header-safe line list that + * the formatting layer appends. Split out so the pure shape is independently + * testable. + */ +export function formatWindowLimits( + limits: readonly CommandCodeWindowLimit[], + now: () => number = Date.now, +): string[] { + const labels: Record = { + fiveHour: "5-hour", + weekly: "Weekly", + } + + return limits.map((limit) => { + const label = labels[limit.window] ?? limit.window + const used = limit.used.toFixed(2) + const cap = limit.cap.toFixed(2) + const pct = limit.cap > 0 ? Math.round((limit.used / limit.cap) * 100) : 0 + const reset = limit.resetAt === null ? "" : ` (resets ${formatResetClock(limit.resetAt, now)})` + return `${label}: ${used} / ${cap} credits (${pct}% used)${reset}` + }) +} + +function formatResetClock(resetAtSeconds: number, now: () => number = Date.now): string { + const date = new Date(resetAtSeconds * 1000) + if (Number.isNaN(date.getTime())) return "unknown" + const nowMs = now() + const diffMs = date.getTime() - nowMs + if (diffMs <= 0) return "soon" + const minutes = Math.ceil(diffMs / 60_000) + if (minutes < 60) return `in ${minutes}m` + const hours = Math.floor(minutes / 60) + const rem = minutes % 60 + if (hours < 24) return rem > 0 ? `in ${hours}h ${rem}m` : `in ${hours}h` + const days = Math.floor(hours / 24) + return days === 1 ? "in 1 day" : `in ${days} days` +} + +/** Derived credits view for the Remaining/Used layout. */ +interface CreditView { + /** Credits remaining (monthly + purchased + free). */ + remaining: number + /** Dollars spent this period (totalCost). */ + spent: number + /** Total pool used as the percentage denominator: remaining + spent. */ + pool: number + /** Percent of the pool used, 0-100. */ + usedPercent: number + hasCreditsInfo: boolean +} + +function creditView(quota: CommandCodeQuota): CreditView { + const credits = quota.credits + const remaining = credits ? credits.remainingCredits : 0 + const spent = quota.summary?.totalCost ?? 0 + const pool = remaining + spent + const hasCreditsInfo = Boolean(credits) || spent > 0 + return { + remaining, + spent, + pool, + usedPercent: hasCreditsInfo ? Math.round((pool > 0 ? spent / pool : 0) * 100) : 0, + hasCreditsInfo, + } +} + +function creditDetailLine(credits: CommandCodeCredits | null): string { + if (!credits) return "" + const parts = [`monthly $${credits.monthlyCredits.toFixed(2)}`] + parts.push(`purchased $${credits.purchasedCredits.toFixed(2)}`) + if (credits.freeCredits > 0) parts.push(`free $${credits.freeCredits.toFixed(2)}`) + return `Sources: ${parts.join(" / ")}` +} + +function subscriptionLine(subscription: CommandCodeSubscription): string { + const rank = subscription.planId ?? "Unknown" + const plan = rank.replace(/[_-]+/g, " ").trim() + const status = subscription.status ? ` (${subscription.status})` : "" + return `Plan: ${plan}${status}` +} + +function accountName(account: CommandCodeQuota["account"]): string { + return account.keyName ?? account.login +} + +/** + * Render a normalized quota snapshot as clean, aligned, dashboard-style text + * suitable for `ui.notify`. Pure so it can be unit tested without a runtime. + */ +export function formatQuota(quota: CommandCodeQuota, now: () => number = Date.now): string { + const lines: string[] = [] + + const credit = creditView(quota) + if (credit.hasCreditsInfo) { + lines.push("") + lines.push("Credits") + lines.push(padValue(`Remaining: $${credit.remaining.toFixed(2)} of $${credit.pool.toFixed(2)}`)) + lines.push(padValue(`Used: $${credit.spent.toFixed(2)}`)) + lines.push(` ${credit.usedPercent}% used`) + } + + const detail = creditDetailLine(quota.credits) + if (detail) lines.push(detail) + + if (quota.subscription) lines.push(subscriptionLine(quota.subscription)) + + if (quota.summary) { + lines.push("") + lines.push("Usage (this month)") + lines.push(padValue(`Cost: $${quota.summary.totalCost.toFixed(2)}`)) + lines.push(padValue(`Requests: ${quota.summary.totalCount.toLocaleString("en-US")}`)) + if (quota.summary.totalTokens && quota.summary.totalTokens > 0) { + lines.push(padValue(`Tokens: ${formatTokens(quota.summary.totalTokens)}`)) + } + } + + lines.push("") + lines.push("Username") + lines.push(padValue(accountName(quota.account))) + + const limits = quota.credits?.windowLimits ?? [] + if (limits.length > 0) { + lines.push("") + lines.push("Usage windows:") + lines.push(...formatWindowLimits(limits, now).map((line) => ` ${line}`)) + } + + lines.push("") + lines.push(`Full detail: https://commandcode.ai/usage`) + + // Trim leading/trailing blank lines so sections stay cleanly separated. + while (lines.length > 0 && lines[0].length === 0) lines.shift() + while (lines.length > 0 && lines[lines.length - 1].length === 0) lines.pop() + return lines.join("\n") +} + +function padValue(value: string): string { + return ` ${value}` +} + +function formatTokens(tokens: number): string { + if (tokens >= 1_000_000_000) return `${(tokens / 1_000_000_000).toFixed(1)}B` + if (tokens >= 1_000_000) return `${(tokens / 1_000_000).toFixed(1)}M` + if (tokens >= 1_000) return `${(tokens / 1_000).toFixed(1)}k` + return String(tokens) +} + +/** + * Fetch the current account quota from Command Code. + * + * Resolution chain: whoami -> org id -> credits + subscription (parallel) -> + * summary (needs the billing period start). Any individual endpoint failing + * degrades gracefully: the remaining data is still reported, and a hard + * failure (auth/config, network) is surfaced as a typed error. + */ +export async function fetchCommandCodeQuota( + options: FetchOptions, +): Promise { + if (!options.apiKey) { + return { + ok: false, + error: { message: "No Command Code API key found", kind: "config" }, + } + } + + const baseUrl = options.baseUrl ?? DEFAULT_API_BASE + const fetchImpl = options.fetchImpl ?? fetch + const timeoutMs = options.timeoutMs ?? QUOTA_TIMEOUT_MS + + const headers = { + "Content-Type": "application/json", + Authorization: `Bearer ${options.apiKey}`, + ...options.extraHeaders, + } + + // One overall deadline shared across the sequential phases (whoami -> billing + // -> summary) so a slow or blackholed dependency cannot compound per-request + // timeouts into a ~45s stall; the command reports within QUOTA_TIMEOUT_MS. + const overallController = new AbortController() + const overallTimer = setTimeout(() => overallController.abort(), timeoutMs) + + const request = async (path: string): Promise => { + // The overall deadline may already have fired (e.g. a prior phase consumed + // the budget) — AbortSignal does not replay past abort events to listeners + // added afterward, so check synchronously instead of relying on the listener. + if (overallController.signal.aborted) { + throw new QuotaTimeoutError() + } + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), timeoutMs) + const onOverallAbort = () => controller.abort() + overallController.signal.addEventListener("abort", onOverallAbort) + try { + const response = await fetchImpl(`${baseUrl}${path}`, { + method: "GET", + headers, + signal: controller.signal, + }) + + if (!response.ok) { + let message = response.statusText + if (response.status === 401 || response.status === 403) { + message = "Command Code rejected the API key (401/403)" + } + return { + __httpError: true, + message, + status: response.status, + body: await response.text().catch(() => ""), + } + } + return await response.json() + } catch (error) { + if (controller.signal.aborted) { + throw new QuotaTimeoutError() + } + throw error + } finally { + clearTimeout(timer) + overallController.signal.removeEventListener("abort", onOverallAbort) + } + } + + /** Optional-endpoint wrapper: never throws. Transport/timeout/parse failures + * become a sentinel so the rest of the dashboard still renders, matching the + * existing graceful degradation for HTTP 5xx responses. */ + const safeRequest = async (path: string): Promise => { + try { + return await request(path) + } catch (error) { + return { + __quotaError: true, + message: errorMessage(error), + kind: error instanceof QuotaTimeoutError ? "timeout" : "network", + } + } + } + + try { + const whoami = await request("/alpha/whoami") + if (isHttpError(whoami)) return httpFailure(whoami, "whoami") + const account = parseWhoami(whoami) + + const orgId = account.orgId ?? undefined + const creditsPath = buildUrl("/alpha/billing/credits", { orgId }) + const subPath = buildUrl("/alpha/billing/subscriptions", { orgId }) + + const [creditsRaw, subRaw] = await Promise.all([safeRequest(creditsPath), safeRequest(subPath)]) + + // Hard auth/permission failures abort; everything else (including thrown + // network/timeout/parse failures) degrades to a null section. + if (isHttpError(creditsRaw) && isBlockingQuotaHttpError(creditsRaw)) { + return httpFailure(creditsRaw, "credits") + } + if (isHttpError(subRaw) && isBlockingQuotaHttpError(subRaw)) { + return httpFailure(subRaw, "subscription") + } + + const credits = + creditsRaw && !isHttpError(creditsRaw) && !isQuotaError(creditsRaw) + ? parseCredits(creditsRaw) + : null + const subscription = + subRaw && !isHttpError(subRaw) && !isQuotaError(subRaw) ? parseSubscription(subRaw) : null + + const since = subscription?.currentPeriodStart ?? undefined + const summaryPath = buildUrl("/alpha/usage/summary", { orgId, since }) + const summaryRaw = await safeRequest(summaryPath) + if (isHttpError(summaryRaw) && isBlockingQuotaHttpError(summaryRaw)) { + return httpFailure(summaryRaw, "summary") + } + const summary = + summaryRaw && !isHttpError(summaryRaw) && !isQuotaError(summaryRaw) + ? parseSummary(summaryRaw) + : null + + if (credits === null && subscription === null && summary === null) { + if (overallController.signal.aborted) { + return { + ok: false, + error: { message: "Command Code quota request timed out", kind: "timeout" }, + } + } + return { + ok: false, + error: { + message: "Command Code returned no usage data for the account", + kind: "http", + }, + } + } + + return { + ok: true, + quota: { account, credits, subscription, summary }, + } + } catch (error) { + if (error instanceof QuotaTimeoutError || overallController.signal.aborted) { + return { + ok: false, + error: { message: "Command Code quota request timed out", kind: "timeout" }, + } + } + return { + ok: false, + error: { + message: redactValue(`Failed to fetch Command Code quota: ${errorMessage(error)}`), + kind: "network", + }, + } + } finally { + clearTimeout(overallTimer) + } +} + +class QuotaTimeoutError extends Error { + constructor() { + super("Command Code quota request timed out") + this.name = "QuotaTimeoutError" + } +} + +interface HttpErrorShape { + __httpError: true + message: string + status: number + body: string +} + +function isHttpError(value: unknown): value is HttpErrorShape { + if (!isRecord(value) || value.__httpError !== true) return false + return ( + typeof value.status === "number" && + typeof value.message === "string" && + typeof value.body === "string" + ) +} + +/** Sentinel produced by safeRequest for thrown transport/timeout/parse failures. */ +interface QuotaErrorShape { + __quotaError: true + message: string + kind: "timeout" | "network" +} + +function isQuotaError(value: unknown): value is QuotaErrorShape { + if (!isRecord(value) || value.__quotaError !== true) return false + return typeof value.message === "string" && (value.kind === "timeout" || value.kind === "network") +} + +/** + * Hard-failure HTTP statuses for the quota dashboard: authentication and + * permission failures. Rate limiting (429) is deliberately NOT included — it + * is a transient dependency condition that should degrade like other non-auth + * endpoint failures, not abort the whole command. + */ +function isBlockingQuotaHttpError(error: HttpErrorShape): boolean { + return error.status === 401 || error.status === 403 +} + +function httpFailure(error: HttpErrorShape, context: string): CommandCodeQuotaResult { + const detail = error.body.trim().slice(0, 200) + const message = detail + ? `${context} request failed (${error.status}): ${detail}` + : `${context} request failed (${error.status}): ${error.message}` + return { + ok: false, + error: { message: redactValue(message), kind: "http" }, + } +} + +/** Best-effort scrub of values that look like tokens/secrets from a message. */ +export function redactValue(value: string): string { + // Reuse the broader Command Code redaction (Bearer, credential key-value + // fields, user_/cc_ tokens, query-string secrets, standalone keys) so quota + // errors get the same protection as stream errors. Additionally catch + // JSON-quoted credential fields ({"apiKey":"..."}) that the upstream pattern + // requires to be adjacent to `=`/`:`. + return redactCommandCodeErrorText(value) + .replace( + /("\s*(?:api[-_ ]?key|apikey|access[-_ ]?token|refresh[-_ ]?token|token|secret|password|authorization)\s*"\s*:\s*")([^"]{8,})/gi, + "$1[redacted]", + ) + .trim() +} diff --git a/tests/test-omp-compat.mjs b/tests/test-omp-compat.mjs index 9daae39..eeb5793 100644 --- a/tests/test-omp-compat.mjs +++ b/tests/test-omp-compat.mjs @@ -165,7 +165,13 @@ function runOmp(args, timeoutMs = 30_000) { try { console.log("[omp-compat] list models through real extension") modelListRequestCount = 0 - const result = await runOmp(["-e", EXT_PATH, "--list-models"]) + // Prefer the flag form `omp -e EXT --list-models`; Homebrew's `omp` + // distribution only exposes the `omp models` subcommand, so fall back to + // that form when the flag invocation is not recognized. + let result = await runOmp(["-e", EXT_PATH, "--list-models"]) + if (result.code !== 0) { + result = await runOmp(["models", "-e", EXT_PATH]) + } assert.equal(result.code, 0, result.stderr) const listOutput = result.stdout || result.stderr assert.match(listOutput, /commandcode/) diff --git a/tests/test-pure-functions.ts b/tests/test-pure-functions.ts index 136025f..d71dba0 100644 --- a/tests/test-pure-functions.ts +++ b/tests/test-pure-functions.ts @@ -16,6 +16,7 @@ import { mapFinishReason, messagesToCC, parseStreamEventLine, + pickCommandCodeApiKey, projectSlugFromPath, textContent, toJsonSchema, @@ -106,6 +107,36 @@ describe("error redaction", () => { }) }) +describe("pickCommandCodeApiKey()", () => { + it("falls back to the host key for a placeholder registry value", () => { + assert.equal(pickCommandCodeApiKey("$COMMANDCODE_API_KEY", "file-key"), "file-key") + assert.equal(pickCommandCodeApiKey("COMMANDCODE_API_KEY", "file-key"), "file-key") + }) + + it("returns undefined when only a placeholder is provided (no fallback)", () => { + assert.equal(pickCommandCodeApiKey("$COMMANDCODE_API_KEY", undefined), undefined) + }) + + it("prefers a real registry key over the host fallback", () => { + assert.equal(pickCommandCodeApiKey("real-registry-key", "file-key"), "real-registry-key") + }) + + it("falls back to the host key when the registry has none", () => { + assert.equal(pickCommandCodeApiKey(undefined, "file-key"), "file-key") + assert.equal(pickCommandCodeApiKey(undefined, undefined), undefined) + }) + + it("falls back to the host key for empty or whitespace registry values", () => { + assert.equal(pickCommandCodeApiKey("", "file-key"), "file-key") + assert.equal(pickCommandCodeApiKey(" ", "file-key"), "file-key") + assert.equal(pickCommandCodeApiKey(" ", undefined), undefined) + }) + + it("trims a real registry key", () => { + assert.equal(pickCommandCodeApiKey(" real-registry-key ", "file-key"), "real-registry-key") + }) +}) + describe("projectSlugFromPath()", () => { it("matches the official CLI-style slug from an absolute working directory", () => { assert.equal( @@ -359,7 +390,7 @@ describe("toJsonSchema()", () => { if (!outputProperties || typeof outputProperties !== "object") { throw new Error("expected object properties") } - assert.ok(Object.prototype.hasOwnProperty.call(outputProperties, "__proto__")) + assert.ok(Object.hasOwn(outputProperties, "__proto__")) assert.deepEqual(Object.getOwnPropertyDescriptor(outputProperties, "__proto__")?.value, { type: "string", }) diff --git a/tests/test-quota.ts b/tests/test-quota.ts new file mode 100644 index 0000000..5eea0d8 --- /dev/null +++ b/tests/test-quota.ts @@ -0,0 +1,395 @@ +/** + * Unit tests for the Command Code quota layer (src/quota.ts). + * + * These are hermetic: no pi runtime and no network. Fetching is exercised with + * a mocked `fetchImpl`, while parsing and formatting are pure function checks. + */ + +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { + DEFAULT_API_BASE, + fetchCommandCodeQuota, + formatQuota, + formatWindowLimits, + redactValue, + windowLimitsFromCredits, +} from "../src/quota.ts" +import type { CommandCodeQuota, CommandCodeCredits, CommandCodeWindowLimit } from "../src/quota.ts" + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }) +} + +function okFetch(handlers: Record) { + const urls: string[] = [] + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + urls.push(url) + for (const [needle, body] of Object.entries(handlers)) { + if (url.includes(needle)) return jsonResponse(body) + } + throw new Error(`Unexpected URL: ${url}`) + } + return { fetchImpl, urls: () => urls } +} + +describe("Command Code quota", () => { + it("parses window limits from the credits windowLimits object", () => { + const limits = windowLimitsFromCredits({ + limited: true, + // resetAt as reported by the live API: milliseconds since epoch. + fiveHour: { used: 8, cap: 14, resetAt: 1_700_000_000_000 }, + weekly: { used: 30, cap: 35, resetAt: 1_700_000_000_000 }, + }) + assert.deepEqual(limits, [ + { window: "fiveHour", used: 8, cap: 14, resetAt: 1_700_000_000 }, + { window: "weekly", used: 30, cap: 35, resetAt: 1_700_000_000 }, + ]) + }) + + it("skips empty window limit entries", () => { + const limits = windowLimitsFromCredits({ + limited: false, + fiveHour: { used: 0, cap: 0, resetAt: null }, + weekly: { used: 0, cap: 0, resetAt: null }, + }) + assert.deepEqual(limits, []) + }) + + it("parses resetAt as numeric string or ISO timestamp string", () => { + const limits = windowLimitsFromCredits({ + fiveHour: { used: 1, cap: 2, resetAt: "1700000000000" }, + weekly: { used: 1, cap: 2, resetAt: "2023-11-14T22:13:20.000Z" }, + }) + // numeric ms string -> epoch seconds; ISO string -> epoch seconds + assert.equal(limits[0]?.resetAt, 1_700_000_000) + assert.equal(limits[1]?.resetAt, 1_700_000_000) + }) + + it("renders a zero request count instead of dropping the Requests line", () => { + const quota: CommandCodeQuota = { + account: { login: "alice", orgId: null }, + credits: null, + subscription: null, + summary: { totalCost: 0, totalCount: 0 }, + } + const output = formatQuota(quota, () => 1_700_000_000_000) + assert.match(output, /Requests: 0/) + }) + + it("formats window limits with percentage and reset clock", () => { + const limits: CommandCodeWindowLimit[] = [ + { window: "fiveHour", used: 7, cap: 14, resetAt: 1_700_000_000 }, + { window: "weekly", used: 0, cap: 35, resetAt: null }, + ] + const lines = formatWindowLimits(limits) + assert.match(lines[0] ?? "", /^5-hour: 7\.00 \/ 14\.00 credits \(50% used\) \(resets/) + assert.match(lines[1] ?? "", /^Weekly: 0\.00 \/ 35\.00 credits \(0% used\)/) + }) + + it("uses the injected clock for the reset countdown", () => { + const limit: CommandCodeWindowLimit = { + window: "fiveHour", + used: 7, + cap: 14, + resetAt: 1_700_000_000, // seconds since epoch + } + // now() shortly before reset -> a short "in Nm" countdown + const soon = formatWindowLimits([limit], () => 1_699_999_000 * 1000)[0] + assert.match(soon ?? "", /\(resets in \d+m\)/) + // already past reset -> "soon" + const past = formatWindowLimits([limit], () => 1_700_100_000 * 1000)[0] + assert.match(past ?? "", /\(resets soon\)/) + }) + + it("fetches and normalizes the full quota snapshot", async () => { + const { fetchImpl, urls } = okFetch({ + whoami: { user: { userName: "alice" }, org: { id: "org_1", login: "alice-inc" } }, + credits: { + credits: { + monthlyCredits: 40, + purchasedCredits: 10, + freeCredits: 5, + planId: "pro", + }, + windowLimits: { + fiveHour: { used: 8, cap: 16, resetAt: 1_700_000_000_000 }, + weekly: { used: 20, cap: 40, resetAt: null }, + }, + }, + subscriptions: { + data: { + planId: "pro", + status: "active", + currentPeriodStart: "2026-01-01T00:00:00Z", + currentPeriodEnd: "2026-02-01T00:00:00Z", + }, + }, + summary: { totalCost: 12.34, totalCount: 1500 }, + }) + + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, true) + if (!result.ok) return + + assert.equal(result.quota.account.login, "alice-inc") + assert.equal(result.quota.account.orgId, "org_1") + assert.deepEqual(result.quota.credits?.remainingCredits, 55) + assert.equal(result.quota.credits?.windowLimits.length, 2) + assert.equal(result.quota.subscription?.planId, "pro") + assert.equal(result.quota.summary?.totalCost, 12.34) + + // Regression: requested URLs must carry the base exactly once (no + // double prefix), and all hit the alpha usage endpoints. + const fetched = urls() + assert.equal(fetched.length, 4) + for (const url of fetched) { + assert.ok( + /^https:\/\/api\.commandcode\.ai\/alpha\//.test(url), + `expected base-prefixed alpha URL, got: ${url}`, + ) + assert.equal((url.match(/https:\/\//g) ?? []).length, 1) + assert.equal(url.includes(`${DEFAULT_API_BASE}${DEFAULT_API_BASE}`), false) + } + }) + + it("degrades gracefully when individual billing endpoints fail", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + if (url.includes("summary")) return jsonResponse({ totalCost: 3.0, totalCount: 10 }) + if (url.includes("credits") || url.includes("subscriptions")) { + return jsonResponse({ error: "boom" }, 500) + } + throw new Error(`Unexpected URL: ${url}`) + } + + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, true) + if (!result.ok) return + assert.equal(result.quota.credits, null) + assert.equal(result.quota.summary?.totalCost, 3.0) + // Optional aggregate tokens are parsed when the summary reports them. + assert.equal(result.quota.summary?.totalTokens, undefined) + }) + + it("degrades on thrown network failures from optional endpoints, not just HTTP 5xx", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + if (url.includes("summary")) return jsonResponse({ totalCost: 3.0, totalCount: 10 }) + if (url.includes("credits")) throw new Error("network down") + if (url.includes("subscriptions")) return jsonResponse({ data: { planId: "pro" } }) + throw new Error(`Unexpected URL: ${url}`) + } + + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, true) + if (!result.ok) return + assert.equal(result.quota.credits, null) + assert.equal(result.quota.subscription?.planId, "pro") + assert.equal(result.quota.summary?.totalCost, 3.0) + }) + + it("fails the command when the summary endpoint rejects auth/permission", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + if (url.includes("credits")) return jsonResponse({ credits: { monthlyCredits: 5 } }) + if (url.includes("subscriptions")) return jsonResponse({ data: { planId: "pro" } }) + if (url.includes("summary")) return jsonResponse({ error: "nope" }, 403) + throw new Error(`Unexpected URL: ${url}`) + } + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.error.kind, "http") + assert.match(result.error.message, /summary/) + }) + + it("does not treat 429 on billing endpoints as fatal", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + if (url.includes("summary")) return jsonResponse({ totalCost: 3.0, totalCount: 10 }) + if (url.includes("credits") || url.includes("subscriptions")) { + return jsonResponse({ error: "rate limited" }, 429) + } + throw new Error(`Unexpected URL: ${url}`) + } + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, true) + if (!result.ok) return + assert.equal(result.quota.credits, null) + assert.equal(result.quota.summary?.totalCost, 3.0) + }) + + it("sends extra headers (ZDR) on quota requests", async () => { + let sent: Headers | undefined + const fetchImpl = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + sent = (init?.headers as Headers) ?? undefined + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + if (url.includes("credits")) return jsonResponse({ credits: { monthlyCredits: 5 } }) + if (url.includes("subscriptions")) return jsonResponse({ data: { planId: "pro" } }) + if (url.includes("summary")) return jsonResponse({ totalCost: 1, totalCount: 1 }) + throw new Error(`Unexpected URL: ${url}`) + } + const result = await fetchCommandCodeQuota({ + apiKey: "cc_test_key", + fetchImpl, + extraHeaders: { "x-cmd-zdr": "1" }, + }) + assert.equal(result.ok, true) + const headers = new Headers(sent) + assert.equal(headers.get("x-cmd-zdr"), "1") + }) + + it("parses optional token count and key name when present", async () => { + const { fetchImpl } = okFetch({ + whoami: { user: { userName: "alice", keyName: "Pi Agent" }, org: null }, + credits: { credits: { monthlyCredits: 5, purchasedCredits: 0, freeCredits: 0 } }, + subscriptions: { data: { planId: "pro", status: "active" } }, + summary: { totalCost: 1.06, totalCount: 654, totalTokens: 74_200_000 }, + }) + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, true) + if (!result.ok) return + assert.equal(result.quota.summary?.totalTokens, 74_200_000) + assert.equal(result.quota.account.keyName, "Pi Agent") + }) + + it("rejects missing API keys as a config error", async () => { + const result = await fetchCommandCodeQuota({ apiKey: "" }) + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.error.kind, "config") + }) + + it("fails with a config-style error when the API key is rejected", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + if (String(input).includes("whoami")) return jsonResponse({ error: "unauthorized" }, 401) + throw new Error(`Unexpected URL: ${input}`) + } + const result = await fetchCommandCodeQuota({ apiKey: "cc_bad_key", fetchImpl }) + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.error.kind, "http") + assert.match(result.error.message, /401/) + }) + + it("formats a complete quota snapshot into readable output", () => { + const quota: CommandCodeQuota = { + account: { login: "alice-inc", orgId: "org_1" }, + credits: { + monthlyCredits: 40, + purchasedCredits: 10, + freeCredits: 5, + remainingCredits: 55, + windowLimits: [ + { window: "fiveHour", used: 8, cap: 16, resetAt: null }, + { window: "weekly", used: 20, cap: 40, resetAt: null }, + ], + } satisfies CommandCodeCredits, + subscription: { + planId: "pro", + status: "active", + currentPeriodStart: "", + currentPeriodEnd: "", + }, + summary: { totalCost: 12.34, totalCount: 1500 }, + } + + const output = formatQuota(quota, () => 1_700_000_000_000) + assert.doesNotMatch(output, /Command Code quota —/) + assert.match(output, /Credits/) + assert.match(output, /Remaining: \$55\.00 of \$67\.34/) + assert.match(output, /Used: \$12\.34/) + assert.match(output, /Sources: monthly \$40\.00 \/ purchased \$10\.00 \/ free \$5\.00/) + assert.match(output, /Plan: pro \(active\)/) + assert.match(output, /Usage \(this month\)/) + assert.match(output, /Cost: \$12\.34/) + assert.match(output, /Requests: 1,500/) + assert.match(output, /Username/) + assert.match(output, /alice-inc/) + assert.match(output, /5-hour: 8\.00 \/ 16\.00 credits/) + assert.match(output, /Weekly: 20\.00 \/ 40\.00 credits/) + assert.match(output, /https:\/\/commandcode\.ai\/usage/) + }) + + it("redacts token-like values from error messages", () => { + // 16+ char run after a credential key is redacted by the shared redactor. + assert.equal(redactValue("api_key=abcdefghijklmnop123456"), "api_key=[redacted]") + assert.equal(redactValue("Bearer user_12345678901234 failed"), "Bearer [redacted] failed") + }) + + it("redacts named credential fields and short tokens from error bodies", () => { + // Credential key-value forms (with = or : separator) are redacted. + assert.equal(redactValue("api_key=abc123"), "api_key=[redacted]") + assert.equal( + redactValue("authorization=Basic abc:def failed"), + "authorization=[redacted] abc:def failed", + ) + assert.equal(redactValue("user_123456789 failed"), "[redacted] failed") + assert.equal(redactValue("cc_abcdefghijkl failed"), "[redacted] failed") + assert.equal( + redactValue("token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.secret"), + "token=[redacted]", + ) + }) + + it("redacts JSON-quoted credential fields in error bodies", () => { + assert.equal( + redactValue('{"apiKey":"sk-abcdefghijklmnop123456","ok":true}'), + '{"apiKey":"[redacted]","ok":true}', + ) + assert.equal( + redactValue('{"error":"bad","access_token":"opaque-internal-token-12345"}'), + '{"error":"bad","access_token":"[redacted]"}', + ) + assert.equal( + redactValue('{"authorization":"Bearer user_1234"}'), + '{"authorization":"[redacted]"}', + ) + }) + + it("redacts thrown network errors from the outer catch path", async () => { + const fetchImpl = async (_input: RequestInfo | URL): Promise => { + throw new Error("connection reset by proxy api_key=supersecretvalue123456") + } + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, false) + if (result.ok) return + assert.doesNotMatch(result.error.message, /supersecretvalue123456/) + assert.match(result.error.kind, /network/) + }) + + it("honors the overall deadline once it has already fired (no phase starts after abort)", async () => { + const start = Date.now() + const fetchImpl = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = String(input) + if (url.includes("whoami")) { + // Never resolve; let the per-request controller abort it at timeoutMs. + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => + reject(Object.assign(new Error("aborted"), { name: "AbortError" })), + ) + }) + } + throw new Error(`Unexpected URL: ${url}`) + } + + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl, timeoutMs: 30 }) + const elapsed = Date.now() - start + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.error.kind, "timeout") + // The overall deadline governs the whole command; no phase may add ~30ms on top. + assert.ok(elapsed < 200, `elapsed ${elapsed}ms exceeded overall deadline`) + }) +}) From 45565a883fa4fc26b5dec85f555e62aca635f10d Mon Sep 17 00:00:00 2001 From: Thomas Byr Date: Sat, 22 Aug 2026 23:24:49 +0200 Subject: [PATCH 17/40] feat(models): add GLM-5.3 pricing and reasoning levels --- CHANGELOG.md | 1 + src/core.ts | 2 +- src/models.ts | 5 +++-- src/pricing.ts | 3 ++- tests/fixtures/commandcode-model-ids.json | 3 ++- tests/fixtures/commandcode-pricing.json | 3 ++- 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee3cd9b..c6d7f95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Add `zai-org/GLM-5.3` with its verified reasoning efforts and display pricing. - Prefer Command Code's Provider API (`/provider/v1/chat/completions` and `/provider/v1/messages`) and automatically fall back to the existing `/alpha/generate` transport only when the Provider API returns `403 upgrade_required` for a Go-plan account. - Remember the detected transport for the running process, re-detect it when credentials change, prevent stale in-flight requests from overwriting the new credential's transport, and never fall back for unrelated authentication, permission, rate-limit, network, or server failures. - Use Pi's native OpenAI- and Anthropic-compatible providers for Provider API streaming, including adaptive thinking for current reasoning-capable Claude models, while preserving the existing hardened generate transport, dynamic model discovery, offline cache, refresh/status commands, pricing, and OAuth credentials. diff --git a/src/core.ts b/src/core.ts index f1898dd..2eb92f8 100644 --- a/src/core.ts +++ b/src/core.ts @@ -43,7 +43,7 @@ export * from "./overflow.ts" export * from "./types.ts" export const DEFAULT_API_BASE = "https://api.commandcode.ai" -export const COMMAND_CODE_CLI_VERSION = "1.15.1" +export const COMMAND_CODE_CLI_VERSION = "1.32.1" const DEFAULT_GENERATE_MAX_TOKENS = 64_000 const DEFAULT_MAX_RETRIES = 0 diff --git a/src/models.ts b/src/models.ts index 3f8c7e9..e102c00 100644 --- a/src/models.ts +++ b/src/models.ts @@ -12,7 +12,7 @@ export type CommandCodeApi = "openai-completions" | "anthropic-messages" export type CommandCodeInputType = "text" | "image" /** - * Model input modalities from the command-code@1.15.1 bundled catalog. + * Model input modalities from the command-code@1.32.1 bundled catalog. * Models omitted here remain text-only so newly discovered IDs never claim * image support without upstream evidence. */ @@ -74,7 +74,7 @@ type CommandCodeReasoningEffort = Exclude * Per-model reasoning efforts supported by Command Code's generate endpoint. * * The Provider API does not expose reasoning metadata. This is an exact - * snapshot of `reasoningEfforts` from the command-code@1.15.1 model catalog + * snapshot of `reasoningEfforts` from the command-code@1.32.1 model catalog * (`packages/shared/src/model-catalog.ts`, also published in the generated * `dist/bundled/command-code-knowledge/reference/models.md`). Models omitted * here let Command Code choose their reasoning depth, matching the CLI. @@ -103,6 +103,7 @@ export const MODEL_EFFORTS: Readonly> = { }, "moonshotai/Kimi-K2.6": { input: 0.95, output: 4, cacheRead: 0.16, cacheWrite: 0 }, "moonshotai/Kimi-K2.5": { input: 0.6, output: 3, cacheRead: 0.1, cacheWrite: 0 }, + "zai-org/GLM-5.3": { input: 1.4, output: 4.4, cacheRead: 0.26, cacheWrite: 0 }, "zai-org/GLM-5.2": { input: 1.4, output: 4.4, cacheRead: 0.26, cacheWrite: 0 }, "zai-org/GLM-5.2-Fast": { input: 3, output: 10.25, cacheRead: 0.5, cacheWrite: 0 }, "zai-org/GLM-5.1": { input: 1.4, output: 4.4, cacheRead: 0.26, cacheWrite: 0 }, diff --git a/tests/fixtures/commandcode-model-ids.json b/tests/fixtures/commandcode-model-ids.json index f0974cd..1b9898a 100644 --- a/tests/fixtures/commandcode-model-ids.json +++ b/tests/fixtures/commandcode-model-ids.json @@ -1,5 +1,5 @@ { - "fetchedAt": "2026-08-04T10:12:57.953Z", + "fetchedAt": "2026-08-22T21:19:37.782Z", "source": "https://api.commandcode.ai/provider/v1/models", "modelIds": [ "claude-sonnet-5", @@ -23,6 +23,7 @@ "moonshotai/Kimi-K2.7-Code-Highspeed", "moonshotai/Kimi-K2.6", "moonshotai/Kimi-K2.5", + "zai-org/GLM-5.3", "zai-org/GLM-5.2", "zai-org/GLM-5.2-Fast", "zai-org/GLM-5.1", diff --git a/tests/fixtures/commandcode-pricing.json b/tests/fixtures/commandcode-pricing.json index 6a508fd..5b2070c 100644 --- a/tests/fixtures/commandcode-pricing.json +++ b/tests/fixtures/commandcode-pricing.json @@ -1,5 +1,5 @@ { - "verifiedAt": "2026-08-20", + "verifiedAt": "2026-08-22", "source": "https://commandcode.ai/docs/resources/pricing-limits", "tierPolicy": "Use request-wide input tiers; the highest threshold exceeded by input plus cache tokens applies to the full request.", "tiers": { @@ -18,6 +18,7 @@ "moonshotai/Kimi-K2.7-Code-Highspeed": [1.9, 8, 0.38, 0], "moonshotai/Kimi-K2.6": [0.95, 4, 0.16, 0], "moonshotai/Kimi-K2.5": [0.6, 3, 0.1, 0], + "zai-org/GLM-5.3": [1.4, 4.4, 0.26, 0], "zai-org/GLM-5.2": [1.4, 4.4, 0.26, 0], "zai-org/GLM-5.2-Fast": [3, 10.25, 0.5, 0], "zai-org/GLM-5.1": [1.4, 4.4, 0.26, 0], From d637a73e06384c5bb9e93aeccb0812b0048e37c5 Mon Sep 17 00:00:00 2001 From: Thomas Byr Date: Sat, 22 Aug 2026 23:25:06 +0200 Subject: [PATCH 18/40] fix(docs): update Command Code version to 1.32.1 --- README.md | 2 +- tests/test-models.ts | 5 +++-- tests/test-pricing.ts | 4 ++-- tests/test-stream.ts | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 4b32189..a4de126 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ The following environment variables are intended for tests, local mocks, and com ## Image input -The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.15.1`; unknown models default to text-only until their upstream metadata is reviewed. +The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.1`; unknown models default to text-only until their upstream metadata is reviewed. For vision-capable models, Pi's native provider adapters forward image blocks from user messages and tool results using the documented OpenAI or Anthropic message schema. Unknown and text-only models remain marked text-only in Pi. diff --git a/tests/test-models.ts b/tests/test-models.ts index 3c7e33c..f10dd7e 100644 --- a/tests/test-models.ts +++ b/tests/test-models.ts @@ -100,7 +100,7 @@ describe("commandCodeModelsFromApiResponse()", () => { ) }) - it("matches command-code@1.15.1 image input capabilities", () => { + it("matches command-code@1.32.1 image input capabilities", () => { assert.deepEqual(inputModalitiesForModel("gpt-5.6-luna"), ["text", "image"]) assert.deepEqual(inputModalitiesForModel("meta/muse-spark-1.2"), ["text", "image"]) assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-pro"), ["text"]) @@ -123,7 +123,7 @@ describe("commandCodeModelsFromApiResponse()", () => { assert.equal(models[1]?.reasoning, false) }) - it("matches the exact command-code@1.15.1 reasoning effort catalog", () => { + it("matches the exact command-code@1.32.1 reasoning effort catalog", () => { assert.deepEqual(MODEL_EFFORTS, { "Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"], "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], @@ -147,6 +147,7 @@ describe("commandCodeModelsFromApiResponse()", () => { "google/gemini-3.6-flash": ["low", "medium", "high"], "sakana/fugu-ultra": ["high", "xhigh"], "xai/grok-4.5": ["low", "medium", "high"], + "zai-org/GLM-5.3": ["low", "high", "max"], "zai-org/GLM-5.2": ["high", "max"], }) }) diff --git a/tests/test-pricing.ts b/tests/test-pricing.ts index 6e17400..202d8aa 100644 --- a/tests/test-pricing.ts +++ b/tests/test-pricing.ts @@ -50,7 +50,7 @@ function assertCost( describe("MODEL_COSTS pricing overlay", () => { it("covers the current Command Code model catalog snapshot", () => { assert.equal(fixture.source, "https://api.commandcode.ai/provider/v1/models") - assert.match(fixture.fetchedAt, /^2026-08-04T/) + assert.match(fixture.fetchedAt, /^2026-08-22T/) const catalogIds = [...fixture.modelIds].sort() const pricedIds = Object.keys(MODEL_COSTS).sort() @@ -169,7 +169,7 @@ describe("MODEL_COSTS pricing overlay", () => { it("tracks pricing provenance", () => { assert.equal(PRICING_SOURCE_URL, "https://commandcode.ai/docs/resources/pricing-limits") - assert.equal(PRICING_LAST_VERIFIED, "2026-08-20") + assert.equal(PRICING_LAST_VERIFIED, "2026-08-22") }) it("fails once temporary pricing needs review", () => { diff --git a/tests/test-stream.ts b/tests/test-stream.ts index e30ac49..a429798 100644 --- a/tests/test-stream.ts +++ b/tests/test-stream.ts @@ -488,7 +488,7 @@ describe("streamCommandCode — request serialization", () => { const headers = server.lastRequestHeaders() assert.equal(headers.authorization, "Bearer mock-key") - assert.equal(headers["x-command-code-version"], "1.15.1") + assert.equal(headers["x-command-code-version"], "1.32.1") assert.equal(headers["x-project-slug"], "repo") assert.equal(headers["x-taste-learning"], "true") assert.equal(headers["x-co-flag"], "false") From 96356698fa046f0e14cd9bdf622454aa8ff4750a Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 13:34:54 +0200 Subject: [PATCH 19/40] fix(quota): harden dashboard integration --- README.md | 4 +- index.ts | 51 +-- package.json | 6 +- src/quota-command.ts | 66 ++++ src/quota-format.ts | 111 +++++++ src/quota-types.ts | 47 +++ src/quota.ts | 630 ++++++++++-------------------------- tests/test-quota-command.ts | 117 +++++++ tests/test-quota.ts | 38 ++- 9 files changed, 556 insertions(+), 514 deletions(-) create mode 100644 src/quota-command.ts create mode 100644 src/quota-format.ts create mode 100644 src/quota-types.ts create mode 100644 tests/test-quota-command.ts diff --git a/README.md b/README.md index 2de94fb..6c757dc 100644 --- a/README.md +++ b/README.md @@ -129,9 +129,9 @@ While pi is running, use these provider commands without restarting: - `/commandcode-refresh` fetches and re-registers the current model catalog. Overlapping refreshes are coalesced, and a failed refresh keeps the last valid catalog active. - `/commandcode-status` shows redacted discovery diagnostics, including the source, model count, timestamps, cache path, endpoint, and warning. -- `/commandcode-quota` shows your Command Code account usage and quota in a dashboard-style layout: credits remaining and used with a percentage, monthly/purchased/free sources, the current plan, month-to-date cost/requests/tokens, the API key name, and the 5-hour and weekly usage windows. +- `/commandcode-quota` shows your Command Code account usage and quota in a dashboard-style layout: credits remaining and used with a percentage, monthly/purchased/free sources, the current plan, available usage totals, the API key name, and the 5-hour and weekly usage windows. -The `commandcode-quota` command reads from the Command Code alpha usage endpoints (the same ones the `cmd` CLI `/usage` command uses): `whoami`, `billing/credits`, `billing/subscriptions`, and `usage/summary`. It authenticates with the same API key the provider already uses. If command cannot reach those endpoints or they change, the command reports a readable error instead of failing. Output is plain text (via `ui.notify`) so it works across pi and compatible hosts such as OMP. +The `commandcode-quota` command reads from the Command Code alpha usage endpoints (the same ones the `cmd` CLI `/usage` command uses): `whoami`, `billing/credits`, `billing/subscriptions`, and `usage/summary`. It authenticates with the same API key the provider already uses. If the command cannot reach those endpoints or an endpoint schema changes, unavailable sections are reported explicitly instead of being displayed as zero usage. Output is plain text (via `ui.notify`) so it works across pi and compatible hosts such as OMP. Set `COMMANDCODE_ZDR=1` to send Command Code's documented `x-cmd-zdr: 1` zero-data-retention header. diff --git a/index.ts b/index.ts index 1a7bd57..d3005d4 100644 --- a/index.ts +++ b/index.ts @@ -18,7 +18,6 @@ import { join } from "node:path" import { getConfiguredApiKey } from "./src/api-key.ts" import { createStreamCommandCode } from "./src/core.ts" import { calculateCommandCodeCost } from "./src/cost.ts" -import { pickCommandCodeApiKey } from "./src/converters.ts" import { apiForModelId, baseUrlForModel, @@ -33,20 +32,10 @@ import { import { getApiKey as getOAuthApiKey, login, refreshToken } from "./src/oauth.ts" import { normalizeCommandCodeMessage } from "./src/overflow.ts" import { MODEL_COSTS, ZERO_MODEL_COST } from "./src/pricing.ts" +import { registerCommandCodeQuota } from "./src/quota-command.ts" import { createCommandCodeRuntime } from "./src/runtime.ts" -import { fetchCommandCodeQuota, formatQuota, redactValue } from "./src/quota.ts" import { createCommandCodeTransportRouter } from "./src/transport.ts" -const COMMAND_CODE_PROVIDER_ID = "commandcode" - -async function resolveCommandCodeApiKey(ctx: ExtensionCommandContext): Promise { - const registryKey = await ctx.modelRegistry?.getApiKeyForProvider?.(COMMAND_CODE_PROVIDER_ID) - // Mirror src/core.ts: OMP may surface an unresolved placeholder; fall back - // to the env/auth-file resolver so we never send a literal placeholder as a - // Bearer token (which caused a 401 on /alpha/whoami). - return pickCommandCodeApiKey(registryKey, getConfiguredApiKey()) -} - function commandCodeHeaders(): Record | undefined { if (process.env.COMMANDCODE_ZDR === "1") { return { "x-cmd-zdr": "1" } @@ -136,41 +125,9 @@ export default async function (pi: ExtensionAPI) { return normalized ? { message: normalized.message } : undefined }) - pi.registerCommand("commandcode-quota", { - description: "Show Command Code account usage and quota", - handler: async (_args, ctx) => { - await ctx.waitForIdle?.() - - // Resolve the key in a host-agnostic way so the command also works on - // OMP (which passes an unresolved "$COMMANDCODE_API_KEY" placeholder - // through the registry): filter placeholders and fall back to the - // env/auth-file resolver, mirroring src/core.ts. - const apiKey = await resolveCommandCodeApiKey(ctx) - if (!apiKey) { - ctx.ui.notify( - "Command Code quota requires an API key. Run /login and select Command Code, or set the COMMANDCODE_API_KEY env var.", - "warning", - ) - return - } - - const result = await fetchCommandCodeQuota({ - apiKey, - // Alpha endpoints live under the legacy base (no /provider/v1), - // same as the fallback generate transport. - baseUrl: legacyApiBase(apiBase), - // Respect the user's zero-data-retention preference on usage/account - // calls too, matching the provider stream path. - extraHeaders: commandCodeHeaders(), - }) - - if (!result.ok) { - ctx.ui.notify(redactValue(result.error.message), "error") - return - } - - ctx.ui.notify(formatQuota(result.quota), "info") - }, + registerCommandCodeQuota(pi, { + apiBase: legacyApiBase(apiBase), + headers: commandCodeHeaders(), }) const runtime = createCommandCodeRuntime(pi, { diff --git a/package.json b/package.json index e00b16f..6e07011 100644 --- a/package.json +++ b/package.json @@ -29,14 +29,14 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", "pi:isolated": "node scripts/pi-isolated.mjs", "pi:authenticated": "node scripts/pi-authenticated.mjs", - "test:quota": "tsx tests/test-quota.ts", - "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", + "test:quota": "tsx tests/test-quota.ts && tsx tests/test-quota-command.ts", + "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", "test:api-key": "tsx tests/test-api-key.ts", "test:models": "tsx tests/test-models.ts", "test:runtime": "tsx tests/test-runtime.ts", diff --git a/src/quota-command.ts b/src/quota-command.ts new file mode 100644 index 0000000..47f24d9 --- /dev/null +++ b/src/quota-command.ts @@ -0,0 +1,66 @@ +import { getConfiguredApiKey } from "./api-key.ts" +import { pickCommandCodeApiKey } from "./converters.ts" +import { fetchCommandCodeQuota, redactValue } from "./quota.ts" +import { formatQuota } from "./quota-format.ts" + +export interface QuotaCommandContext { + waitForIdle?: () => Promise + modelRegistry?: { + getApiKeyForProvider?: (provider: string) => Promise + } + ui: { + notify(message: string, type?: "info" | "warning" | "error"): void + } +} + +interface QuotaCommandApi { + registerCommand( + name: string, + options: { + description: string + handler: (args: string, ctx: QuotaCommandContext) => Promise + }, + ): void +} + +interface RegisterQuotaCommandOptions { + apiBase: string + headers?: Record + getConfiguredKey?: () => string | undefined + fetchQuota?: typeof fetchCommandCodeQuota +} + +export function registerCommandCodeQuota( + pi: QuotaCommandApi, + options: RegisterQuotaCommandOptions, +): void { + const getConfiguredKey = options.getConfiguredKey ?? getConfiguredApiKey + const fetchQuota = options.fetchQuota ?? fetchCommandCodeQuota + + pi.registerCommand("commandcode-quota", { + description: "Show Command Code account usage and quota", + handler: async (_args, ctx) => { + await ctx.waitForIdle?.() + const registryKey = await ctx.modelRegistry?.getApiKeyForProvider?.("commandcode") + const apiKey = pickCommandCodeApiKey(registryKey, getConfiguredKey()) + if (!apiKey) { + ctx.ui.notify( + "Command Code quota requires an API key. Run /login and select Command Code, or set COMMANDCODE_API_KEY.", + "warning", + ) + return + } + + const result = await fetchQuota({ + apiKey, + baseUrl: options.apiBase, + extraHeaders: options.headers, + }) + if (!result.ok) { + ctx.ui.notify(redactValue(result.error.message), "error") + return + } + ctx.ui.notify(formatQuota(result.quota), "info") + }, + }) +} diff --git a/src/quota-format.ts b/src/quota-format.ts new file mode 100644 index 0000000..9bed62c --- /dev/null +++ b/src/quota-format.ts @@ -0,0 +1,111 @@ +import type { + CommandCodeCredits, + CommandCodeQuota, + CommandCodeSubscription, + CommandCodeWindowLimit, +} from "./quota-types.ts" + +export function formatWindowLimits( + limits: readonly CommandCodeWindowLimit[], + now: () => number = Date.now, +): string[] { + const labels: Record = { + fiveHour: "5-hour", + weekly: "Weekly", + } + + return limits.map((limit) => { + const used = limit.used.toFixed(2) + const cap = limit.cap.toFixed(2) + const percent = limit.cap > 0 ? Math.round((limit.used / limit.cap) * 100) : 0 + const reset = limit.resetAt === null ? "" : ` (resets ${formatResetClock(limit.resetAt, now)})` + return `${labels[limit.window]}: ${used} / ${cap} credits (${percent}% used)${reset}` + }) +} + +function formatResetClock(resetAtSeconds: number, now: () => number): string { + const date = new Date(resetAtSeconds * 1000) + if (Number.isNaN(date.getTime())) return "unknown" + const diffMs = date.getTime() - now() + if (diffMs <= 0) return "soon" + const minutes = Math.ceil(diffMs / 60_000) + if (minutes < 60) return `in ${minutes}m` + const hours = Math.floor(minutes / 60) + const remainingMinutes = minutes % 60 + if (hours < 24) { + return remainingMinutes > 0 ? `in ${hours}h ${remainingMinutes}m` : `in ${hours}h` + } + const days = Math.floor(hours / 24) + return days === 1 ? "in 1 day" : `in ${days} days` +} + +function creditsDetail(credits: CommandCodeCredits | null): string | undefined { + if (!credits) return undefined + const parts = [ + `monthly $${credits.monthlyCredits.toFixed(2)}`, + `purchased $${credits.purchasedCredits.toFixed(2)}`, + ] + if (credits.freeCredits > 0) parts.push(`free $${credits.freeCredits.toFixed(2)}`) + return `Sources: ${parts.join(" / ")}` +} + +function subscriptionLine(subscription: CommandCodeSubscription): string { + const plan = (subscription.planId ?? "Unknown").replace(/[_-]+/g, " ").trim() + const status = subscription.status ? ` (${subscription.status})` : "" + return `Plan: ${plan}${status}` +} + +function formatTokens(tokens: number): string { + if (tokens >= 1_000_000_000) return `${(tokens / 1_000_000_000).toFixed(1)}B` + if (tokens >= 1_000_000) return `${(tokens / 1_000_000).toFixed(1)}M` + if (tokens >= 1_000) return `${(tokens / 1_000).toFixed(1)}k` + return String(tokens) +} + +export function formatQuota(quota: CommandCodeQuota, now: () => number = Date.now): string { + const lines: string[] = [] + const remaining = quota.credits?.remainingCredits ?? 0 + const spent = quota.summary?.totalCost ?? 0 + const pool = remaining + spent + + if (quota.credits || quota.summary) { + lines.push("Credits") + lines.push(` Remaining: $${remaining.toFixed(2)} of $${pool.toFixed(2)}`) + lines.push(` Used: $${spent.toFixed(2)}`) + lines.push(` ${pool > 0 ? Math.round((spent / pool) * 100) : 0}% used`) + } + + const detail = creditsDetail(quota.credits) + if (detail) lines.push(detail) + if (quota.subscription) lines.push(subscriptionLine(quota.subscription)) + + if (quota.summary) { + lines.push("") + lines.push(quota.subscription?.currentPeriodStart ? "Usage (billing period)" : "Usage") + lines.push(` Cost: $${quota.summary.totalCost.toFixed(2)}`) + lines.push(` Requests: ${quota.summary.totalCount.toLocaleString("en-US")}`) + if (quota.summary.totalTokens !== undefined) { + lines.push(` Tokens: ${formatTokens(quota.summary.totalTokens)}`) + } + } + + lines.push("") + lines.push("Account") + lines.push(` ${quota.account.keyName ?? quota.account.login}`) + + const limits = quota.credits?.windowLimits ?? [] + if (limits.length > 0) { + lines.push("") + lines.push("Usage windows:") + lines.push(...formatWindowLimits(limits, now).map((line) => ` ${line}`)) + } + + if ((quota.unavailable?.length ?? 0) > 0) { + lines.push("") + lines.push(`Unavailable: ${quota.unavailable?.join(", ")}`) + } + + lines.push("") + lines.push("Full detail: https://commandcode.ai/usage") + return lines.join("\n") +} diff --git a/src/quota-types.ts b/src/quota-types.ts new file mode 100644 index 0000000..07f670e --- /dev/null +++ b/src/quota-types.ts @@ -0,0 +1,47 @@ +export interface CommandCodeWindowLimit { + window: "fiveHour" | "weekly" + used: number + cap: number + resetAt: number | null +} + +export interface CommandCodeCredits { + monthlyCredits: number + purchasedCredits: number + freeCredits: number + remainingCredits: number + windowLimits: CommandCodeWindowLimit[] +} + +export interface CommandCodeSubscription { + planId: string | null + status: string | null + currentPeriodStart: string | null + currentPeriodEnd: string | null +} + +export interface CommandCodeUsageSummary { + totalCost: number + totalCount: number + totalTokens?: number +} + +export type CommandCodeQuotaSection = "credits" | "subscription" | "usage" + +export interface CommandCodeQuota { + account: { + login: string + orgId: string | null + keyName?: string + } + credits: CommandCodeCredits | null + subscription: CommandCodeSubscription | null + summary: CommandCodeUsageSummary | null + unavailable?: readonly CommandCodeQuotaSection[] +} + +export type CommandCodeQuotaErrorKind = "config" | "http" | "network" | "timeout" + +export type CommandCodeQuotaResult = + | { ok: true; quota: CommandCodeQuota } + | { ok: false; error: { message: string; kind: CommandCodeQuotaErrorKind } } diff --git a/src/quota.ts b/src/quota.ts index 15a0263..e595b08 100644 --- a/src/quota.ts +++ b/src/quota.ts @@ -1,83 +1,34 @@ -/** - * Command Code usage/quota fetch layer for the `/commandcode-quota` command. - * - * Command Code exposes account usage through a set of authenticated alpha - * endpoints (the same ones the `cmd` CLI `/usage` command uses): - * - * - `/alpha/whoami` -> resolved account + optional org id - * - `/alpha/billing/credits` -> monthly/purchased/free credits + window limits - * - `/alpha/billing/subscriptions`-> plan id, status, billing period - * - `/alpha/usage/summary` -> period totals (cost, request count, optional tokens) - * - * These endpoints are not part of the documented public Provider API - * (`/provider/v1/*`) but are shipped with every `command-code` CLI release and - * authenticate with the same API key the provider already uses. Fetches are - * wrapped defensively so the quota command degrades to a readable error rather - * than surfacing raw transport details. - */ - import { redactCommandCodeErrorText } from "./overflow.ts" +import type { + CommandCodeCredits, + CommandCodeQuotaResult, + CommandCodeQuotaSection, + CommandCodeSubscription, + CommandCodeUsageSummary, + CommandCodeWindowLimit, +} from "./quota-types.ts" export const DEFAULT_API_BASE = "https://api.commandcode.ai" - export const QUOTA_TIMEOUT_MS = 15_000 -/** - * A single rolling usage window (the 5-hour or weekly cap on a plan's monthly - * credits). Values are measured in credit value, not request count. - */ -export interface CommandCodeWindowLimit { - window: "fiveHour" | "weekly" - used: number - cap: number - /** Unix epoch seconds when this window resets, normalized from seconds or ms. */ - resetAt: number | null +interface FetchOptions { + apiKey: string + baseUrl?: string + fetchImpl?: typeof fetch + timeoutMs?: number + extraHeaders?: Record } -/** Credits exposed by the `/alpha/billing/credits` endpoint. */ -export interface CommandCodeCredits { - monthlyCredits: number - purchasedCredits: number - freeCredits: number - remainingCredits: number - windowLimits: CommandCodeWindowLimit[] +interface HttpErrorShape { + __httpError: true + message: string + status: number + body: string } -/** Subscription/plan info exposed by `/alpha/billing/subscriptions`. */ -export interface CommandCodeSubscription { - planId: string | null - status: string | null - currentPeriodStart: string | null - currentPeriodEnd: string | null -} - -/** Period totals exposed by `/alpha/usage/summary`. */ -export interface CommandCodeUsageSummary { - totalCost: number - totalCount: number - /** Optional aggregate token count; only shown when the endpoint reports it. */ - totalTokens?: number -} - -/** Fully normalized quota snapshot for display. */ -export interface CommandCodeQuota { - account: { - login: string - orgId: string | null - /** Optional API key / account display name; falls back to login. */ - keyName?: string - } - credits: CommandCodeCredits | null - subscription: CommandCodeSubscription | null - summary: CommandCodeUsageSummary | null -} - -export type CommandCodeQuotaResult = - | { ok: true; quota: CommandCodeQuota } - | { ok: false; error: { message: string; kind: "config" | "http" | "network" | "timeout" } } - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error) +interface QuotaErrorShape { + __quotaError: true + kind: "timeout" | "network" } function isRecord(value: unknown): value is Record { @@ -85,442 +36,275 @@ function isRecord(value: unknown): value is Record { } function numberValue(value: unknown): number | undefined { - if (typeof value !== "number" || !Number.isFinite(value)) return undefined - return value >= 0 ? value : undefined + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined } function stringValue(value: unknown): string | undefined { return typeof value === "string" && value.length > 0 ? value : undefined } -interface FetchOptions { - apiKey: string - baseUrl?: string - fetchImpl?: typeof fetch - timeoutMs?: number - /** Extra HTTP headers merged after Content-Type/Authorization (e.g. ZDR). */ - extraHeaders?: Record +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) } -function buildUrl(path: string, params: Record): string { - const search = new URLSearchParams() - for (const [key, value] of Object.entries(params)) { - if (value !== undefined && value !== null && value !== "") search.set(key, value) +function normalizeResetAt(value: unknown): number | null { + let timestamp: number | undefined + if (typeof value === "number" && Number.isFinite(value)) timestamp = value + if (typeof value === "string" && value.length > 0) { + const trimmed = value.trim() + timestamp = /^\d+$/.test(trimmed) ? Number(trimmed) : Date.parse(trimmed) } - const query = search.toString() - return `${path}${query ? `?${query}` : ""}` + if (timestamp === undefined || !Number.isFinite(timestamp) || timestamp < 0) return null + return timestamp >= 1e12 ? Math.round(timestamp / 1000) : timestamp } -/** Extract the WindowLimit array from the top-level `windowLimits` object. */ export function windowLimitsFromCredits(value: unknown): CommandCodeWindowLimit[] { if (!isRecord(value)) return [] const limits: CommandCodeWindowLimit[] = [] - for (const [window, entry] of [ ["fiveHour", value.fiveHour], ["weekly", value.weekly], ] as const) { if (!isRecord(entry)) continue - const used = numberValue(entry.used) ?? 0 - const cap = numberValue(entry.cap) ?? 0 - if (cap <= 0 && used <= 0) continue - limits.push({ - window, - used, - cap, - resetAt: normalizeResetAt(entry.resetAt), - }) + const used = numberValue(entry.used) + const cap = numberValue(entry.cap) + if (used === undefined || cap === undefined || (used === 0 && cap === 0)) continue + limits.push({ window, used, cap, resetAt: normalizeResetAt(entry.resetAt) }) } - return limits } -/** - * Normalize a `resetAt` value to epoch seconds. Accepts seconds (10-digit), - * milliseconds (13-digit, live API), a numeric string, or an ISO timestamp - * string, then converts ms -> s consistently. Invalid/negative values -> null. - */ -function normalizeResetAt(value: unknown): number | null { - let num: number | undefined - if (typeof value === "number" && Number.isFinite(value)) { - num = value - } else if (typeof value === "string" && value.length > 0) { - const trimmed = value.trim() - if (/^\d+$/.test(trimmed)) { - num = Number(trimmed) - } else { - const parsed = Date.parse(trimmed) - if (!Number.isNaN(parsed)) num = Math.round(parsed / 1000) - } - } - if (num === undefined || num < 0) return null - return num >= 1e12 ? Math.round(num / 1000) : num -} - function parseCredits(value: unknown): CommandCodeCredits | null { - const credits = isRecord(value) ? value.credits : undefined - if (!isRecord(credits)) return null - - // `windowLimits` is a top-level sibling of `credits` in the - // `/alpha/billing/credits` response, not nested inside it. - const windowLimits = isRecord(value) ? value.windowLimits : undefined - - const monthlyCredits = numberValue(credits.monthlyCredits) ?? 0 - const purchasedCredits = numberValue(credits.purchasedCredits) ?? 0 - const freeCredits = numberValue(credits.freeCredits) ?? 0 - + if (!isRecord(value) || !isRecord(value.credits)) return null + const credits = value.credits + const monthlyCredits = numberValue(credits.monthlyCredits) + const purchasedCredits = numberValue(credits.purchasedCredits) + const freeCredits = numberValue(credits.freeCredits) + if (monthlyCredits === undefined && purchasedCredits === undefined && freeCredits === undefined) { + return null + } + const monthly = monthlyCredits ?? 0 + const purchased = purchasedCredits ?? 0 + const free = freeCredits ?? 0 return { - monthlyCredits, - purchasedCredits, - freeCredits, - remainingCredits: monthlyCredits + purchasedCredits + freeCredits, - windowLimits: windowLimitsFromCredits(windowLimits), + monthlyCredits: monthly, + purchasedCredits: purchased, + freeCredits: free, + remainingCredits: monthly + purchased + free, + windowLimits: windowLimitsFromCredits(value.windowLimits), } } function parseSubscription(value: unknown): CommandCodeSubscription | null { - const data = isRecord(value) ? value.data : undefined - if (!isRecord(data)) return null - + if (!isRecord(value) || !isRecord(value.data)) return null + const data = value.data + const planId = stringValue(data.planId) + const status = stringValue(data.status) + const currentPeriodStart = stringValue(data.currentPeriodStart) + const currentPeriodEnd = stringValue(data.currentPeriodEnd) + if (!planId && !status && !currentPeriodStart && !currentPeriodEnd) return null return { - planId: stringValue(data.planId) ?? null, - status: stringValue(data.status) ?? null, - currentPeriodStart: stringValue(data.currentPeriodStart) ?? null, - currentPeriodEnd: stringValue(data.currentPeriodEnd) ?? null, + planId: planId ?? null, + status: status ?? null, + currentPeriodStart: currentPeriodStart ?? null, + currentPeriodEnd: currentPeriodEnd ?? null, } } function parseSummary(value: unknown): CommandCodeUsageSummary | null { if (!isRecord(value)) return null + const totalCost = numberValue(value.totalCost) + const totalCount = numberValue(value.totalCount) + if (totalCost === undefined || totalCount === undefined) return null const totalTokens = numberValue(value.totalTokens) ?? numberValue(value.tokens) - return { - totalCost: numberValue(value.totalCost) ?? 0, - totalCount: numberValue(value.totalCount) ?? 0, - ...(totalTokens === undefined ? {} : { totalTokens }), - } + return { totalCost, totalCount, ...(totalTokens === undefined ? {} : { totalTokens }) } } function parseWhoami(value: unknown): { login: string orgId: string | null keyName?: string -} { - const org = isRecord(value) ? value.org : undefined - const user = isRecord(value) ? value.user : undefined +} | null { + if (!isRecord(value)) return null + const org = isRecord(value.org) ? value.org : undefined + const user = isRecord(value.user) ? value.user : undefined + const login = + (org ? stringValue(org.login) : undefined) ?? + (user ? (stringValue(user.userName) ?? stringValue(user.name)) : undefined) + if (!login) return null + const orgId = org ? stringValue(org.id) : undefined + const keyName = user ? (stringValue(user.keyName) ?? stringValue(user.displayName)) : undefined + return { login, orgId: orgId ?? null, ...(keyName ? { keyName } : {}) } +} - const orgLogin = isRecord(org) ? stringValue(org.login) : undefined - const orgId = isRecord(org) ? stringValue(org.id) : undefined - const userLogin = - (isRecord(user) ? stringValue(user.userName) : undefined) ?? - (isRecord(user) ? stringValue(user.name) : undefined) +function buildUrl(path: string, params: Record): string { + const search = new URLSearchParams() + for (const [key, value] of Object.entries(params)) { + if (value) search.set(key, value) + } + const query = search.toString() + return `${path}${query ? `?${query}` : ""}` +} - const keyName = - stringValue(isRecord(user) ? user.keyName : undefined) ?? - stringValue(isRecord(user) ? user.displayName : undefined) +function isHttpError(value: unknown): value is HttpErrorShape { + return ( + isRecord(value) && + value.__httpError === true && + typeof value.message === "string" && + typeof value.status === "number" && + typeof value.body === "string" + ) +} +function isQuotaError(value: unknown): value is QuotaErrorShape { + return ( + isRecord(value) && + value.__quotaError === true && + (value.kind === "timeout" || value.kind === "network") + ) +} + +function isBlockingHttpError(error: HttpErrorShape): boolean { + return error.status === 401 || error.status === 403 +} + +function httpFailure(error: HttpErrorShape, context: string): CommandCodeQuotaResult { + const detail = error.body.trim().slice(0, 200) return { - login: orgLogin ?? userLogin ?? "Unknown account", - orgId: orgId ?? null, - ...(keyName === undefined ? {} : { keyName }), + ok: false, + error: { + kind: "http", + message: redactValue( + `${context} request failed (${error.status}): ${detail || error.message}`, + ), + }, } } -/** - * Parse the `windowLimits` into a human-readable, header-safe line list that - * the formatting layer appends. Split out so the pure shape is independently - * testable. - */ -export function formatWindowLimits( - limits: readonly CommandCodeWindowLimit[], - now: () => number = Date.now, -): string[] { - const labels: Record = { - fiveHour: "5-hour", - weekly: "Weekly", - } +class QuotaTimeoutError extends Error {} - return limits.map((limit) => { - const label = labels[limit.window] ?? limit.window - const used = limit.used.toFixed(2) - const cap = limit.cap.toFixed(2) - const pct = limit.cap > 0 ? Math.round((limit.used / limit.cap) * 100) : 0 - const reset = limit.resetAt === null ? "" : ` (resets ${formatResetClock(limit.resetAt, now)})` - return `${label}: ${used} / ${cap} credits (${pct}% used)${reset}` - }) -} - -function formatResetClock(resetAtSeconds: number, now: () => number = Date.now): string { - const date = new Date(resetAtSeconds * 1000) - if (Number.isNaN(date.getTime())) return "unknown" - const nowMs = now() - const diffMs = date.getTime() - nowMs - if (diffMs <= 0) return "soon" - const minutes = Math.ceil(diffMs / 60_000) - if (minutes < 60) return `in ${minutes}m` - const hours = Math.floor(minutes / 60) - const rem = minutes % 60 - if (hours < 24) return rem > 0 ? `in ${hours}h ${rem}m` : `in ${hours}h` - const days = Math.floor(hours / 24) - return days === 1 ? "in 1 day" : `in ${days} days` -} - -/** Derived credits view for the Remaining/Used layout. */ -interface CreditView { - /** Credits remaining (monthly + purchased + free). */ - remaining: number - /** Dollars spent this period (totalCost). */ - spent: number - /** Total pool used as the percentage denominator: remaining + spent. */ - pool: number - /** Percent of the pool used, 0-100. */ - usedPercent: number - hasCreditsInfo: boolean -} - -function creditView(quota: CommandCodeQuota): CreditView { - const credits = quota.credits - const remaining = credits ? credits.remainingCredits : 0 - const spent = quota.summary?.totalCost ?? 0 - const pool = remaining + spent - const hasCreditsInfo = Boolean(credits) || spent > 0 - return { - remaining, - spent, - pool, - usedPercent: hasCreditsInfo ? Math.round((pool > 0 ? spent / pool : 0) * 100) : 0, - hasCreditsInfo, - } -} - -function creditDetailLine(credits: CommandCodeCredits | null): string { - if (!credits) return "" - const parts = [`monthly $${credits.monthlyCredits.toFixed(2)}`] - parts.push(`purchased $${credits.purchasedCredits.toFixed(2)}`) - if (credits.freeCredits > 0) parts.push(`free $${credits.freeCredits.toFixed(2)}`) - return `Sources: ${parts.join(" / ")}` -} - -function subscriptionLine(subscription: CommandCodeSubscription): string { - const rank = subscription.planId ?? "Unknown" - const plan = rank.replace(/[_-]+/g, " ").trim() - const status = subscription.status ? ` (${subscription.status})` : "" - return `Plan: ${plan}${status}` -} - -function accountName(account: CommandCodeQuota["account"]): string { - return account.keyName ?? account.login -} - -/** - * Render a normalized quota snapshot as clean, aligned, dashboard-style text - * suitable for `ui.notify`. Pure so it can be unit tested without a runtime. - */ -export function formatQuota(quota: CommandCodeQuota, now: () => number = Date.now): string { - const lines: string[] = [] - - const credit = creditView(quota) - if (credit.hasCreditsInfo) { - lines.push("") - lines.push("Credits") - lines.push(padValue(`Remaining: $${credit.remaining.toFixed(2)} of $${credit.pool.toFixed(2)}`)) - lines.push(padValue(`Used: $${credit.spent.toFixed(2)}`)) - lines.push(` ${credit.usedPercent}% used`) - } - - const detail = creditDetailLine(quota.credits) - if (detail) lines.push(detail) - - if (quota.subscription) lines.push(subscriptionLine(quota.subscription)) - - if (quota.summary) { - lines.push("") - lines.push("Usage (this month)") - lines.push(padValue(`Cost: $${quota.summary.totalCost.toFixed(2)}`)) - lines.push(padValue(`Requests: ${quota.summary.totalCount.toLocaleString("en-US")}`)) - if (quota.summary.totalTokens && quota.summary.totalTokens > 0) { - lines.push(padValue(`Tokens: ${formatTokens(quota.summary.totalTokens)}`)) - } - } - - lines.push("") - lines.push("Username") - lines.push(padValue(accountName(quota.account))) - - const limits = quota.credits?.windowLimits ?? [] - if (limits.length > 0) { - lines.push("") - lines.push("Usage windows:") - lines.push(...formatWindowLimits(limits, now).map((line) => ` ${line}`)) - } - - lines.push("") - lines.push(`Full detail: https://commandcode.ai/usage`) - - // Trim leading/trailing blank lines so sections stay cleanly separated. - while (lines.length > 0 && lines[0].length === 0) lines.shift() - while (lines.length > 0 && lines[lines.length - 1].length === 0) lines.pop() - return lines.join("\n") -} - -function padValue(value: string): string { - return ` ${value}` -} - -function formatTokens(tokens: number): string { - if (tokens >= 1_000_000_000) return `${(tokens / 1_000_000_000).toFixed(1)}B` - if (tokens >= 1_000_000) return `${(tokens / 1_000_000).toFixed(1)}M` - if (tokens >= 1_000) return `${(tokens / 1_000).toFixed(1)}k` - return String(tokens) -} - -/** - * Fetch the current account quota from Command Code. - * - * Resolution chain: whoami -> org id -> credits + subscription (parallel) -> - * summary (needs the billing period start). Any individual endpoint failing - * degrades gracefully: the remaining data is still reported, and a hard - * failure (auth/config, network) is surfaced as a typed error. - */ export async function fetchCommandCodeQuota( options: FetchOptions, ): Promise { if (!options.apiKey) { - return { - ok: false, - error: { message: "No Command Code API key found", kind: "config" }, - } + return { ok: false, error: { message: "No Command Code API key found", kind: "config" } } } const baseUrl = options.baseUrl ?? DEFAULT_API_BASE const fetchImpl = options.fetchImpl ?? fetch const timeoutMs = options.timeoutMs ?? QUOTA_TIMEOUT_MS - + const overallController = new AbortController() + const overallTimer = setTimeout(() => overallController.abort(), timeoutMs) const headers = { - "Content-Type": "application/json", + accept: "application/json", Authorization: `Bearer ${options.apiKey}`, ...options.extraHeaders, } - // One overall deadline shared across the sequential phases (whoami -> billing - // -> summary) so a slow or blackholed dependency cannot compound per-request - // timeouts into a ~45s stall; the command reports within QUOTA_TIMEOUT_MS. - const overallController = new AbortController() - const overallTimer = setTimeout(() => overallController.abort(), timeoutMs) - const request = async (path: string): Promise => { - // The overall deadline may already have fired (e.g. a prior phase consumed - // the budget) — AbortSignal does not replay past abort events to listeners - // added afterward, so check synchronously instead of relying on the listener. - if (overallController.signal.aborted) { - throw new QuotaTimeoutError() - } - const controller = new AbortController() - const timer = setTimeout(() => controller.abort(), timeoutMs) - const onOverallAbort = () => controller.abort() - overallController.signal.addEventListener("abort", onOverallAbort) + if (overallController.signal.aborted) throw new QuotaTimeoutError() try { const response = await fetchImpl(`${baseUrl}${path}`, { method: "GET", headers, - signal: controller.signal, + signal: overallController.signal, }) - if (!response.ok) { - let message = response.statusText - if (response.status === 401 || response.status === 403) { - message = "Command Code rejected the API key (401/403)" - } return { __httpError: true, - message, + message: + response.status === 401 || response.status === 403 + ? "Command Code rejected the API key" + : response.statusText, status: response.status, body: await response.text().catch(() => ""), - } + } satisfies HttpErrorShape } return await response.json() } catch (error) { - if (controller.signal.aborted) { - throw new QuotaTimeoutError() - } + if (overallController.signal.aborted) throw new QuotaTimeoutError() throw error - } finally { - clearTimeout(timer) - overallController.signal.removeEventListener("abort", onOverallAbort) } } - /** Optional-endpoint wrapper: never throws. Transport/timeout/parse failures - * become a sentinel so the rest of the dashboard still renders, matching the - * existing graceful degradation for HTTP 5xx responses. */ const safeRequest = async (path: string): Promise => { try { return await request(path) } catch (error) { return { __quotaError: true, - message: errorMessage(error), kind: error instanceof QuotaTimeoutError ? "timeout" : "network", - } + } satisfies QuotaErrorShape } } try { - const whoami = await request("/alpha/whoami") - if (isHttpError(whoami)) return httpFailure(whoami, "whoami") - const account = parseWhoami(whoami) + const whoamiRaw = await request("/alpha/whoami") + if (isHttpError(whoamiRaw)) return httpFailure(whoamiRaw, "whoami") + const account = parseWhoami(whoamiRaw) + if (!account) { + return { + ok: false, + error: { kind: "http", message: "Command Code returned an unrecognized account response" }, + } + } const orgId = account.orgId ?? undefined - const creditsPath = buildUrl("/alpha/billing/credits", { orgId }) - const subPath = buildUrl("/alpha/billing/subscriptions", { orgId }) - - const [creditsRaw, subRaw] = await Promise.all([safeRequest(creditsPath), safeRequest(subPath)]) - - // Hard auth/permission failures abort; everything else (including thrown - // network/timeout/parse failures) degrades to a null section. - if (isHttpError(creditsRaw) && isBlockingQuotaHttpError(creditsRaw)) { + const [creditsRaw, subscriptionRaw] = await Promise.all([ + safeRequest(buildUrl("/alpha/billing/credits", { orgId })), + safeRequest(buildUrl("/alpha/billing/subscriptions", { orgId })), + ]) + if (isHttpError(creditsRaw) && isBlockingHttpError(creditsRaw)) { return httpFailure(creditsRaw, "credits") } - if (isHttpError(subRaw) && isBlockingQuotaHttpError(subRaw)) { - return httpFailure(subRaw, "subscription") + if (isHttpError(subscriptionRaw) && isBlockingHttpError(subscriptionRaw)) { + return httpFailure(subscriptionRaw, "subscription") } + const unavailable: CommandCodeQuotaSection[] = [] const credits = - creditsRaw && !isHttpError(creditsRaw) && !isQuotaError(creditsRaw) - ? parseCredits(creditsRaw) - : null + isHttpError(creditsRaw) || isQuotaError(creditsRaw) ? null : parseCredits(creditsRaw) + if (!credits) unavailable.push("credits") const subscription = - subRaw && !isHttpError(subRaw) && !isQuotaError(subRaw) ? parseSubscription(subRaw) : null + isHttpError(subscriptionRaw) || isQuotaError(subscriptionRaw) + ? null + : parseSubscription(subscriptionRaw) + if (!subscription) unavailable.push("subscription") - const since = subscription?.currentPeriodStart ?? undefined - const summaryPath = buildUrl("/alpha/usage/summary", { orgId, since }) - const summaryRaw = await safeRequest(summaryPath) - if (isHttpError(summaryRaw) && isBlockingQuotaHttpError(summaryRaw)) { + const summaryRaw = await safeRequest( + buildUrl("/alpha/usage/summary", { + orgId, + since: subscription?.currentPeriodStart ?? undefined, + }), + ) + if (isHttpError(summaryRaw) && isBlockingHttpError(summaryRaw)) { return httpFailure(summaryRaw, "summary") } const summary = - summaryRaw && !isHttpError(summaryRaw) && !isQuotaError(summaryRaw) - ? parseSummary(summaryRaw) - : null + isHttpError(summaryRaw) || isQuotaError(summaryRaw) ? null : parseSummary(summaryRaw) + if (!summary) unavailable.push("usage") - if (credits === null && subscription === null && summary === null) { - if (overallController.signal.aborted) { - return { - ok: false, - error: { message: "Command Code quota request timed out", kind: "timeout" }, - } - } + if (!credits && !subscription && !summary) { return { ok: false, error: { - message: "Command Code returned no usage data for the account", - kind: "http", + kind: overallController.signal.aborted ? "timeout" : "http", + message: overallController.signal.aborted + ? "Command Code quota request timed out" + : "Command Code returned no recognized usage data for the account", }, } } return { ok: true, - quota: { account, credits, subscription, summary }, + quota: { + account, + credits, + subscription, + summary, + ...(unavailable.length > 0 ? { unavailable } : {}), + }, } } catch (error) { if (error instanceof QuotaTimeoutError || overallController.signal.aborted) { @@ -541,69 +325,7 @@ export async function fetchCommandCodeQuota( } } -class QuotaTimeoutError extends Error { - constructor() { - super("Command Code quota request timed out") - this.name = "QuotaTimeoutError" - } -} - -interface HttpErrorShape { - __httpError: true - message: string - status: number - body: string -} - -function isHttpError(value: unknown): value is HttpErrorShape { - if (!isRecord(value) || value.__httpError !== true) return false - return ( - typeof value.status === "number" && - typeof value.message === "string" && - typeof value.body === "string" - ) -} - -/** Sentinel produced by safeRequest for thrown transport/timeout/parse failures. */ -interface QuotaErrorShape { - __quotaError: true - message: string - kind: "timeout" | "network" -} - -function isQuotaError(value: unknown): value is QuotaErrorShape { - if (!isRecord(value) || value.__quotaError !== true) return false - return typeof value.message === "string" && (value.kind === "timeout" || value.kind === "network") -} - -/** - * Hard-failure HTTP statuses for the quota dashboard: authentication and - * permission failures. Rate limiting (429) is deliberately NOT included — it - * is a transient dependency condition that should degrade like other non-auth - * endpoint failures, not abort the whole command. - */ -function isBlockingQuotaHttpError(error: HttpErrorShape): boolean { - return error.status === 401 || error.status === 403 -} - -function httpFailure(error: HttpErrorShape, context: string): CommandCodeQuotaResult { - const detail = error.body.trim().slice(0, 200) - const message = detail - ? `${context} request failed (${error.status}): ${detail}` - : `${context} request failed (${error.status}): ${error.message}` - return { - ok: false, - error: { message: redactValue(message), kind: "http" }, - } -} - -/** Best-effort scrub of values that look like tokens/secrets from a message. */ export function redactValue(value: string): string { - // Reuse the broader Command Code redaction (Bearer, credential key-value - // fields, user_/cc_ tokens, query-string secrets, standalone keys) so quota - // errors get the same protection as stream errors. Additionally catch - // JSON-quoted credential fields ({"apiKey":"..."}) that the upstream pattern - // requires to be adjacent to `=`/`:`. return redactCommandCodeErrorText(value) .replace( /("\s*(?:api[-_ ]?key|apikey|access[-_ ]?token|refresh[-_ ]?token|token|secret|password|authorization)\s*"\s*:\s*")([^"]{8,})/gi, diff --git a/tests/test-quota-command.ts b/tests/test-quota-command.ts new file mode 100644 index 0000000..95692cd --- /dev/null +++ b/tests/test-quota-command.ts @@ -0,0 +1,117 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { registerCommandCodeQuota, type QuotaCommandContext } from "../src/quota-command.ts" +import type { CommandCodeQuotaResult } from "../src/quota-types.ts" + +class CommandApiDouble { + handler?: (args: string, ctx: QuotaCommandContext) => Promise + + registerCommand( + name: string, + options: { + description: string + handler: (args: string, ctx: QuotaCommandContext) => Promise + }, + ): void { + assert.equal(name, "commandcode-quota") + assert.match(options.description, /usage and quota/) + this.handler = options.handler + } +} + +function context(registryKey: string | undefined) { + const notifications: Array<{ message: string; type?: "info" | "warning" | "error" }> = [] + let waited = false + const value = { + async waitForIdle() { + waited = true + }, + modelRegistry: { + async getApiKeyForProvider(provider: string) { + assert.equal(provider, "commandcode") + return registryKey + }, + }, + ui: { + notify(message: string, type?: "info" | "warning" | "error") { + notifications.push({ message, type }) + }, + }, + } satisfies QuotaCommandContext + return { value, notifications, waited: () => waited } +} + +const quotaResult: CommandCodeQuotaResult = { + ok: true, + quota: { + account: { login: "alice", orgId: null }, + credits: null, + subscription: null, + summary: { totalCost: 1, totalCount: 2 }, + }, +} + +describe("commandcode-quota command", () => { + it("registers the command and resolves OMP placeholders through the fallback key", async () => { + const pi = new CommandApiDouble() + let requestKey = "" + let requestBase = "" + registerCommandCodeQuota(pi, { + apiBase: "https://api.commandcode.ai", + getConfiguredKey: () => "fallback-key", + fetchQuota: async (options) => { + requestKey = options.apiKey + requestBase = options.baseUrl ?? "" + return quotaResult + }, + }) + + assert.ok(pi.handler) + const ctx = context("$COMMANDCODE_API_KEY") + await pi.handler("", ctx.value) + assert.equal(ctx.waited(), true) + assert.equal(requestKey, "fallback-key") + assert.equal(requestBase, "https://api.commandcode.ai") + assert.equal(ctx.notifications.at(-1)?.type, "info") + assert.match(ctx.notifications.at(-1)?.message ?? "", /Requests: 2/) + }) + + it("warns without calling the endpoint when no API key is available", async () => { + const pi = new CommandApiDouble() + let called = false + registerCommandCodeQuota(pi, { + apiBase: "https://api.commandcode.ai", + getConfiguredKey: () => undefined, + fetchQuota: async () => { + called = true + return quotaResult + }, + }) + + assert.ok(pi.handler) + const ctx = context(undefined) + await pi.handler("", ctx.value) + assert.equal(called, false) + assert.equal(ctx.notifications.at(-1)?.type, "warning") + assert.match(ctx.notifications.at(-1)?.message ?? "", /requires an API key/) + }) + + it("redacts endpoint failures before notifying the host", async () => { + const pi = new CommandApiDouble() + registerCommandCodeQuota(pi, { + apiBase: "https://api.commandcode.ai", + getConfiguredKey: () => "real-key", + fetchQuota: async () => ({ + ok: false, + error: { kind: "http", message: "api_key=supersecretvalue123456 failed" }, + }), + }) + + assert.ok(pi.handler) + const ctx = context("real-key") + await pi.handler("", ctx.value) + assert.equal(ctx.notifications.at(-1)?.type, "error") + assert.doesNotMatch(ctx.notifications.at(-1)?.message ?? "", /supersecret/) + }) +}) diff --git a/tests/test-quota.ts b/tests/test-quota.ts index 5eea0d8..30f5a9c 100644 --- a/tests/test-quota.ts +++ b/tests/test-quota.ts @@ -8,15 +8,18 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" +import { formatQuota, formatWindowLimits } from "../src/quota-format.ts" import { DEFAULT_API_BASE, fetchCommandCodeQuota, - formatQuota, - formatWindowLimits, redactValue, windowLimitsFromCredits, } from "../src/quota.ts" -import type { CommandCodeQuota, CommandCodeCredits, CommandCodeWindowLimit } from "../src/quota.ts" +import type { + CommandCodeCredits, + CommandCodeQuota, + CommandCodeWindowLimit, +} from "../src/quota-types.ts" function jsonResponse(body: unknown, status = 200): Response { return new Response(JSON.stringify(body), { @@ -71,7 +74,7 @@ describe("Command Code quota", () => { assert.equal(limits[1]?.resetAt, 1_700_000_000) }) - it("renders a zero request count instead of dropping the Requests line", () => { + it("renders valid zero usage without claiming an unknown billing period", () => { const quota: CommandCodeQuota = { account: { login: "alice", orgId: null }, credits: null, @@ -79,6 +82,8 @@ describe("Command Code quota", () => { summary: { totalCost: 0, totalCount: 0 }, } const output = formatQuota(quota, () => 1_700_000_000_000) + assert.match(output, /Usage\n/) + assert.doesNotMatch(output, /billing period/) assert.match(output, /Requests: 0/) }) @@ -158,6 +163,20 @@ describe("Command Code quota", () => { } }) + it("rejects unrecognized successful endpoint schemas instead of displaying zero usage", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + return jsonResponse({ changed: "schema" }) + } + + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.error.kind, "http") + assert.match(result.error.message, /no recognized usage data/i) + }) + it("degrades gracefully when individual billing endpoints fail", async () => { const fetchImpl = async (input: RequestInfo | URL): Promise => { const url = String(input) @@ -174,6 +193,8 @@ describe("Command Code quota", () => { if (!result.ok) return assert.equal(result.quota.credits, null) assert.equal(result.quota.summary?.totalCost, 3.0) + assert.deepEqual(result.quota.unavailable, ["credits", "subscription"]) + assert.match(formatQuota(result.quota), /Unavailable: credits, subscription/) // Optional aggregate tokens are parsed when the summary reports them. assert.equal(result.quota.summary?.totalTokens, undefined) }) @@ -194,6 +215,7 @@ describe("Command Code quota", () => { assert.equal(result.quota.credits, null) assert.equal(result.quota.subscription?.planId, "pro") assert.equal(result.quota.summary?.totalCost, 3.0) + assert.deepEqual(result.quota.unavailable, ["credits"]) }) it("fails the command when the summary endpoint rejects auth/permission", async () => { @@ -299,8 +321,8 @@ describe("Command Code quota", () => { subscription: { planId: "pro", status: "active", - currentPeriodStart: "", - currentPeriodEnd: "", + currentPeriodStart: "2026-01-01T00:00:00Z", + currentPeriodEnd: "2026-02-01T00:00:00Z", }, summary: { totalCost: 12.34, totalCount: 1500 }, } @@ -312,10 +334,10 @@ describe("Command Code quota", () => { assert.match(output, /Used: \$12\.34/) assert.match(output, /Sources: monthly \$40\.00 \/ purchased \$10\.00 \/ free \$5\.00/) assert.match(output, /Plan: pro \(active\)/) - assert.match(output, /Usage \(this month\)/) + assert.match(output, /Usage \(billing period\)/) assert.match(output, /Cost: \$12\.34/) assert.match(output, /Requests: 1,500/) - assert.match(output, /Username/) + assert.match(output, /Account/) assert.match(output, /alice-inc/) assert.match(output, /5-hour: 8\.00 \/ 16\.00 credits/) assert.match(output, /Weekly: 20\.00 \/ 40\.00 credits/) From d14c2f0febdb3bb76d0482a3c1ea2165133574c2 Mon Sep 17 00:00:00 2001 From: laijxa Date: Tue, 25 Aug 2026 19:42:10 +0800 Subject: [PATCH 20/40] fix(models): add image support for deepseek-v4-flash-vision-exp deepseek/deepseek-v4-flash-vision-exp is served by the Provider API but was missing from the hardcoded MODEL_INPUT_MODALITIES allowlist, so pi rejected any conversation containing an image block. Notably this also rejected images returned by the read tool via toolResult: Error: Selected Command Code model does not support image content in tool results Adding the allowlist entry lets modelSupportsImageInput() return true and the converters forward images using the current Command Code wire format. Coverage: - regression test for modelSupportsImageInput("deepseek/deepseek-v4-flash-vision-exp") - end-to-end stream test for a tool-result image forwarded as a following user image (the concrete read reproduction), not only a user-attached image - end-to-end stream test asserting a text-only model still rejects tool-result images before any network access Refs: https://github.com/patlux/pi-commandcode-provider/issues/54 --- src/models.ts | 1 + tests/test-models.ts | 4 ++- tests/test-stream.ts | 79 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/src/models.ts b/src/models.ts index e102c00..706689d 100644 --- a/src/models.ts +++ b/src/models.ts @@ -43,6 +43,7 @@ export const MODEL_INPUT_MODALITIES: Readonly { it("matches command-code@1.32.1 image input capabilities", () => { assert.deepEqual(inputModalitiesForModel("gpt-5.6-luna"), ["text", "image"]) assert.deepEqual(inputModalitiesForModel("meta/muse-spark-1.2"), ["text", "image"]) + assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-flash-vision-exp"), ["text", "image"]) assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-pro"), ["text"]) assert.deepEqual(inputModalitiesForModel("unknown-new-model"), ["text"]) assert.equal(modelSupportsImageInput("gpt-5.6-luna"), true) + assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-flash-vision-exp"), true) assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-pro"), false) - assert.equal(Object.keys(MODEL_INPUT_MODALITIES).length, 37) + assert.equal(Object.keys(MODEL_INPUT_MODALITIES).length, 38) }) it("marks only known reasoning models as reasoning-capable", () => { diff --git a/tests/test-stream.ts b/tests/test-stream.ts index a429798..fa7626c 100644 --- a/tests/test-stream.ts +++ b/tests/test-stream.ts @@ -173,6 +173,85 @@ describe("streamCommandCode — successful streams", () => { ) }) + it("forwards a tool-result image as a following user image for vision-capable models", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode( + makeModel({ id: "deepseek/deepseek-v4-flash-vision-exp" }), + makeContext({ + messages: [ + { role: "user", content: "read the image" }, + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: [ + { type: "text", text: "image attached" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, + ], + }, + ], + }), + { apiKey: "mock-key" }, + ), + ) + + // No error: the tool-result image must not be rejected for this model. + assert.equal(events.at(-1)?.type, "done") + + const body = server.lastRequestBody() + // The tool-result text is forwarded on the tool message at index 2. + assert.equal( + objectAt(body, ["params", "messages", "2", "content", "0", "output", "value"]), + "image attached", + ) + // The tool-result image is forwarded as a following user image message at index 3. + assert.equal(objectAt(body, ["params", "messages", "3", "role"]), "user") + assert.equal( + objectAt(body, ["params", "messages", "3", "content", "0", "image"]), + "data:image/png;base64,aGVsbG8=", + ) + }) + + it("rejects a tool-result image before network access for text-only models", async () => { + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode( + makeModel({ id: "deepseek/deepseek-v4-pro" }), + makeContext({ + messages: [ + { role: "user", content: "read the image" }, + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }], + }, + ], + }), + { apiKey: "mock-key" }, + ), + ) + + assert.equal(events.at(-1)?.type, "error") + assert.match(events.at(-1)?.error.errorMessage ?? "", /does not support image content/i) + assert.equal(server.requestCount(), 0) + }) + it("rejects images before network access for text-only models", async () => { const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) From f5e2fece8ca624c952ac2511666d3ea7af7fc6ef Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 13:54:33 +0200 Subject: [PATCH 21/40] test(models): fix DeepSeek vision regressions --- tests/test-models.ts | 5 ++++- tests/test-stream.ts | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/test-models.ts b/tests/test-models.ts index 583ed5e..0bf34a3 100644 --- a/tests/test-models.ts +++ b/tests/test-models.ts @@ -103,7 +103,10 @@ describe("commandCodeModelsFromApiResponse()", () => { it("matches command-code@1.32.1 image input capabilities", () => { assert.deepEqual(inputModalitiesForModel("gpt-5.6-luna"), ["text", "image"]) assert.deepEqual(inputModalitiesForModel("meta/muse-spark-1.2"), ["text", "image"]) - assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-flash-vision-exp"), ["text", "image"]) + assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-flash-vision-exp"), [ + "text", + "image", + ]) assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-pro"), ["text"]) assert.deepEqual(inputModalitiesForModel("unknown-new-model"), ["text"]) assert.equal(modelSupportsImageInput("gpt-5.6-luna"), true) diff --git a/tests/test-stream.ts b/tests/test-stream.ts index fa7626c..f831dc2 100644 --- a/tests/test-stream.ts +++ b/tests/test-stream.ts @@ -247,8 +247,10 @@ describe("streamCommandCode — successful streams", () => { ), ) - assert.equal(events.at(-1)?.type, "error") - assert.match(events.at(-1)?.error.errorMessage ?? "", /does not support image content/i) + const lastEvent = events.at(-1) + assert.equal(lastEvent?.type, "error") + if (lastEvent?.type !== "error") throw new Error("expected error event") + assert.match(lastEvent.error.errorMessage ?? "", /does not support image content/i) assert.equal(server.requestCount(), 0) }) From 003d571b5e73e6eb4c48ef3c8d26cdec52d892b9 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 14:12:00 +0200 Subject: [PATCH 22/40] fix(models): refresh Command Code capabilities --- CHANGELOG.md | 1 + README.md | 2 +- src/core.ts | 2 +- src/models.ts | 14 +++++++++++--- tests/test-models.ts | 18 ++++++++++++++---- tests/test-stream.ts | 2 +- 6 files changed, 29 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f861b13..99640d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Refresh static model capabilities from `command-code@1.32.2`, including new image and reasoning metadata. - Add `/commandcode-quota` with live credits, plan, usage totals, and rolling-limit diagnostics from Command Code's alpha usage endpoints. - Add `zai-org/GLM-5.3` with its verified reasoning efforts and display pricing. - Prefer Command Code's Provider API (`/provider/v1/chat/completions` and `/provider/v1/messages`) and automatically fall back to the existing `/alpha/generate` transport only when the Provider API returns `403 upgrade_required` for a Go-plan account. diff --git a/README.md b/README.md index 6c757dc..6caf1d2 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ The following environment variables are intended for tests, local mocks, and com ## Image input -The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.1`; unknown models default to text-only until their upstream metadata is reviewed. +The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed. For vision-capable models, Pi's native provider adapters forward image blocks from user messages and tool results using the documented OpenAI or Anthropic message schema. Unknown and text-only models remain marked text-only in Pi. diff --git a/src/core.ts b/src/core.ts index 2eb92f8..d2b95f8 100644 --- a/src/core.ts +++ b/src/core.ts @@ -43,7 +43,7 @@ export * from "./overflow.ts" export * from "./types.ts" export const DEFAULT_API_BASE = "https://api.commandcode.ai" -export const COMMAND_CODE_CLI_VERSION = "1.32.1" +export const COMMAND_CODE_CLI_VERSION = "1.32.2" const DEFAULT_GENERATE_MAX_TOKENS = 64_000 const DEFAULT_MAX_RETRIES = 0 diff --git a/src/models.ts b/src/models.ts index 706689d..0b30631 100644 --- a/src/models.ts +++ b/src/models.ts @@ -12,7 +12,7 @@ export type CommandCodeApi = "openai-completions" | "anthropic-messages" export type CommandCodeInputType = "text" | "image" /** - * Model input modalities from the command-code@1.32.1 bundled catalog. + * Model input modalities from the command-code@1.32.2 bundled catalog. * Models omitted here remain text-only so newly discovered IDs never claim * image support without upstream evidence. */ @@ -21,6 +21,7 @@ export const MODEL_INPUT_MODALITIES: Readonly * Per-model reasoning efforts supported by Command Code's generate endpoint. * * The Provider API does not expose reasoning metadata. This is an exact - * snapshot of `reasoningEfforts` from the command-code@1.32.1 model catalog + * snapshot of `reasoningEfforts` from the command-code@1.32.2 model catalog * (`packages/shared/src/model-catalog.ts`, also published in the generated * `dist/bundled/command-code-knowledge/reference/models.md`). Models omitted * here let Command Code choose their reasoning depth, matching the CLI. */ export const MODEL_EFFORTS: Readonly> = { + "Qwen/Qwen3.8-27B": ["low", "medium", "xhigh"], "Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"], "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], "claude-opus-4-7": ["low", "medium", "high", "xhigh", "max"], @@ -89,6 +93,7 @@ export const MODEL_EFFORTS: Readonly { ) }) - it("matches command-code@1.32.1 image input capabilities", () => { + it("matches command-code@1.32.2 image input capabilities", () => { assert.deepEqual(inputModalitiesForModel("gpt-5.6-luna"), ["text", "image"]) assert.deepEqual(inputModalitiesForModel("meta/muse-spark-1.2"), ["text", "image"]) assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-flash-vision-exp"), [ "text", "image", ]) + assert.deepEqual(inputModalitiesForModel("Qwen/Qwen3.8-27B"), ["text", "image"]) + assert.deepEqual(inputModalitiesForModel("google/gemini-3.7-flash"), ["text", "image"]) + assert.deepEqual(inputModalitiesForModel("stealth/ox-alpha"), ["text", "image"]) assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-pro"), ["text"]) + assert.deepEqual(inputModalitiesForModel("zai-org/GLM-5.3"), ["text"]) assert.deepEqual(inputModalitiesForModel("unknown-new-model"), ["text"]) assert.equal(modelSupportsImageInput("gpt-5.6-luna"), true) assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-flash-vision-exp"), true) + assert.equal(modelSupportsImageInput("stealth/ox-alpha"), true) assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-pro"), false) - assert.equal(Object.keys(MODEL_INPUT_MODALITIES).length, 38) + assert.equal(Object.keys(MODEL_INPUT_MODALITIES).length, 41) }) it("marks only known reasoning models as reasoning-capable", () => { @@ -128,8 +133,9 @@ describe("commandCodeModelsFromApiResponse()", () => { assert.equal(models[1]?.reasoning, false) }) - it("matches the exact command-code@1.32.1 reasoning effort catalog", () => { + it("matches the exact command-code@1.32.2 reasoning effort catalog", () => { assert.deepEqual(MODEL_EFFORTS, { + "Qwen/Qwen3.8-27B": ["low", "medium", "xhigh"], "Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"], "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], "claude-opus-4-7": ["low", "medium", "high", "xhigh", "max"], @@ -138,6 +144,7 @@ describe("commandCodeModelsFromApiResponse()", () => { "claude-sonnet-4-6": ["low", "medium", "high", "xhigh", "max"], "claude-sonnet-5": ["low", "medium", "high", "xhigh", "max"], "deepseek/deepseek-v4-flash": ["high", "max"], + "deepseek/deepseek-v4-flash-vision-exp": ["high", "max"], "deepseek/deepseek-v4-pro": ["high", "max"], "gpt-5.3-codex": ["low", "medium", "high", "xhigh"], "gpt-5.4": ["low", "medium", "high", "xhigh"], @@ -150,10 +157,13 @@ describe("commandCodeModelsFromApiResponse()", () => { "google/gemini-3.5-flash": ["low", "medium", "high"], "google/gemini-3.5-flash-lite": ["low", "medium", "high"], "google/gemini-3.6-flash": ["low", "medium", "high"], + "google/gemini-3.7-flash": ["low", "medium", "high"], "sakana/fugu-ultra": ["high", "xhigh"], + "stealth/ox-alpha": ["low", "high", "max"], "xai/grok-4.5": ["low", "medium", "high"], - "zai-org/GLM-5.3": ["low", "high", "max"], + "xai/grok-4.6": ["low", "medium", "high", "xhigh"], "zai-org/GLM-5.2": ["high", "max"], + "zai-org/GLM-5.3": ["low", "high", "max"], }) }) diff --git a/tests/test-stream.ts b/tests/test-stream.ts index f831dc2..6346416 100644 --- a/tests/test-stream.ts +++ b/tests/test-stream.ts @@ -569,7 +569,7 @@ describe("streamCommandCode — request serialization", () => { const headers = server.lastRequestHeaders() assert.equal(headers.authorization, "Bearer mock-key") - assert.equal(headers["x-command-code-version"], "1.32.1") + assert.equal(headers["x-command-code-version"], "1.32.2") assert.equal(headers["x-project-slug"], "repo") assert.equal(headers["x-taste-learning"], "true") assert.equal(headers["x-co-flag"], "false") From f23863d326be864dd2682f8db6f87deb5148af7a Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 14:14:44 +0200 Subject: [PATCH 23/40] ci(models): check Command Code metadata daily --- .../check-commandcode-model-metadata.ts | 327 ++++++++++++++++++ .github/workflows/model-metadata.yml | 28 ++ CHANGELOG.md | 1 + README.md | 2 +- package.json | 7 +- tests/test-model-metadata-check.ts | 88 +++++ tsconfig.json | 2 +- 7 files changed, 450 insertions(+), 5 deletions(-) create mode 100644 .github/scripts/check-commandcode-model-metadata.ts create mode 100644 .github/workflows/model-metadata.yml create mode 100644 tests/test-model-metadata-check.ts diff --git a/.github/scripts/check-commandcode-model-metadata.ts b/.github/scripts/check-commandcode-model-metadata.ts new file mode 100644 index 0000000..7182cc5 --- /dev/null +++ b/.github/scripts/check-commandcode-model-metadata.ts @@ -0,0 +1,327 @@ +import { execFile } from "node:child_process" +import { appendFile, mkdtemp, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { pathToFileURL } from "node:url" +import { promisify } from "node:util" + +import { COMMAND_CODE_CLI_VERSION } from "../../src/core.ts" +import { MODEL_EFFORTS, MODEL_INPUT_MODALITIES } from "../../src/models.ts" + +const execFileAsync = promisify(execFile) +const MODELS_REFERENCE_PATH = "dist/bundled/command-code-knowledge/reference/models.md" +const CLI_BUNDLE_PATH = "dist/cli.mjs" +const TEXT_ONLY_MARKER = ',__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' +const VALID_EFFORTS = new Set(["low", "medium", "high", "xhigh", "max"]) + +export interface CommandCodeModelMetadata { + imageModelIds: readonly string[] + reasoningEfforts: Readonly> +} + +export interface ModelMetadataDiff { + addedImageModelIds: readonly string[] + removedImageModelIds: readonly string[] + addedReasoningModelIds: readonly string[] + removedReasoningModelIds: readonly string[] + changedReasoningModelIds: readonly string[] +} + +interface PackedPackage { + filename: string +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((entry) => typeof entry === "string") +} + +function sorted(values: Iterable): string[] { + return [...values].sort((left, right) => left.localeCompare(right)) +} + +function parsePackedPackage(value: unknown): PackedPackage { + if (!Array.isArray(value) || value.length !== 1 || !isRecord(value[0])) { + throw new Error("Expected npm pack to return one package") + } + + const filename = value[0].filename + if (typeof filename !== "string" || filename.length === 0) { + throw new Error("Expected npm pack to return a tarball filename") + } + + return { filename } +} + +export function parsePackageVersion(value: unknown): string { + if (typeof value !== "string" || !/^\d+\.\d+\.\d+(?:[-+].+)?$/.test(value)) { + throw new Error("Expected npm view to return one semantic version") + } + return value +} + +export function parseModelsReference(markdown: string): { + modelIds: readonly string[] + reasoningEfforts: Readonly> +} { + const modelIds = new Set() + const reasoningEfforts: Record = {} + + for (const line of markdown.split("\n")) { + const match = /^\| `([^`]+)` \| [^|]* \| [^|]* \| ([^|]*) \|/.exec(line) + if (!match) continue + + const modelId = match[1] + const effortsColumn = match[2]?.trim() + if (!modelId || !effortsColumn) throw new Error(`Could not parse model row: ${line}`) + if (modelIds.has(modelId)) throw new Error(`Duplicate model id in reference: ${modelId}`) + modelIds.add(modelId) + + if (effortsColumn === "—") continue + + const efforts = effortsColumn.split(",").map((effort) => effort.trim()) + if (efforts.length === 0 || efforts.some((effort) => !VALID_EFFORTS.has(effort))) { + throw new Error(`Unexpected reasoning efforts for ${modelId}: ${effortsColumn}`) + } + reasoningEfforts[modelId] = efforts + } + + if (modelIds.size === 0) throw new Error("No model rows found in Command Code reference") + + return { + modelIds: sorted(modelIds), + reasoningEfforts: Object.fromEntries( + Object.entries(reasoningEfforts).sort(([left], [right]) => left.localeCompare(right)), + ), + } +} + +export function parseKnownTextOnlyModelIds(bundle: string): readonly string[] { + const markerIndex = bundle.indexOf(TEXT_ONLY_MARKER) + if (markerIndex < 0) { + throw new Error("Could not find Command Code's isKnownTextOnlyModel catalog") + } + + const setStart = bundle.lastIndexOf("new Set([", markerIndex) + if (setStart < 0) throw new Error("Could not find the text-only model set") + + const arrayStart = setStart + "new Set(".length + const arrayEnd = markerIndex - 1 + const literal = bundle.slice(arrayStart, arrayEnd) + const parsed: unknown = JSON.parse(literal) + if (!isStringArray(parsed)) throw new Error("Expected the text-only model catalog to be strings") + + return sorted(new Set(parsed)) +} + +export function commandCodeModelMetadataFromContents( + modelsReference: string, + cliBundle: string, +): CommandCodeModelMetadata { + const reference = parseModelsReference(modelsReference) + const textOnlyModelIds = new Set(parseKnownTextOnlyModelIds(cliBundle)) + + return { + imageModelIds: reference.modelIds.filter((modelId) => !textOnlyModelIds.has(modelId)), + reasoningEfforts: reference.reasoningEfforts, + } +} + +export function currentModelMetadata(): CommandCodeModelMetadata { + return { + imageModelIds: sorted(Object.keys(MODEL_INPUT_MODALITIES)), + reasoningEfforts: Object.fromEntries( + Object.entries(MODEL_EFFORTS) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([modelId, efforts]) => [modelId, [...efforts]]), + ), + } +} + +export function diffModelMetadata( + current: CommandCodeModelMetadata, + upstream: CommandCodeModelMetadata, +): ModelMetadataDiff { + const currentImages = new Set(current.imageModelIds) + const upstreamImages = new Set(upstream.imageModelIds) + const currentReasoningIds = Object.keys(current.reasoningEfforts) + const upstreamReasoningIds = Object.keys(upstream.reasoningEfforts) + const currentReasoningSet = new Set(currentReasoningIds) + const upstreamReasoningSet = new Set(upstreamReasoningIds) + + return { + addedImageModelIds: sorted( + upstream.imageModelIds.filter((modelId) => !currentImages.has(modelId)), + ), + removedImageModelIds: sorted( + current.imageModelIds.filter((modelId) => !upstreamImages.has(modelId)), + ), + addedReasoningModelIds: sorted( + upstreamReasoningIds.filter((modelId) => !currentReasoningSet.has(modelId)), + ), + removedReasoningModelIds: sorted( + currentReasoningIds.filter((modelId) => !upstreamReasoningSet.has(modelId)), + ), + changedReasoningModelIds: sorted( + upstreamReasoningIds.filter( + (modelId) => + currentReasoningSet.has(modelId) && + JSON.stringify(current.reasoningEfforts[modelId]) !== + JSON.stringify(upstream.reasoningEfforts[modelId]), + ), + ), + } +} + +export function hasModelMetadataDiff(diff: ModelMetadataDiff): boolean { + return Object.values(diff).some((modelIds) => modelIds.length > 0) +} + +function formatList(modelIds: readonly string[]): string { + return modelIds.length > 0 ? modelIds.map((modelId) => `\`${modelId}\``).join(", ") : "None" +} + +function formatReasoningChanges( + modelIds: readonly string[], + current: CommandCodeModelMetadata, + upstream: CommandCodeModelMetadata, +): string { + if (modelIds.length === 0) return "None" + return modelIds + .map( + (modelId) => + `\`${modelId}\`: \`${(current.reasoningEfforts[modelId] ?? []).join(", ")}\` → \`${( + upstream.reasoningEfforts[modelId] ?? [] + ).join(", ")}\``, + ) + .join("
") +} + +function metadataReport( + packageVersion: string, + current: CommandCodeModelMetadata, + upstream: CommandCodeModelMetadata, + diff: ModelMetadataDiff, +): string { + const status = hasModelMetadataDiff(diff) ? "❌ Drift detected" : "✅ Metadata is current" + return [ + "## Command Code static model metadata", + "", + `**${status}**`, + "", + `- Repository snapshot: \`command-code@${COMMAND_CODE_CLI_VERSION}\``, + `- Inspected package: \`command-code@${packageVersion}\``, + `- Image-capable models: ${current.imageModelIds.length} repository / ${upstream.imageModelIds.length} upstream`, + `- Reasoning models: ${Object.keys(current.reasoningEfforts).length} repository / ${Object.keys(upstream.reasoningEfforts).length} upstream`, + "", + "| Change | Models |", + "| --- | --- |", + `| New image support | ${formatList(diff.addedImageModelIds)} |`, + `| Removed image support | ${formatList(diff.removedImageModelIds)} |`, + `| New reasoning metadata | ${formatList(diff.addedReasoningModelIds)} |`, + `| Removed reasoning metadata | ${formatList(diff.removedReasoningModelIds)} |`, + `| Changed reasoning efforts | ${formatReasoningChanges(diff.changedReasoningModelIds, current, upstream)} |`, + "", + ].join("\n") +} + +async function resolvePackageSpec( + packageSpec: string, + directory: string, + npmCacheDirectory: string, +): Promise { + if (packageSpec !== "command-code@latest") return packageSpec + + const { stdout } = await execFileAsync( + "npm", + ["view", packageSpec, "version", "--json", "--prefer-online", "--cache", npmCacheDirectory], + { + cwd: directory, + encoding: "utf-8", + }, + ) + return `command-code@${parsePackageVersion(JSON.parse(stdout) as unknown)}` +} + +async function inspectPackedPackage(packageSpec: string): Promise<{ + packageVersion: string + metadata: CommandCodeModelMetadata +}> { + const directory = await mkdtemp(join(tmpdir(), "pi-commandcode-model-check-")) + const npmCacheDirectory = join(directory, "npm-cache") + + try { + const resolvedPackageSpec = await resolvePackageSpec(packageSpec, directory, npmCacheDirectory) + const { stdout } = await execFileAsync( + "npm", + ["pack", resolvedPackageSpec, "--json", "--prefer-online", "--cache", npmCacheDirectory], + { + cwd: directory, + encoding: "utf-8", + maxBuffer: 10 * 1024 * 1024, + }, + ) + const packed = parsePackedPackage(JSON.parse(stdout) as unknown) + await execFileAsync("tar", ["-xzf", packed.filename], { cwd: directory }) + + const packageDirectory = join(directory, "package") + const packageJsonContents = await readFile(join(packageDirectory, "package.json"), "utf-8") + const packageJson: unknown = JSON.parse(packageJsonContents) + if (!isRecord(packageJson) || typeof packageJson.version !== "string") { + throw new Error("Expected command-code package.json to contain a version") + } + + const [modelsReference, cliBundle] = await Promise.all([ + readFile(join(packageDirectory, MODELS_REFERENCE_PATH), "utf-8"), + readFile(join(packageDirectory, CLI_BUNDLE_PATH), "utf-8"), + ]) + + return { + packageVersion: packageJson.version, + metadata: commandCodeModelMetadataFromContents(modelsReference, cliBundle), + } + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +async function main(): Promise { + const packageSpec = process.argv[2] ?? "command-code@latest" + const current = currentModelMetadata() + const upstreamPackage = await inspectPackedPackage(packageSpec) + const diff = diffModelMetadata(current, upstreamPackage.metadata) + const report = metadataReport( + upstreamPackage.packageVersion, + current, + upstreamPackage.metadata, + diff, + ) + + console.log(report) + + const summaryPath = process.env.GITHUB_STEP_SUMMARY + if (summaryPath) await appendFile(summaryPath, report, "utf-8") + + if (hasModelMetadataDiff(diff)) { + throw new Error( + `Static model metadata differs from command-code@${upstreamPackage.packageVersion}. Update src/models.ts and the snapshot version.`, + ) + } +} + +function isMainModule(): boolean { + const entrypoint = process.argv[1] + return entrypoint !== undefined && pathToFileURL(resolve(entrypoint)).href === import.meta.url +} + +if (isMainModule()) { + try { + await main() + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 + } +} diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml new file mode 100644 index 0000000..c933905 --- /dev/null +++ b/.github/workflows/model-metadata.yml @@ -0,0 +1,28 @@ +name: Command Code model metadata + +on: + schedule: + - cron: "17 6 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: commandcode-model-metadata + cancel-in-progress: true + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + registry-url: https://registry.npmjs.org + - run: npm ci + - name: Compare static metadata with the latest Command Code CLI + run: npm run check:model-metadata diff --git a/CHANGELOG.md b/CHANGELOG.md index 99640d1..c4e2930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Add a daily GitHub Actions check that compares static image and reasoning metadata with the latest published Command Code CLI catalog. - Refresh static model capabilities from `command-code@1.32.2`, including new image and reasoning metadata. - Add `/commandcode-quota` with live credits, plan, usage totals, and rolling-limit diagnostics from Command Code's alpha usage endpoints. - Add `zai-org/GLM-5.3` with its verified reasoning efforts and display pricing. diff --git a/README.md b/README.md index 6caf1d2..7657981 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ The following environment variables are intended for tests, local mocks, and com ## Image input -The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed. +The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed. A daily GitHub Actions check compares the static image and reasoning metadata with the latest published CLI package and reports any drift. For vision-capable models, Pi's native provider adapters forward image blocks from user messages and tool results using the documented OpenAI or Anthropic message schema. Unknown and text-only models remain marked text-only in Pi. diff --git a/package.json b/package.json index 6e07011..8d14353 100644 --- a/package.json +++ b/package.json @@ -29,16 +29,17 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", "pi:isolated": "node scripts/pi-isolated.mjs", "pi:authenticated": "node scripts/pi-authenticated.mjs", + "check:model-metadata": "tsx .github/scripts/check-commandcode-model-metadata.ts command-code@latest", "test:quota": "tsx tests/test-quota.ts && tsx tests/test-quota-command.ts", - "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", + "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", "test:api-key": "tsx tests/test-api-key.ts", - "test:models": "tsx tests/test-models.ts", + "test:models": "tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts", "test:runtime": "tsx tests/test-runtime.ts", "test:pricing": "tsx tests/test-pricing.ts", "test:oauth": "tsx tests/test-oauth.ts", diff --git a/tests/test-model-metadata-check.ts b/tests/test-model-metadata-check.ts new file mode 100644 index 0000000..145b298 --- /dev/null +++ b/tests/test-model-metadata-check.ts @@ -0,0 +1,88 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { + commandCodeModelMetadataFromContents, + diffModelMetadata, + hasModelMetadataDiff, + parseKnownTextOnlyModelIds, + parseModelsReference, + parsePackageVersion, + type CommandCodeModelMetadata, +} from "../.github/scripts/check-commandcode-model-metadata.ts" + +const MODELS_REFERENCE = ` +| Id (use EXACTLY this) | Name | Context | Efforts | $/1M in/out · cache read | Min plan | Best for | +|---|---|---|---|---|---|---| +| \`vision-model\` | Vision | 1M | low, high | $1/$2 | Go | images | +| \`text-model\` | Text | 200K | — | $1/$2 | Go | text | +` + +const CLI_BUNDLE = + 'const catalog=new Set(["text-model"]),__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' + +describe("Command Code model metadata checker", () => { + it("parses model ids and reasoning efforts from the generated reference", () => { + assert.deepEqual(parseModelsReference(MODELS_REFERENCE), { + modelIds: ["text-model", "vision-model"], + reasoningEfforts: { "vision-model": ["low", "high"] }, + }) + }) + + it("extracts the text-only set from the bundled CLI catalog", () => { + assert.deepEqual(parseKnownTextOnlyModelIds(CLI_BUNDLE), ["text-model"]) + }) + + it("accepts one exact npm registry version and rejects stale-looking output shapes", () => { + assert.equal(parsePackageVersion("1.32.2"), "1.32.2") + assert.equal(parsePackageVersion("2.0.0-beta.1"), "2.0.0-beta.1") + assert.throws(() => parsePackageVersion(["1.32.1", "1.32.2"]), /one semantic version/) + assert.throws(() => parsePackageVersion("latest"), /one semantic version/) + }) + + it("derives image support by excluding known text-only models", () => { + assert.deepEqual(commandCodeModelMetadataFromContents(MODELS_REFERENCE, CLI_BUNDLE), { + imageModelIds: ["vision-model"], + reasoningEfforts: { "vision-model": ["low", "high"] }, + }) + }) + + it("reports additions, removals, and changed reasoning efforts", () => { + const current: CommandCodeModelMetadata = { + imageModelIds: ["removed-image", "stable-image"], + reasoningEfforts: { + "changed-reasoning": ["low"], + "removed-reasoning": ["high"], + "stable-reasoning": ["low", "high"], + }, + } + const upstream: CommandCodeModelMetadata = { + imageModelIds: ["added-image", "stable-image"], + reasoningEfforts: { + "added-reasoning": ["max"], + "changed-reasoning": ["low", "high"], + "stable-reasoning": ["low", "high"], + }, + } + + const diff = diffModelMetadata(current, upstream) + + assert.deepEqual(diff, { + addedImageModelIds: ["added-image"], + removedImageModelIds: ["removed-image"], + addedReasoningModelIds: ["added-reasoning"], + removedReasoningModelIds: ["removed-reasoning"], + changedReasoningModelIds: ["changed-reasoning"], + }) + assert.equal(hasModelMetadataDiff(diff), true) + }) + + it("rejects unexpected upstream structures instead of silently passing", () => { + assert.throws(() => parseModelsReference("# no catalog"), /No model rows/) + assert.throws( + () => parseModelsReference(MODELS_REFERENCE.replace("low, high", "low, turbo")), + /Unexpected reasoning efforts/, + ) + assert.throws(() => parseKnownTextOnlyModelIds("const unrelated = true"), /Could not find/) + }) +}) diff --git a/tsconfig.json b/tsconfig.json index c691796..2cf50c2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,5 +9,5 @@ "strict": true, "types": ["node"] }, - "include": ["src/**/*.ts", "tests/**/*.ts"] + "include": [".github/scripts/**/*.ts", "src/**/*.ts", "tests/**/*.ts"] } From 7ed38d5f70cc864b111925082d66dfb525552ba2 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 14:43:46 +0200 Subject: [PATCH 24/40] ci(models): run metadata check on pull requests --- .github/scripts/check-commandcode-model-metadata.ts | 5 +---- .github/workflows/model-metadata.yml | 13 ++++++++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/scripts/check-commandcode-model-metadata.ts b/.github/scripts/check-commandcode-model-metadata.ts index 7182cc5..bfc72ab 100644 --- a/.github/scripts/check-commandcode-model-metadata.ts +++ b/.github/scripts/check-commandcode-model-metadata.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process" -import { appendFile, mkdtemp, readFile, rm } from "node:fs/promises" +import { mkdtemp, readFile, rm } from "node:fs/promises" import { tmpdir } from "node:os" import { join, resolve } from "node:path" import { pathToFileURL } from "node:url" @@ -302,9 +302,6 @@ async function main(): Promise { console.log(report) - const summaryPath = process.env.GITHUB_STEP_SUMMARY - if (summaryPath) await appendFile(summaryPath, report, "utf-8") - if (hasModelMetadataDiff(diff)) { throw new Error( `Static model metadata differs from command-code@${upstreamPackage.packageVersion}. Update src/models.ts and the snapshot version.`, diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml index c933905..809eef8 100644 --- a/.github/workflows/model-metadata.yml +++ b/.github/workflows/model-metadata.yml @@ -1,6 +1,14 @@ name: Command Code model metadata on: + pull_request: + branches: [main] + paths: + - ".github/scripts/check-commandcode-model-metadata.ts" + - ".github/workflows/model-metadata.yml" + - "src/core.ts" + - "src/models.ts" + - "tests/test-model-metadata-check.ts" schedule: - cron: "17 6 * * *" workflow_dispatch: @@ -25,4 +33,7 @@ jobs: registry-url: https://registry.npmjs.org - run: npm ci - name: Compare static metadata with the latest Command Code CLI - run: npm run check:model-metadata + run: npm run check:model-metadata | tee model-metadata-report.md + - name: Publish metadata report + if: always() + run: cat model-metadata-report.md >> "$GITHUB_STEP_SUMMARY" From af5c41aaa187bf4be678c236280f0d70f05250f6 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 14:48:03 +0200 Subject: [PATCH 25/40] ci(models): limit metadata check to scheduled runs --- .github/workflows/model-metadata.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml index 809eef8..de26c03 100644 --- a/.github/workflows/model-metadata.yml +++ b/.github/workflows/model-metadata.yml @@ -1,14 +1,6 @@ name: Command Code model metadata on: - pull_request: - branches: [main] - paths: - - ".github/scripts/check-commandcode-model-metadata.ts" - - ".github/workflows/model-metadata.yml" - - "src/core.ts" - - "src/models.ts" - - "tests/test-model-metadata-check.ts" schedule: - cron: "17 6 * * *" workflow_dispatch: From ef5d723182db3294bb00e3d441bed315f2f9b98a Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 14:48:29 +0200 Subject: [PATCH 26/40] Revert "ci(models): limit metadata check to scheduled runs" This reverts commit af5c41aaa187bf4be678c236280f0d70f05250f6. --- .github/workflows/model-metadata.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml index de26c03..809eef8 100644 --- a/.github/workflows/model-metadata.yml +++ b/.github/workflows/model-metadata.yml @@ -1,6 +1,14 @@ name: Command Code model metadata on: + pull_request: + branches: [main] + paths: + - ".github/scripts/check-commandcode-model-metadata.ts" + - ".github/workflows/model-metadata.yml" + - "src/core.ts" + - "src/models.ts" + - "tests/test-model-metadata-check.ts" schedule: - cron: "17 6 * * *" workflow_dispatch: From 3d8758fe3b65472651384f116a23a36727012462 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 15:08:22 +0200 Subject: [PATCH 27/40] feat(models): generate Command Code catalog snapshot --- .../check-commandcode-model-metadata.ts | 101 +++++++++++++++-- package.json | 3 +- src/commandcode-catalog.ts | 84 ++++++++++++++ src/core.ts | 3 +- src/models.ts | 103 ++---------------- tests/test-model-metadata-check.ts | 61 +++++++++++ tests/test-models.ts | 48 +++----- tests/test-stream.ts | 3 +- 8 files changed, 269 insertions(+), 137 deletions(-) create mode 100644 src/commandcode-catalog.ts diff --git a/.github/scripts/check-commandcode-model-metadata.ts b/.github/scripts/check-commandcode-model-metadata.ts index bfc72ab..4cfa4e2 100644 --- a/.github/scripts/check-commandcode-model-metadata.ts +++ b/.github/scripts/check-commandcode-model-metadata.ts @@ -1,18 +1,24 @@ import { execFile } from "node:child_process" -import { mkdtemp, readFile, rm } from "node:fs/promises" +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" import { join, resolve } from "node:path" import { pathToFileURL } from "node:url" import { promisify } from "node:util" -import { COMMAND_CODE_CLI_VERSION } from "../../src/core.ts" -import { MODEL_EFFORTS, MODEL_INPUT_MODALITIES } from "../../src/models.ts" +import { + COMMAND_CODE_CLI_VERSION, + MODEL_EFFORTS, + MODEL_INPUT_MODALITIES, +} from "../../src/commandcode-catalog.ts" const execFileAsync = promisify(execFile) const MODELS_REFERENCE_PATH = "dist/bundled/command-code-knowledge/reference/models.md" const CLI_BUNDLE_PATH = "dist/cli.mjs" const TEXT_ONLY_MARKER = ',__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' -const VALID_EFFORTS = new Set(["low", "medium", "high", "xhigh", "max"]) +const VALID_EFFORTS = new Set(["minimal", "low", "medium", "high", "xhigh", "max"]) +const CATALOG_SOURCE_PATH = new URL("../../src/commandcode-catalog.ts", import.meta.url) +const README_PATH = new URL("../../README.md", import.meta.url) +const CHANGELOG_PATH = new URL("../../CHANGELOG.md", import.meta.url) export interface CommandCodeModelMetadata { imageModelIds: readonly string[] @@ -20,6 +26,7 @@ export interface CommandCodeModelMetadata { } export interface ModelMetadataDiff { + versionChanged: boolean addedImageModelIds: readonly string[] removedImageModelIds: readonly string[] addedReasoningModelIds: readonly string[] @@ -144,6 +151,8 @@ export function currentModelMetadata(): CommandCodeModelMetadata { export function diffModelMetadata( current: CommandCodeModelMetadata, upstream: CommandCodeModelMetadata, + currentVersion = COMMAND_CODE_CLI_VERSION, + upstreamVersion = COMMAND_CODE_CLI_VERSION, ): ModelMetadataDiff { const currentImages = new Set(current.imageModelIds) const upstreamImages = new Set(upstream.imageModelIds) @@ -153,6 +162,7 @@ export function diffModelMetadata( const upstreamReasoningSet = new Set(upstreamReasoningIds) return { + versionChanged: currentVersion !== upstreamVersion, addedImageModelIds: sorted( upstream.imageModelIds.filter((modelId) => !currentImages.has(modelId)), ), @@ -177,7 +187,10 @@ export function diffModelMetadata( } export function hasModelMetadataDiff(diff: ModelMetadataDiff): boolean { - return Object.values(diff).some((modelIds) => modelIds.length > 0) + return ( + diff.versionChanged || + Object.entries(diff).some(([key, modelIds]) => key !== "versionChanged" && modelIds.length > 0) + ) } function formatList(modelIds: readonly string[]): string { @@ -200,6 +213,66 @@ function formatReasoningChanges( .join("
") } +function quoted(value: string): string { + return JSON.stringify(value) +} + +function recordEntries( + values: Readonly>, +): readonly [string, readonly string[]][] { + return Object.entries(values).sort(([left], [right]) => left.localeCompare(right)) +} + +export function renderCommandCodeCatalog( + packageVersion: string, + metadata: CommandCodeModelMetadata, +): string { + const imageEntries = sorted(metadata.imageModelIds) + .map((modelId) => ` ${quoted(modelId)}: ["text", "image"],`) + .join("\n") + const reasoningEntries = recordEntries(metadata.reasoningEfforts) + .map( + ([modelId, efforts]) => + ` ${quoted(modelId)}: [${efforts.map((effort) => quoted(effort)).join(", ")}],`, + ) + .join("\n") + + return `export const COMMAND_CODE_CLI_VERSION = ${quoted(packageVersion)}\n\nexport type CommandCodeInputType = "text" | "image"\nexport type CommandCodeReasoningEffort = "minimal" | "low" | "medium" | "high" | "xhigh" | "max"\n\n/**\n * Generated from command-code@${packageVersion} by \`npm run sync:commandcode-catalog\`.\n * Do not edit manually.\n */\nexport const MODEL_INPUT_MODALITIES: Readonly> = {\n${imageEntries}\n}\n\nexport const MODEL_EFFORTS: Readonly> = {\n${reasoningEntries}\n}\n` +} + +function updateDocumentedCatalogVersion( + contents: string, + packageVersion: string, + context: string, +): string { + const pattern = /command-code@\d+\.\d+\.\d+(?:[-+][^`\s,]+)?/ + if (!pattern.test(contents)) throw new Error(`Could not find the ${context} catalog version`) + return contents.replace(pattern, `command-code@${packageVersion}`) +} + +export function updateReadmeCatalogVersion(readme: string, packageVersion: string): string { + return updateDocumentedCatalogVersion(readme, packageVersion, "README") +} + +export function updateChangelogCatalogVersion(changelog: string, packageVersion: string): string { + return updateDocumentedCatalogVersion(changelog, packageVersion, "changelog") +} + +async function writeSynchronizedCatalog( + packageVersion: string, + metadata: CommandCodeModelMetadata, +): Promise { + const [readme, changelog] = await Promise.all([ + readFile(README_PATH, "utf-8"), + readFile(CHANGELOG_PATH, "utf-8"), + ]) + await Promise.all([ + writeFile(CATALOG_SOURCE_PATH, renderCommandCodeCatalog(packageVersion, metadata), "utf-8"), + writeFile(README_PATH, updateReadmeCatalogVersion(readme, packageVersion), "utf-8"), + writeFile(CHANGELOG_PATH, updateChangelogCatalogVersion(changelog, packageVersion), "utf-8"), + ]) +} + function metadataReport( packageVersion: string, current: CommandCodeModelMetadata, @@ -219,6 +292,7 @@ function metadataReport( "", "| Change | Models |", "| --- | --- |", + `| CLI version | ${diff.versionChanged ? `\`${COMMAND_CODE_CLI_VERSION}\` → \`${packageVersion}\`` : "Current"} |`, `| New image support | ${formatList(diff.addedImageModelIds)} |`, `| Removed image support | ${formatList(diff.removedImageModelIds)} |`, `| New reasoning metadata | ${formatList(diff.addedReasoningModelIds)} |`, @@ -289,10 +363,17 @@ async function inspectPackedPackage(packageSpec: string): Promise<{ } async function main(): Promise { - const packageSpec = process.argv[2] ?? "command-code@latest" + const write = process.argv.includes("--write") + const packageSpec = + process.argv.find((argument) => argument.startsWith("command-code@")) ?? "command-code@latest" const current = currentModelMetadata() const upstreamPackage = await inspectPackedPackage(packageSpec) - const diff = diffModelMetadata(current, upstreamPackage.metadata) + const diff = diffModelMetadata( + current, + upstreamPackage.metadata, + COMMAND_CODE_CLI_VERSION, + upstreamPackage.packageVersion, + ) const report = metadataReport( upstreamPackage.packageVersion, current, @@ -302,6 +383,12 @@ async function main(): Promise { console.log(report) + if (write) { + await writeSynchronizedCatalog(upstreamPackage.packageVersion, upstreamPackage.metadata) + console.log(`Synchronized static metadata with command-code@${upstreamPackage.packageVersion}.`) + return + } + if (hasModelMetadataDiff(diff)) { throw new Error( `Static model metadata differs from command-code@${upstreamPackage.packageVersion}. Update src/models.ts and the snapshot version.`, diff --git a/package.json b/package.json index 8d14353..660b6c1 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,8 @@ "format": "prettier --write '**/*.{ts,mjs,json,md}'", "pi:isolated": "node scripts/pi-isolated.mjs", "pi:authenticated": "node scripts/pi-authenticated.mjs", - "check:model-metadata": "tsx .github/scripts/check-commandcode-model-metadata.ts command-code@latest", + "check:commandcode-catalog": "tsx .github/scripts/check-commandcode-model-metadata.ts command-code@latest", + "sync:commandcode-catalog": "tsx .github/scripts/check-commandcode-model-metadata.ts command-code@latest --write", "test:quota": "tsx tests/test-quota.ts && tsx tests/test-quota-command.ts", "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", "test:api-key": "tsx tests/test-api-key.ts", diff --git a/src/commandcode-catalog.ts b/src/commandcode-catalog.ts new file mode 100644 index 0000000..4249514 --- /dev/null +++ b/src/commandcode-catalog.ts @@ -0,0 +1,84 @@ +export const COMMAND_CODE_CLI_VERSION = "1.32.2" + +export type CommandCodeInputType = "text" | "image" +export type CommandCodeReasoningEffort = "minimal" | "low" | "medium" | "high" | "xhigh" | "max" + +/** + * Generated from command-code@1.32.2 by `npm run sync:commandcode-catalog`. + * Do not edit manually. + */ +export const MODEL_INPUT_MODALITIES: Readonly> = { + "claude-fable-5": ["text", "image"], + "claude-haiku-4-5-20251001": ["text", "image"], + "claude-opus-4-7": ["text", "image"], + "claude-opus-4-8": ["text", "image"], + "claude-opus-5": ["text", "image"], + "claude-sonnet-4-6": ["text", "image"], + "claude-sonnet-5": ["text", "image"], + "deepseek/deepseek-v4-flash-vision-exp": ["text", "image"], + "google/gemini-3.1-flash-lite": ["text", "image"], + "google/gemini-3.5-flash": ["text", "image"], + "google/gemini-3.5-flash-lite": ["text", "image"], + "google/gemini-3.6-flash": ["text", "image"], + "google/gemini-3.7-flash": ["text", "image"], + "gpt-5.3-codex": ["text", "image"], + "gpt-5.4": ["text", "image"], + "gpt-5.4-mini": ["text", "image"], + "gpt-5.5": ["text", "image"], + "gpt-5.6-luna": ["text", "image"], + "gpt-5.6-sol": ["text", "image"], + "gpt-5.6-terra": ["text", "image"], + "meta/muse-spark-1.1": ["text", "image"], + "meta/muse-spark-1.2": ["text", "image"], + "meta/muse-spark-1.2-contributor": ["text", "image"], + "MiniMaxAI/MiniMax-M3": ["text", "image"], + "moonshotai/Kimi-K2.5": ["text", "image"], + "moonshotai/Kimi-K2.6": ["text", "image"], + "moonshotai/Kimi-K2.7-Code": ["text", "image"], + "moonshotai/Kimi-K2.7-Code-Highspeed": ["text", "image"], + "moonshotai/Kimi-K3": ["text", "image"], + "Qwen/Qwen3.6-Plus": ["text", "image"], + "Qwen/Qwen3.7-Flash": ["text", "image"], + "Qwen/Qwen3.7-Plus": ["text", "image"], + "Qwen/Qwen3.8-27B": ["text", "image"], + "Qwen/Qwen3.8-Max": ["text", "image"], + "sakana/fugu-ultra": ["text", "image"], + "stealth/ox-alpha": ["text", "image"], + "stepfun/Step-3.7-Flash": ["text", "image"], + "thinkingmachines/inkling": ["text", "image"], + "thinkingmachines/inkling-small": ["text", "image"], + "xai/grok-4.5": ["text", "image"], + "xiaomi/mimo-v2.5": ["text", "image"], +} + +export const MODEL_EFFORTS: Readonly> = { + "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], + "claude-opus-4-7": ["low", "medium", "high", "xhigh", "max"], + "claude-opus-4-8": ["low", "medium", "high", "xhigh", "max"], + "claude-opus-5": ["low", "medium", "high", "xhigh", "max"], + "claude-sonnet-4-6": ["low", "medium", "high", "xhigh", "max"], + "claude-sonnet-5": ["low", "medium", "high", "xhigh", "max"], + "deepseek/deepseek-v4-flash": ["high", "max"], + "deepseek/deepseek-v4-flash-vision-exp": ["high", "max"], + "deepseek/deepseek-v4-pro": ["high", "max"], + "google/gemini-3.1-flash-lite": ["low", "medium", "high"], + "google/gemini-3.5-flash": ["low", "medium", "high"], + "google/gemini-3.5-flash-lite": ["low", "medium", "high"], + "google/gemini-3.6-flash": ["low", "medium", "high"], + "google/gemini-3.7-flash": ["low", "medium", "high"], + "gpt-5.3-codex": ["low", "medium", "high", "xhigh"], + "gpt-5.4": ["low", "medium", "high", "xhigh"], + "gpt-5.4-mini": ["low", "medium", "high"], + "gpt-5.5": ["low", "medium", "high", "xhigh"], + "gpt-5.6-luna": ["low", "medium", "high", "xhigh", "max"], + "gpt-5.6-sol": ["low", "medium", "high", "xhigh", "max"], + "gpt-5.6-terra": ["low", "medium", "high", "xhigh", "max"], + "Qwen/Qwen3.8-27B": ["low", "medium", "xhigh"], + "Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"], + "sakana/fugu-ultra": ["high", "xhigh"], + "stealth/ox-alpha": ["low", "high", "max"], + "xai/grok-4.5": ["low", "medium", "high"], + "xai/grok-4.6": ["low", "medium", "high", "xhigh"], + "zai-org/GLM-5.2": ["high", "max"], + "zai-org/GLM-5.3": ["low", "high", "max"], +} diff --git a/src/core.ts b/src/core.ts index d2b95f8..57ed455 100644 --- a/src/core.ts +++ b/src/core.ts @@ -7,6 +7,7 @@ import { randomUUID } from "node:crypto" +import { COMMAND_CODE_CLI_VERSION } from "./commandcode-catalog.ts" import { commandCodeErrorMessage, redactCommandCodeErrorText } from "./overflow.ts" import { modelSupportsImageInput } from "./models.ts" import { @@ -43,7 +44,7 @@ export * from "./overflow.ts" export * from "./types.ts" export const DEFAULT_API_BASE = "https://api.commandcode.ai" -export const COMMAND_CODE_CLI_VERSION = "1.32.2" +export { COMMAND_CODE_CLI_VERSION } const DEFAULT_GENERATE_MAX_TOKENS = 64_000 const DEFAULT_MAX_RETRIES = 0 diff --git a/src/models.ts b/src/models.ts index 0b30631..9b1ec93 100644 --- a/src/models.ts +++ b/src/models.ts @@ -1,6 +1,16 @@ import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises" import { dirname } from "node:path" +import { + MODEL_EFFORTS, + MODEL_INPUT_MODALITIES, + type CommandCodeInputType, + type CommandCodeReasoningEffort, +} from "./commandcode-catalog.ts" + +export { MODEL_EFFORTS, MODEL_INPUT_MODALITIES } +export type { CommandCodeInputType } + export const DEFAULT_PROVIDER_API_BASE = "https://api.commandcode.ai/provider/v1" export const DEFAULT_MODELS_URL = `${DEFAULT_PROVIDER_API_BASE}/models` export const DEFAULT_MODELS_TIMEOUT_MS = 10_000 @@ -9,56 +19,6 @@ const DEFAULT_MAX_OUTPUT_TOKENS = 65_536 const MODEL_CACHE_VERSION = 1 export type CommandCodeApi = "openai-completions" | "anthropic-messages" -export type CommandCodeInputType = "text" | "image" - -/** - * Model input modalities from the command-code@1.32.2 bundled catalog. - * Models omitted here remain text-only so newly discovered IDs never claim - * image support without upstream evidence. - */ -export const MODEL_INPUT_MODALITIES: Readonly> = { - "MiniMaxAI/MiniMax-M3": ["text", "image"], - "Qwen/Qwen3.6-Plus": ["text", "image"], - "Qwen/Qwen3.7-Flash": ["text", "image"], - "Qwen/Qwen3.7-Plus": ["text", "image"], - "Qwen/Qwen3.8-27B": ["text", "image"], - "Qwen/Qwen3.8-Max": ["text", "image"], - "claude-fable-5": ["text", "image"], - "claude-haiku-4-5-20251001": ["text", "image"], - "claude-opus-4-7": ["text", "image"], - "claude-opus-4-8": ["text", "image"], - "claude-opus-5": ["text", "image"], - "claude-sonnet-4-6": ["text", "image"], - "claude-sonnet-5": ["text", "image"], - "deepseek/deepseek-v4-flash-vision-exp": ["text", "image"], - "google/gemini-3.1-flash-lite": ["text", "image"], - "google/gemini-3.5-flash": ["text", "image"], - "google/gemini-3.5-flash-lite": ["text", "image"], - "google/gemini-3.6-flash": ["text", "image"], - "google/gemini-3.7-flash": ["text", "image"], - "gpt-5.3-codex": ["text", "image"], - "gpt-5.4": ["text", "image"], - "gpt-5.4-mini": ["text", "image"], - "gpt-5.5": ["text", "image"], - "gpt-5.6-luna": ["text", "image"], - "gpt-5.6-sol": ["text", "image"], - "gpt-5.6-terra": ["text", "image"], - "meta/muse-spark-1.1": ["text", "image"], - "meta/muse-spark-1.2": ["text", "image"], - "meta/muse-spark-1.2-contributor": ["text", "image"], - "moonshotai/Kimi-K2.5": ["text", "image"], - "moonshotai/Kimi-K2.6": ["text", "image"], - "moonshotai/Kimi-K2.7-Code": ["text", "image"], - "moonshotai/Kimi-K2.7-Code-Highspeed": ["text", "image"], - "moonshotai/Kimi-K3": ["text", "image"], - "sakana/fugu-ultra": ["text", "image"], - "stealth/ox-alpha": ["text", "image"], - "stepfun/Step-3.7-Flash": ["text", "image"], - "thinkingmachines/inkling": ["text", "image"], - "thinkingmachines/inkling-small": ["text", "image"], - "xai/grok-4.5": ["text", "image"], - "xiaomi/mimo-v2.5": ["text", "image"], -} const TEXT_INPUT_ONLY = ["text"] as const @@ -72,49 +32,6 @@ export function modelSupportsImageInput(modelId: string): boolean { export type PiThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" -type CommandCodeReasoningEffort = Exclude - -/** - * Per-model reasoning efforts supported by Command Code's generate endpoint. - * - * The Provider API does not expose reasoning metadata. This is an exact - * snapshot of `reasoningEfforts` from the command-code@1.32.2 model catalog - * (`packages/shared/src/model-catalog.ts`, also published in the generated - * `dist/bundled/command-code-knowledge/reference/models.md`). Models omitted - * here let Command Code choose their reasoning depth, matching the CLI. - */ -export const MODEL_EFFORTS: Readonly> = { - "Qwen/Qwen3.8-27B": ["low", "medium", "xhigh"], - "Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"], - "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-4-7": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-4-8": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-5": ["low", "medium", "high", "xhigh", "max"], - "claude-sonnet-4-6": ["low", "medium", "high", "xhigh", "max"], - "claude-sonnet-5": ["low", "medium", "high", "xhigh", "max"], - "deepseek/deepseek-v4-flash": ["high", "max"], - "deepseek/deepseek-v4-flash-vision-exp": ["high", "max"], - "deepseek/deepseek-v4-pro": ["high", "max"], - "gpt-5.3-codex": ["low", "medium", "high", "xhigh"], - "gpt-5.4": ["low", "medium", "high", "xhigh"], - "gpt-5.4-mini": ["low", "medium", "high"], - "gpt-5.5": ["low", "medium", "high", "xhigh"], - "gpt-5.6-luna": ["low", "medium", "high", "xhigh", "max"], - "gpt-5.6-sol": ["low", "medium", "high", "xhigh", "max"], - "gpt-5.6-terra": ["low", "medium", "high", "xhigh", "max"], - "google/gemini-3.1-flash-lite": ["low", "medium", "high"], - "google/gemini-3.5-flash": ["low", "medium", "high"], - "google/gemini-3.5-flash-lite": ["low", "medium", "high"], - "google/gemini-3.6-flash": ["low", "medium", "high"], - "google/gemini-3.7-flash": ["low", "medium", "high"], - "sakana/fugu-ultra": ["high", "xhigh"], - "stealth/ox-alpha": ["low", "high", "max"], - "xai/grok-4.5": ["low", "medium", "high"], - "xai/grok-4.6": ["low", "medium", "high", "xhigh"], - "zai-org/GLM-5.2": ["high", "max"], - "zai-org/GLM-5.3": ["low", "high", "max"], -} - const PI_THINKING_LEVELS: readonly PiThinkingLevel[] = [ "off", "minimal", diff --git a/tests/test-model-metadata-check.ts b/tests/test-model-metadata-check.ts index 145b298..c2989b6 100644 --- a/tests/test-model-metadata-check.ts +++ b/tests/test-model-metadata-check.ts @@ -8,6 +8,9 @@ import { parseKnownTextOnlyModelIds, parseModelsReference, parsePackageVersion, + renderCommandCodeCatalog, + updateChangelogCatalogVersion, + updateReadmeCatalogVersion, type CommandCodeModelMetadata, } from "../.github/scripts/check-commandcode-model-metadata.ts" @@ -68,6 +71,7 @@ describe("Command Code model metadata checker", () => { const diff = diffModelMetadata(current, upstream) assert.deepEqual(diff, { + versionChanged: false, addedImageModelIds: ["added-image"], removedImageModelIds: ["removed-image"], addedReasoningModelIds: ["added-reasoning"], @@ -77,6 +81,63 @@ describe("Command Code model metadata checker", () => { assert.equal(hasModelMetadataDiff(diff), true) }) + it("reports CLI version drift even when model metadata is unchanged", () => { + const metadata: CommandCodeModelMetadata = { + imageModelIds: ["vision-model"], + reasoningEfforts: { "vision-model": ["low"] }, + } + + const diff = diffModelMetadata(metadata, metadata, "1.32.2", "1.33.0") + + assert.equal(diff.versionChanged, true) + assert.equal(hasModelMetadataDiff(diff), true) + }) + + it("renders a deterministic generated catalog and updates the README version", () => { + assert.equal( + renderCommandCodeCatalog("1.33.0", { + imageModelIds: ["b-model", "a-model"], + reasoningEfforts: { + "b-model": ["high", "max"], + "a-model": ["low"], + }, + }), + `export const COMMAND_CODE_CLI_VERSION = "1.33.0" + +export type CommandCodeInputType = "text" | "image" +export type CommandCodeReasoningEffort = "minimal" | "low" | "medium" | "high" | "xhigh" | "max" + +/** + * Generated from command-code@1.33.0 by \`npm run sync:commandcode-catalog\`. + * Do not edit manually. + */ +export const MODEL_INPUT_MODALITIES: Readonly> = { + "a-model": ["text", "image"], + "b-model": ["text", "image"], +} + +export const MODEL_EFFORTS: Readonly> = { + "a-model": ["low"], + "b-model": ["high", "max"], +} +`, + ) + assert.equal( + updateReadmeCatalogVersion( + "The capability snapshot currently follows `command-code@1.32.2`.", + "1.33.0", + ), + "The capability snapshot currently follows `command-code@1.33.0`.", + ) + assert.equal( + updateChangelogCatalogVersion( + "- Refresh capabilities from `command-code@1.32.2`, including metadata.", + "1.33.0", + ), + "- Refresh capabilities from `command-code@1.33.0`, including metadata.", + ) + }) + it("rejects unexpected upstream structures instead of silently passing", () => { assert.throws(() => parseModelsReference("# no catalog"), /No model rows/) assert.throws( diff --git a/tests/test-models.ts b/tests/test-models.ts index f283400..391e06b 100644 --- a/tests/test-models.ts +++ b/tests/test-models.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { describe, it } from "node:test" +import { COMMAND_CODE_CLI_VERSION } from "../src/commandcode-catalog.ts" import { apiForModelId, baseUrlForModel, @@ -100,7 +101,7 @@ describe("commandCodeModelsFromApiResponse()", () => { ) }) - it("matches command-code@1.32.2 image input capabilities", () => { + it(`uses the command-code@${COMMAND_CODE_CLI_VERSION} image capability catalog`, () => { assert.deepEqual(inputModalitiesForModel("gpt-5.6-luna"), ["text", "image"]) assert.deepEqual(inputModalitiesForModel("meta/muse-spark-1.2"), ["text", "image"]) assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-flash-vision-exp"), [ @@ -117,7 +118,10 @@ describe("commandCodeModelsFromApiResponse()", () => { assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-flash-vision-exp"), true) assert.equal(modelSupportsImageInput("stealth/ox-alpha"), true) assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-pro"), false) - assert.equal(Object.keys(MODEL_INPUT_MODALITIES).length, 41) + assert.ok(Object.keys(MODEL_INPUT_MODALITIES).length > 0) + for (const modalities of Object.values(MODEL_INPUT_MODALITIES)) { + assert.deepEqual(modalities, ["text", "image"]) + } }) it("marks only known reasoning models as reasoning-capable", () => { @@ -133,38 +137,14 @@ describe("commandCodeModelsFromApiResponse()", () => { assert.equal(models[1]?.reasoning, false) }) - it("matches the exact command-code@1.32.2 reasoning effort catalog", () => { - assert.deepEqual(MODEL_EFFORTS, { - "Qwen/Qwen3.8-27B": ["low", "medium", "xhigh"], - "Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"], - "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-4-7": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-4-8": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-5": ["low", "medium", "high", "xhigh", "max"], - "claude-sonnet-4-6": ["low", "medium", "high", "xhigh", "max"], - "claude-sonnet-5": ["low", "medium", "high", "xhigh", "max"], - "deepseek/deepseek-v4-flash": ["high", "max"], - "deepseek/deepseek-v4-flash-vision-exp": ["high", "max"], - "deepseek/deepseek-v4-pro": ["high", "max"], - "gpt-5.3-codex": ["low", "medium", "high", "xhigh"], - "gpt-5.4": ["low", "medium", "high", "xhigh"], - "gpt-5.4-mini": ["low", "medium", "high"], - "gpt-5.5": ["low", "medium", "high", "xhigh"], - "gpt-5.6-luna": ["low", "medium", "high", "xhigh", "max"], - "gpt-5.6-sol": ["low", "medium", "high", "xhigh", "max"], - "gpt-5.6-terra": ["low", "medium", "high", "xhigh", "max"], - "google/gemini-3.1-flash-lite": ["low", "medium", "high"], - "google/gemini-3.5-flash": ["low", "medium", "high"], - "google/gemini-3.5-flash-lite": ["low", "medium", "high"], - "google/gemini-3.6-flash": ["low", "medium", "high"], - "google/gemini-3.7-flash": ["low", "medium", "high"], - "sakana/fugu-ultra": ["high", "xhigh"], - "stealth/ox-alpha": ["low", "high", "max"], - "xai/grok-4.5": ["low", "medium", "high"], - "xai/grok-4.6": ["low", "medium", "high", "xhigh"], - "zai-org/GLM-5.2": ["high", "max"], - "zai-org/GLM-5.3": ["low", "high", "max"], - }) + it(`uses the command-code@${COMMAND_CODE_CLI_VERSION} reasoning effort catalog`, () => { + const validEfforts = new Set(["minimal", "low", "medium", "high", "xhigh", "max"]) + assert.ok(Object.keys(MODEL_EFFORTS).length > 0) + for (const efforts of Object.values(MODEL_EFFORTS)) { + assert.ok(efforts.length > 0) + assert.equal(new Set(efforts).size, efforts.length) + assert.ok(efforts.every((effort) => validEfforts.has(effort))) + } }) it("builds separate canonical pi and OMP metadata", () => { diff --git a/tests/test-stream.ts b/tests/test-stream.ts index 6346416..a015e75 100644 --- a/tests/test-stream.ts +++ b/tests/test-stream.ts @@ -6,6 +6,7 @@ import assert from "node:assert/strict" import { after, before, beforeEach, describe, it } from "node:test" +import { COMMAND_CODE_CLI_VERSION } from "../src/commandcode-catalog.ts" import type { AssistantMessageEvent } from "../src/core.ts" import { MODEL_EFFORTS, thinkingLevelMapForEfforts } from "../src/models.ts" import { @@ -569,7 +570,7 @@ describe("streamCommandCode — request serialization", () => { const headers = server.lastRequestHeaders() assert.equal(headers.authorization, "Bearer mock-key") - assert.equal(headers["x-command-code-version"], "1.32.2") + assert.equal(headers["x-command-code-version"], COMMAND_CODE_CLI_VERSION) assert.equal(headers["x-project-slug"], "repo") assert.equal(headers["x-taste-learning"], "true") assert.equal(headers["x-co-flag"], "false") From c3a383447cbe96f04d10c2483e1566d7ba318485 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 15:08:29 +0200 Subject: [PATCH 28/40] ci(models): open PRs for Command Code catalog updates --- .github/workflows/model-metadata.yml | 69 ++++++++++++++++++++++++---- CHANGELOG.md | 2 +- README.md | 2 +- 3 files changed, 62 insertions(+), 11 deletions(-) diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml index 809eef8..196b7b5 100644 --- a/.github/workflows/model-metadata.yml +++ b/.github/workflows/model-metadata.yml @@ -1,4 +1,4 @@ -name: Command Code model metadata +name: Command Code catalog sync on: pull_request: @@ -6,6 +6,7 @@ on: paths: - ".github/scripts/check-commandcode-model-metadata.ts" - ".github/workflows/model-metadata.yml" + - "src/commandcode-catalog.ts" - "src/core.ts" - "src/models.ts" - "tests/test-model-metadata-check.ts" @@ -13,17 +14,17 @@ on: - cron: "17 6 * * *" workflow_dispatch: -permissions: - contents: read - concurrency: - group: commandcode-model-metadata + group: commandcode-catalog-${{ github.event_name == 'pull_request' && github.event.pull_request.number || 'sync' }} cancel-in-progress: true jobs: check: + if: github.event_name == 'pull_request' runs-on: ubuntu-latest timeout-minutes: 10 + permissions: + contents: read steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -32,8 +33,58 @@ jobs: cache: npm registry-url: https://registry.npmjs.org - run: npm ci - - name: Compare static metadata with the latest Command Code CLI - run: npm run check:model-metadata | tee model-metadata-report.md - - name: Publish metadata report + - name: Compare with the latest Command Code CLI + run: npm run check:commandcode-catalog | tee commandcode-catalog-report.md + - name: Publish catalog report if: always() - run: cat model-metadata-report.md >> "$GITHUB_STEP_SUMMARY" + run: cat commandcode-catalog-report.md >> "$GITHUB_STEP_SUMMARY" + + sync: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + registry-url: https://registry.npmjs.org + - run: npm ci + - name: Synchronize with the latest Command Code CLI + run: npm run sync:commandcode-catalog | tee commandcode-catalog-report.md + - name: Format and verify synchronized files + run: | + npm run format -- src/commandcode-catalog.ts README.md CHANGELOG.md + npm run typecheck + npm run test:models + npm run format:check + git diff --check + - name: Publish catalog report + if: always() + run: cat commandcode-catalog-report.md >> "$GITHUB_STEP_SUMMARY" + - name: Create or update synchronization PR + uses: peter-evans/create-pull-request@v8 + with: + branch: automation/commandcode-catalog + delete-branch: true + commit-message: "chore(models): sync Command Code catalog" + title: "chore(models): sync Command Code catalog" + body: | + Automated synchronization with the latest published `command-code` CLI package. + + This updates only machine-readable compatibility metadata: + - CLI version used in the `x-command-code-version` header + - image-input capabilities + - supported reasoning efforts + - documented catalog snapshot version + + Pricing remains review-only because CLI documentation does not represent every pricing tier and temporary promotion used by the provider. + assignees: patlux + add-paths: | + src/commandcode-catalog.ts + README.md + CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c4e2930..5730962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Unreleased -- Add a daily GitHub Actions check that compares static image and reasoning metadata with the latest published Command Code CLI catalog. +- Add a daily GitHub Actions synchronization job that opens or updates a pull request for CLI version, image capability, and reasoning-effort changes in the latest published Command Code catalog. - Refresh static model capabilities from `command-code@1.32.2`, including new image and reasoning metadata. - Add `/commandcode-quota` with live credits, plan, usage totals, and rolling-limit diagnostics from Command Code's alpha usage endpoints. - Add `zai-org/GLM-5.3` with its verified reasoning efforts and display pricing. diff --git a/README.md b/README.md index 7657981..83c8295 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ The following environment variables are intended for tests, local mocks, and com ## Image input -The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed. A daily GitHub Actions check compares the static image and reasoning metadata with the latest published CLI package and reports any drift. +The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed. A daily GitHub Actions job synchronizes the CLI version, image capabilities, and reasoning efforts with the latest published CLI package and opens or updates a reviewable pull request when they change. Pricing remains manually reviewed because the CLI catalog does not expose every pricing tier and temporary promotion used by the provider. For vision-capable models, Pi's native provider adapters forward image blocks from user messages and tool results using the documented OpenAI or Anthropic message schema. Unknown and text-only models remain marked text-only in Pi. From 41e98a1d67a7495ecb3126e4c73fbfebea4343ba Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 15:09:54 +0200 Subject: [PATCH 29/40] fix(models): keep generated sync scope reviewable --- .github/scripts/check-commandcode-model-metadata.ts | 11 +---------- .github/workflows/model-metadata.yml | 3 +-- tests/test-model-metadata-check.ts | 8 -------- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/.github/scripts/check-commandcode-model-metadata.ts b/.github/scripts/check-commandcode-model-metadata.ts index 4cfa4e2..292a4d3 100644 --- a/.github/scripts/check-commandcode-model-metadata.ts +++ b/.github/scripts/check-commandcode-model-metadata.ts @@ -18,7 +18,6 @@ const TEXT_ONLY_MARKER = ',__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' const VALID_EFFORTS = new Set(["minimal", "low", "medium", "high", "xhigh", "max"]) const CATALOG_SOURCE_PATH = new URL("../../src/commandcode-catalog.ts", import.meta.url) const README_PATH = new URL("../../README.md", import.meta.url) -const CHANGELOG_PATH = new URL("../../CHANGELOG.md", import.meta.url) export interface CommandCodeModelMetadata { imageModelIds: readonly string[] @@ -254,22 +253,14 @@ export function updateReadmeCatalogVersion(readme: string, packageVersion: strin return updateDocumentedCatalogVersion(readme, packageVersion, "README") } -export function updateChangelogCatalogVersion(changelog: string, packageVersion: string): string { - return updateDocumentedCatalogVersion(changelog, packageVersion, "changelog") -} - async function writeSynchronizedCatalog( packageVersion: string, metadata: CommandCodeModelMetadata, ): Promise { - const [readme, changelog] = await Promise.all([ - readFile(README_PATH, "utf-8"), - readFile(CHANGELOG_PATH, "utf-8"), - ]) + const readme = await readFile(README_PATH, "utf-8") await Promise.all([ writeFile(CATALOG_SOURCE_PATH, renderCommandCodeCatalog(packageVersion, metadata), "utf-8"), writeFile(README_PATH, updateReadmeCatalogVersion(readme, packageVersion), "utf-8"), - writeFile(CHANGELOG_PATH, updateChangelogCatalogVersion(changelog, packageVersion), "utf-8"), ]) } diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml index 196b7b5..de5a26f 100644 --- a/.github/workflows/model-metadata.yml +++ b/.github/workflows/model-metadata.yml @@ -58,7 +58,7 @@ jobs: run: npm run sync:commandcode-catalog | tee commandcode-catalog-report.md - name: Format and verify synchronized files run: | - npm run format -- src/commandcode-catalog.ts README.md CHANGELOG.md + npm run format -- src/commandcode-catalog.ts README.md npm run typecheck npm run test:models npm run format:check @@ -87,4 +87,3 @@ jobs: add-paths: | src/commandcode-catalog.ts README.md - CHANGELOG.md diff --git a/tests/test-model-metadata-check.ts b/tests/test-model-metadata-check.ts index c2989b6..ac1b61f 100644 --- a/tests/test-model-metadata-check.ts +++ b/tests/test-model-metadata-check.ts @@ -9,7 +9,6 @@ import { parseModelsReference, parsePackageVersion, renderCommandCodeCatalog, - updateChangelogCatalogVersion, updateReadmeCatalogVersion, type CommandCodeModelMetadata, } from "../.github/scripts/check-commandcode-model-metadata.ts" @@ -129,13 +128,6 @@ export const MODEL_EFFORTS: Readonly { From 349e50f829cba0280f5b552c7b0b9e4ae4a7bdfe Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 15:54:09 +0200 Subject: [PATCH 30/40] fix(models): align Command Code catalog metadata --- .../check-commandcode-model-metadata.ts | 149 ++++++++++++++++-- .github/workflows/model-metadata.yml | 3 +- src/commandcode-catalog.ts | 57 +++++++ src/models.ts | 39 +++-- src/pricing.ts | 44 +++++- tests/fixtures/commandcode-model-ids.json | 12 +- tests/fixtures/commandcode-pricing.json | 31 ++-- tests/test-model-metadata-check.ts | 47 ++++-- tests/test-models.ts | 47 +++++- tests/test-pricing.ts | 33 +++- 10 files changed, 399 insertions(+), 63 deletions(-) diff --git a/.github/scripts/check-commandcode-model-metadata.ts b/.github/scripts/check-commandcode-model-metadata.ts index 292a4d3..b83c386 100644 --- a/.github/scripts/check-commandcode-model-metadata.ts +++ b/.github/scripts/check-commandcode-model-metadata.ts @@ -9,6 +9,8 @@ import { COMMAND_CODE_CLI_VERSION, MODEL_EFFORTS, MODEL_INPUT_MODALITIES, + MODEL_MAX_OUTPUT_TOKENS, + MODEL_REASONING, } from "../../src/commandcode-catalog.ts" const execFileAsync = promisify(execFile) @@ -21,7 +23,9 @@ const README_PATH = new URL("../../README.md", import.meta.url) export interface CommandCodeModelMetadata { imageModelIds: readonly string[] + reasoningModelIds: readonly string[] reasoningEfforts: Readonly> + maxOutputTokens: Readonly> } export interface ModelMetadataDiff { @@ -30,7 +34,12 @@ export interface ModelMetadataDiff { removedImageModelIds: readonly string[] addedReasoningModelIds: readonly string[] removedReasoningModelIds: readonly string[] - changedReasoningModelIds: readonly string[] + addedEffortModelIds: readonly string[] + removedEffortModelIds: readonly string[] + changedEffortModelIds: readonly string[] + addedMaxOutputModelIds: readonly string[] + removedMaxOutputModelIds: readonly string[] + changedMaxOutputModelIds: readonly string[] } interface PackedPackage { @@ -123,27 +132,93 @@ export function parseKnownTextOnlyModelIds(bundle: string): readonly string[] { return sorted(new Set(parsed)) } +function modelObject(bundle: string, modelId: string): string { + const start = bundle.indexOf(`{id:${JSON.stringify(modelId)},inputModalities:`) + if (start < 0) throw new Error(`Could not find model metadata for ${modelId}`) + + let depth = 0 + let quote = "" + let escaped = false + for (let index = start; index < bundle.length; index += 1) { + const character = bundle[index] ?? "" + if (quote) { + if (escaped) escaped = false + else if (character === "\\") escaped = true + else if (character === quote) quote = "" + continue + } + if (character === '"' || character === "'" || character === "`") { + quote = character + continue + } + if (character === "{") depth += 1 + else if (character === "}" && --depth === 0) return bundle.slice(start, index + 1) + } + + throw new Error(`Unterminated model metadata for ${modelId}`) +} + +export function parseBundleModelCapabilities( + bundle: string, + modelIds: readonly string[], +): { + reasoningModelIds: readonly string[] + maxOutputTokens: Readonly> +} { + const reasoningModelIds: string[] = [] + const maxOutputTokens: Record = {} + + for (const modelId of modelIds) { + const entry = modelObject(bundle, modelId) + if (entry.includes("reasoning:!0") || entry.includes("reasoningEfforts:[")) { + reasoningModelIds.push(modelId) + } + const maxOutput = /maxOutputTokens:([^,}]+)/.exec(entry)?.[1] + if (maxOutput) { + const value = Number(maxOutput) + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`Unexpected max output tokens for ${modelId}: ${maxOutput}`) + } + maxOutputTokens[modelId] = value + } + } + + return { + reasoningModelIds: sorted(reasoningModelIds), + maxOutputTokens: Object.fromEntries( + Object.entries(maxOutputTokens).sort(([left], [right]) => left.localeCompare(right)), + ), + } +} + export function commandCodeModelMetadataFromContents( modelsReference: string, cliBundle: string, ): CommandCodeModelMetadata { const reference = parseModelsReference(modelsReference) const textOnlyModelIds = new Set(parseKnownTextOnlyModelIds(cliBundle)) + const capabilities = parseBundleModelCapabilities(cliBundle, reference.modelIds) return { imageModelIds: reference.modelIds.filter((modelId) => !textOnlyModelIds.has(modelId)), + reasoningModelIds: capabilities.reasoningModelIds, reasoningEfforts: reference.reasoningEfforts, + maxOutputTokens: capabilities.maxOutputTokens, } } export function currentModelMetadata(): CommandCodeModelMetadata { return { imageModelIds: sorted(Object.keys(MODEL_INPUT_MODALITIES)), + reasoningModelIds: sorted(Object.keys(MODEL_REASONING)), reasoningEfforts: Object.fromEntries( Object.entries(MODEL_EFFORTS) .sort(([left], [right]) => left.localeCompare(right)) .map(([modelId, efforts]) => [modelId, [...efforts]]), ), + maxOutputTokens: Object.fromEntries( + Object.entries(MODEL_MAX_OUTPUT_TOKENS).sort(([left], [right]) => left.localeCompare(right)), + ), } } @@ -155,10 +230,16 @@ export function diffModelMetadata( ): ModelMetadataDiff { const currentImages = new Set(current.imageModelIds) const upstreamImages = new Set(upstream.imageModelIds) - const currentReasoningIds = Object.keys(current.reasoningEfforts) - const upstreamReasoningIds = Object.keys(upstream.reasoningEfforts) - const currentReasoningSet = new Set(currentReasoningIds) - const upstreamReasoningSet = new Set(upstreamReasoningIds) + const currentReasoning = new Set(current.reasoningModelIds) + const upstreamReasoning = new Set(upstream.reasoningModelIds) + const currentEffortIds = Object.keys(current.reasoningEfforts) + const upstreamEffortIds = Object.keys(upstream.reasoningEfforts) + const currentEffortSet = new Set(currentEffortIds) + const upstreamEffortSet = new Set(upstreamEffortIds) + const currentMaxOutputIds = Object.keys(current.maxOutputTokens) + const upstreamMaxOutputIds = Object.keys(upstream.maxOutputTokens) + const currentMaxOutputSet = new Set(currentMaxOutputIds) + const upstreamMaxOutputSet = new Set(upstreamMaxOutputIds) return { versionChanged: currentVersion !== upstreamVersion, @@ -169,19 +250,38 @@ export function diffModelMetadata( current.imageModelIds.filter((modelId) => !upstreamImages.has(modelId)), ), addedReasoningModelIds: sorted( - upstreamReasoningIds.filter((modelId) => !currentReasoningSet.has(modelId)), + upstream.reasoningModelIds.filter((modelId) => !currentReasoning.has(modelId)), ), removedReasoningModelIds: sorted( - currentReasoningIds.filter((modelId) => !upstreamReasoningSet.has(modelId)), + current.reasoningModelIds.filter((modelId) => !upstreamReasoning.has(modelId)), ), - changedReasoningModelIds: sorted( - upstreamReasoningIds.filter( + addedEffortModelIds: sorted( + upstreamEffortIds.filter((modelId) => !currentEffortSet.has(modelId)), + ), + removedEffortModelIds: sorted( + currentEffortIds.filter((modelId) => !upstreamEffortSet.has(modelId)), + ), + changedEffortModelIds: sorted( + upstreamEffortIds.filter( (modelId) => - currentReasoningSet.has(modelId) && + currentEffortSet.has(modelId) && JSON.stringify(current.reasoningEfforts[modelId]) !== JSON.stringify(upstream.reasoningEfforts[modelId]), ), ), + addedMaxOutputModelIds: sorted( + upstreamMaxOutputIds.filter((modelId) => !currentMaxOutputSet.has(modelId)), + ), + removedMaxOutputModelIds: sorted( + currentMaxOutputIds.filter((modelId) => !upstreamMaxOutputSet.has(modelId)), + ), + changedMaxOutputModelIds: sorted( + upstreamMaxOutputIds.filter( + (modelId) => + currentMaxOutputSet.has(modelId) && + current.maxOutputTokens[modelId] !== upstream.maxOutputTokens[modelId], + ), + ), } } @@ -229,14 +329,24 @@ export function renderCommandCodeCatalog( const imageEntries = sorted(metadata.imageModelIds) .map((modelId) => ` ${quoted(modelId)}: ["text", "image"],`) .join("\n") - const reasoningEntries = recordEntries(metadata.reasoningEfforts) + const reasoningEntries = sorted(metadata.reasoningModelIds) + .map((modelId) => ` ${quoted(modelId)}: true,`) + .join("\n") + const effortEntries = recordEntries(metadata.reasoningEfforts) .map( ([modelId, efforts]) => ` ${quoted(modelId)}: [${efforts.map((effort) => quoted(effort)).join(", ")}],`, ) .join("\n") + const maxOutputEntries = Object.entries(metadata.maxOutputTokens) + .sort(([left], [right]) => left.localeCompare(right)) + .map( + ([modelId, value]) => + ` ${quoted(modelId)}: ${value.toLocaleString("en-US").replaceAll(",", "_")},`, + ) + .join("\n") - return `export const COMMAND_CODE_CLI_VERSION = ${quoted(packageVersion)}\n\nexport type CommandCodeInputType = "text" | "image"\nexport type CommandCodeReasoningEffort = "minimal" | "low" | "medium" | "high" | "xhigh" | "max"\n\n/**\n * Generated from command-code@${packageVersion} by \`npm run sync:commandcode-catalog\`.\n * Do not edit manually.\n */\nexport const MODEL_INPUT_MODALITIES: Readonly> = {\n${imageEntries}\n}\n\nexport const MODEL_EFFORTS: Readonly> = {\n${reasoningEntries}\n}\n` + return `export const COMMAND_CODE_CLI_VERSION = ${quoted(packageVersion)}\n\nexport type CommandCodeInputType = "text" | "image"\nexport type CommandCodeReasoningEffort = "minimal" | "low" | "medium" | "high" | "xhigh" | "max"\n\n/**\n * Generated from command-code@${packageVersion} by \`npm run sync:commandcode-catalog\`.\n * Do not edit manually.\n */\nexport const MODEL_INPUT_MODALITIES: Readonly> = {\n${imageEntries}\n}\n\nexport const MODEL_REASONING: Readonly> = {\n${reasoningEntries}\n}\n\nexport const MODEL_EFFORTS: Readonly> = {\n${effortEntries}\n}\n\nexport const MODEL_MAX_OUTPUT_TOKENS: Readonly> = {\n${maxOutputEntries}\n}\n` } function updateDocumentedCatalogVersion( @@ -279,16 +389,23 @@ function metadataReport( `- Repository snapshot: \`command-code@${COMMAND_CODE_CLI_VERSION}\``, `- Inspected package: \`command-code@${packageVersion}\``, `- Image-capable models: ${current.imageModelIds.length} repository / ${upstream.imageModelIds.length} upstream`, - `- Reasoning models: ${Object.keys(current.reasoningEfforts).length} repository / ${Object.keys(upstream.reasoningEfforts).length} upstream`, + `- Reasoning models: ${current.reasoningModelIds.length} repository / ${upstream.reasoningModelIds.length} upstream`, + `- Models with selectable efforts: ${Object.keys(current.reasoningEfforts).length} repository / ${Object.keys(upstream.reasoningEfforts).length} upstream`, + `- Model-specific output limits: ${Object.keys(current.maxOutputTokens).length} repository / ${Object.keys(upstream.maxOutputTokens).length} upstream`, "", "| Change | Models |", "| --- | --- |", `| CLI version | ${diff.versionChanged ? `\`${COMMAND_CODE_CLI_VERSION}\` → \`${packageVersion}\`` : "Current"} |`, `| New image support | ${formatList(diff.addedImageModelIds)} |`, `| Removed image support | ${formatList(diff.removedImageModelIds)} |`, - `| New reasoning metadata | ${formatList(diff.addedReasoningModelIds)} |`, - `| Removed reasoning metadata | ${formatList(diff.removedReasoningModelIds)} |`, - `| Changed reasoning efforts | ${formatReasoningChanges(diff.changedReasoningModelIds, current, upstream)} |`, + `| New reasoning models | ${formatList(diff.addedReasoningModelIds)} |`, + `| Removed reasoning models | ${formatList(diff.removedReasoningModelIds)} |`, + `| New effort metadata | ${formatList(diff.addedEffortModelIds)} |`, + `| Removed effort metadata | ${formatList(diff.removedEffortModelIds)} |`, + `| Changed reasoning efforts | ${formatReasoningChanges(diff.changedEffortModelIds, current, upstream)} |`, + `| New output limits | ${formatList(diff.addedMaxOutputModelIds)} |`, + `| Removed output limits | ${formatList(diff.removedMaxOutputModelIds)} |`, + `| Changed output limits | ${formatList(diff.changedMaxOutputModelIds)} |`, "", ].join("\n") } diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml index de5a26f..88c9ca5 100644 --- a/.github/workflows/model-metadata.yml +++ b/.github/workflows/model-metadata.yml @@ -79,7 +79,8 @@ jobs: This updates only machine-readable compatibility metadata: - CLI version used in the `x-command-code-version` header - image-input capabilities - - supported reasoning efforts + - reasoning capability and selectable effort levels + - model-specific maximum output limits - documented catalog snapshot version Pricing remains review-only because CLI documentation does not represent every pricing tier and temporary promotion used by the provider. diff --git a/src/commandcode-catalog.ts b/src/commandcode-catalog.ts index 4249514..6840f5f 100644 --- a/src/commandcode-catalog.ts +++ b/src/commandcode-catalog.ts @@ -51,6 +51,57 @@ export const MODEL_INPUT_MODALITIES: Readonly> = { + "claude-fable-5": true, + "claude-opus-4-7": true, + "claude-opus-4-8": true, + "claude-opus-5": true, + "claude-sonnet-4-6": true, + "claude-sonnet-5": true, + "deepseek/deepseek-v4-flash": true, + "deepseek/deepseek-v4-flash-vision-exp": true, + "deepseek/deepseek-v4-pro": true, + "google/gemini-3.1-flash-lite": true, + "google/gemini-3.5-flash": true, + "google/gemini-3.5-flash-lite": true, + "google/gemini-3.6-flash": true, + "google/gemini-3.7-flash": true, + "gpt-5.3-codex": true, + "gpt-5.4": true, + "gpt-5.4-mini": true, + "gpt-5.5": true, + "gpt-5.6-luna": true, + "gpt-5.6-sol": true, + "gpt-5.6-terra": true, + "meta/muse-spark-1.1": true, + "meta/muse-spark-1.2": true, + "meta/muse-spark-1.2-contributor": true, + "MiniMaxAI/MiniMax-M3": true, + "moonshotai/Kimi-K2.7-Code": true, + "moonshotai/Kimi-K2.7-Code-Highspeed": true, + "moonshotai/Kimi-K3": true, + "nvidia/nemotron-3-ultra-550b-a55b": true, + "poolside/laguna-s-2.1-free": true, + "Qwen/Qwen3.6-Max-Preview": true, + "Qwen/Qwen3.6-Plus": true, + "Qwen/Qwen3.7-Flash": true, + "Qwen/Qwen3.7-Max": true, + "Qwen/Qwen3.7-Plus": true, + "Qwen/Qwen3.8-27B": true, + "Qwen/Qwen3.8-Max": true, + "sakana/fugu-ultra": true, + "stealth/ox-alpha": true, + "stepfun/Step-3.5-Flash": true, + "stepfun/Step-3.7-Flash": true, + "tencent/hy3-paid": true, + "thinkingmachines/inkling": true, + "thinkingmachines/inkling-small": true, + "xai/grok-4.5": true, + "xai/grok-4.6": true, + "zai-org/GLM-5.2": true, + "zai-org/GLM-5.3": true, +} + export const MODEL_EFFORTS: Readonly> = { "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], "claude-opus-4-7": ["low", "medium", "high", "xhigh", "max"], @@ -82,3 +133,9 @@ export const MODEL_EFFORTS: Readonly> = { + "poolside/laguna-s-2.1-free": 32_768, + "Qwen/Qwen3.8-27B": 32_768, + "stealth/ox-alpha": 131_072, +} diff --git a/src/models.ts b/src/models.ts index 9b1ec93..be7c674 100644 --- a/src/models.ts +++ b/src/models.ts @@ -4,11 +4,13 @@ import { dirname } from "node:path" import { MODEL_EFFORTS, MODEL_INPUT_MODALITIES, + MODEL_MAX_OUTPUT_TOKENS, + MODEL_REASONING, type CommandCodeInputType, type CommandCodeReasoningEffort, } from "./commandcode-catalog.ts" -export { MODEL_EFFORTS, MODEL_INPUT_MODALITIES } +export { MODEL_EFFORTS, MODEL_INPUT_MODALITIES, MODEL_MAX_OUTPUT_TOKENS, MODEL_REASONING } export type { CommandCodeInputType } export const DEFAULT_PROVIDER_API_BASE = "https://api.commandcode.ai/provider/v1" @@ -55,7 +57,7 @@ export function thinkingLevelMapForEfforts( export interface ThinkingMetadata { thinkingLevelMap: Partial> - thinking: { + thinking?: { mode: "effort" effortMap: Partial> efforts: readonly CommandCodeReasoningEffort[] @@ -64,19 +66,26 @@ export interface ThinkingMetadata { export function thinkingMetadataForModel(modelId: string): ThinkingMetadata | undefined { const efforts = MODEL_EFFORTS[modelId] - if (!efforts) return undefined - return { - thinkingLevelMap: thinkingLevelMapForEfforts(efforts), - thinking: { - mode: "effort", - effortMap: Object.fromEntries(efforts.map((effort) => [effort, effort])), - efforts, - }, + if (efforts) { + return { + thinkingLevelMap: thinkingLevelMapForEfforts(efforts), + thinking: { + mode: "effort", + effortMap: Object.fromEntries(efforts.map((effort) => [effort, effort])), + efforts, + }, + } } + if (!isReasoningModel(modelId)) return undefined + return { thinkingLevelMap: thinkingLevelMapForEfforts([]) } } function isReasoningModel(modelId: string): boolean { - return MODEL_EFFORTS[modelId] !== undefined + return MODEL_REASONING[modelId] === true +} + +function maxOutputTokensForModel(modelId: string, contextLength: number): number { + return Math.min(contextLength, MODEL_MAX_OUTPUT_TOKENS[modelId] ?? DEFAULT_MAX_OUTPUT_TOKENS) } interface ApiModel { @@ -162,13 +171,15 @@ function parseCachedModel(value: unknown): CommandCodeModel { const id = stringField(value, "id") booleanField(value, "reasoning") + positiveNumberField(value, "maxTokens") + const contextWindow = positiveNumberField(value, "contextWindow") return { id, name: stringField(value, "name"), api: apiForModelId(id), reasoning: isReasoningModel(id), - contextWindow: positiveNumberField(value, "contextWindow"), - maxTokens: positiveNumberField(value, "maxTokens"), + contextWindow, + maxTokens: maxOutputTokensForModel(id, contextWindow), } } @@ -273,7 +284,7 @@ export function commandCodeModelsFromApiResponse(value: unknown): readonly Comma api: apiForModelId(model.id), reasoning: isReasoningModel(model.id), contextWindow: model.contextLength, - maxTokens: Math.min(model.contextLength, DEFAULT_MAX_OUTPUT_TOKENS), + maxTokens: maxOutputTokensForModel(model.id, model.contextLength), })) } diff --git a/src/pricing.ts b/src/pricing.ts index 5f3a951..fddd2a3 100644 --- a/src/pricing.ts +++ b/src/pricing.ts @@ -20,7 +20,7 @@ export interface TemporaryPricing { } export const PRICING_SOURCE_URL = "https://commandcode.ai/docs/resources/pricing-limits" -export const PRICING_LAST_VERIFIED = "2026-08-22" +export const PRICING_LAST_VERIFIED = "2026-08-25" export const ZERO_MODEL_COST: CommandCodeModelCost = { input: 0, @@ -40,7 +40,7 @@ export const ZERO_MODEL_COST: CommandCodeModelCost = { export const MODEL_COSTS: Readonly> = { // Free models "poolside/laguna-s-2.1-free": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - "inclusionai/ling-3.0-flash-free": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + "stealth/ox-alpha": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, // Open and open-weight models "tencent/hy3-paid": { input: 0.14, output: 0.58, cacheRead: 0.035, cacheWrite: 0 }, @@ -76,7 +76,14 @@ export const MODEL_COSTS: Readonly> = { cacheRead: 0.007, cacheWrite: 0, }, + "deepseek/deepseek-v4-flash-vision-exp": { + input: 0.22, + output: 0.66, + cacheRead: 0.007, + cacheWrite: 0, + }, "Qwen/Qwen3.8-Max": { input: 2, output: 6, cacheRead: 0.25, cacheWrite: 2.5 }, + "Qwen/Qwen3.8-27B": { input: 0.4, output: 3, cacheRead: 0.04, cacheWrite: 0 }, "Qwen/Qwen3.7-Max": { input: 2.5, output: 7.5, cacheRead: 0.5, cacheWrite: 3.13 }, "Qwen/Qwen3.7-Plus": { input: 0.4, @@ -142,6 +149,13 @@ export const MODEL_COSTS: Readonly> = { cacheWrite: 0, }, "meta/muse-spark-1.1": { input: 1.25, output: 4.25, cacheRead: 0.15, cacheWrite: 0 }, + "meta/muse-spark-1.2": { input: 1.25, output: 4.25, cacheRead: 0.15, cacheWrite: 0 }, + "meta/muse-spark-1.2-contributor": { + input: 0.1, + output: 0.2, + cacheRead: 0.002, + cacheWrite: 0, + }, // Anthropic // Introductory pricing through 2026-08-31. @@ -168,6 +182,12 @@ export const MODEL_COSTS: Readonly> = { "gpt-5.4-mini": { input: 0.75, output: 4.5, cacheRead: 0.075, cacheWrite: 0 }, // Google and xAI + "google/gemini-3.7-flash": { + input: 0.75, + output: 3.75, + cacheRead: 0.075, + cacheWrite: 0.04167, + }, "google/gemini-3.6-flash": { input: 1.5, output: 7.5, cacheRead: 0.15, cacheWrite: 0 }, "google/gemini-3.5-flash": { input: 1.5, output: 9, cacheRead: 0.15, cacheWrite: 0 }, "google/gemini-3.5-flash-lite": { @@ -183,6 +203,21 @@ export const MODEL_COSTS: Readonly> = { cacheWrite: 0, }, "xai/grok-4.5": { input: 2, output: 6, cacheRead: 0.5, cacheWrite: 0 }, + "xai/grok-4.6": { + input: 2, + output: 6, + cacheRead: 0.5, + cacheWrite: 0, + tiers: [ + { + inputTokensAbove: 200_000, + input: 4, + output: 12, + cacheRead: 1, + cacheWrite: 0, + }, + ], + }, } export const TEMPORARY_PRICING: readonly TemporaryPricing[] = [ @@ -191,4 +226,9 @@ export const TEMPORARY_PRICING: readonly TemporaryPricing[] = [ expiresOn: "2026-08-31", description: "introductory pricing", }, + { + models: ["google/gemini-3.7-flash"], + expiresOn: "2026-12-31", + description: "50% promotional pricing", + }, ] diff --git a/tests/fixtures/commandcode-model-ids.json b/tests/fixtures/commandcode-model-ids.json index 1b9898a..2a69168 100644 --- a/tests/fixtures/commandcode-model-ids.json +++ b/tests/fixtures/commandcode-model-ids.json @@ -1,5 +1,5 @@ { - "fetchedAt": "2026-08-22T21:19:37.782Z", + "fetchedAt": "2026-08-25T13:32:11.631Z", "source": "https://api.commandcode.ai/provider/v1/models", "modelIds": [ "claude-sonnet-5", @@ -18,6 +18,7 @@ "gpt-5.4-mini", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-flash", + "deepseek/deepseek-v4-flash-vision-exp", "moonshotai/Kimi-K3", "moonshotai/Kimi-K2.7-Code", "moonshotai/Kimi-K2.7-Code-Highspeed", @@ -34,6 +35,7 @@ "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.5", "Qwen/Qwen3.8-Max", + "Qwen/Qwen3.8-27B", "Qwen/Qwen3.7-Max", "Qwen/Qwen3.7-Plus", "Qwen/Qwen3.7-Flash", @@ -42,6 +44,7 @@ "stepfun/Step-3.7-Flash", "stepfun/Step-3.5-Flash", "tencent/hy3-paid", + "google/gemini-3.7-flash", "google/gemini-3.6-flash", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", @@ -50,9 +53,12 @@ "nvidia/nemotron-3-ultra-550b-a55b", "thinkingmachines/inkling", "thinkingmachines/inkling-small", + "stealth/ox-alpha", "poolside/laguna-s-2.1-free", - "inclusionai/ling-3.0-flash-free", "meta/muse-spark-1.1", - "xai/grok-4.5" + "meta/muse-spark-1.2", + "meta/muse-spark-1.2-contributor", + "xai/grok-4.5", + "xai/grok-4.6" ] } diff --git a/tests/fixtures/commandcode-pricing.json b/tests/fixtures/commandcode-pricing.json index 5b2070c..c65ae0c 100644 --- a/tests/fixtures/commandcode-pricing.json +++ b/tests/fixtures/commandcode-pricing.json @@ -1,5 +1,5 @@ { - "verifiedAt": "2026-08-22", + "verifiedAt": "2026-08-25", "source": "https://commandcode.ai/docs/resources/pricing-limits", "tierPolicy": "Use request-wide input tiers; the highest threshold exceeded by input plus cache tokens applies to the full request.", "tiers": { @@ -7,12 +7,13 @@ "Qwen/Qwen3.7-Flash": [ [32000, 0.1, 0.4, 0.02, 0.125], [256000, 0.2, 0.8, 0.04, 0.25] - ] + ], + "xai/grok-4.6": [[200000, 4, 12, 1, 0]] }, "costs": { - "poolside/laguna-s-2.1-free": [0, 0, 0, 0], - "inclusionai/ling-3.0-flash-free": [0, 0, 0, 0], - "tencent/hy3-paid": [0.14, 0.58, 0.035, 0], + "deepseek/deepseek-v4-pro": [0.66, 1.98, 0.022, 0], + "deepseek/deepseek-v4-flash": [0.22, 0.66, 0.007, 0], + "deepseek/deepseek-v4-flash-vision-exp": [0.22, 0.66, 0.007, 0], "moonshotai/Kimi-K3": [3, 15, 0.3, 0], "moonshotai/Kimi-K2.7-Code": [0.95, 4, 0.19, 0], "moonshotai/Kimi-K2.7-Code-Highspeed": [1.9, 8, 0.38, 0], @@ -26,9 +27,10 @@ "MiniMaxAI/MiniMax-M3": [0.3, 1.2, 0.06, 0], "MiniMaxAI/MiniMax-M2.7": [0.3, 1.2, 0.06, 0], "MiniMaxAI/MiniMax-M2.5": [0.3, 1.2, 0.03, 0], - "deepseek/deepseek-v4-pro": [0.66, 1.98, 0.022, 0], - "deepseek/deepseek-v4-flash": [0.22, 0.66, 0.007, 0], + "xiaomi/mimo-v2.5-pro": [0.435, 0.87, 0.0036, 0], + "xiaomi/mimo-v2.5": [0.14, 0.28, 0.0028, 0], "Qwen/Qwen3.8-Max": [2, 6, 0.25, 2.5], + "Qwen/Qwen3.8-27B": [0.4, 3, 0.04, 0], "Qwen/Qwen3.7-Max": [2.5, 7.5, 0.5, 3.13], "Qwen/Qwen3.7-Plus": [0.4, 1.6, 0.08, 0.5], "Qwen/Qwen3.7-Flash": [0.03, 0.13, 0.006, 0.038], @@ -36,13 +38,12 @@ "Qwen/Qwen3.6-Plus": [0.5, 3, 0.1, 0], "stepfun/Step-3.7-Flash": [0.2, 1.15, 0.04, 0], "stepfun/Step-3.5-Flash": [0.1, 0.3, 0.02, 0], - "xiaomi/mimo-v2.5-pro": [0.435, 0.87, 0.0036, 0], - "xiaomi/mimo-v2.5": [0.14, 0.28, 0.0028, 0], + "tencent/hy3-paid": [0.14, 0.58, 0.035, 0], "nvidia/nemotron-3-ultra-550b-a55b": [0.6, 2.4, 0.12, 0], - "sakana/fugu-ultra": [5, 30, 0.5, 0], "thinkingmachines/inkling": [1, 4.05, 0.17, 0], "thinkingmachines/inkling-small": [0.5, 1.2, 0.1, 0], - "meta/muse-spark-1.1": [1.25, 4.25, 0.15, 0], + "poolside/laguna-s-2.1-free": [0, 0, 0, 0], + "stealth/ox-alpha": [0, 0, 0, 0], "claude-sonnet-5": [2, 10, 0.2, 2.5], "claude-sonnet-4-6": [3, 15, 0.3, 3.75], "claude-fable-5": [10, 50, 1, 12.5], @@ -57,10 +58,16 @@ "gpt-5.4": [2.5, 15, 0.25, 0], "gpt-5.3-codex": [2, 8, 0.5, 0], "gpt-5.4-mini": [0.75, 4.5, 0.075, 0], + "google/gemini-3.7-flash": [0.75, 3.75, 0.075, 0.04167], "google/gemini-3.6-flash": [1.5, 7.5, 0.15, 0], "google/gemini-3.5-flash": [1.5, 9, 0.15, 0], "google/gemini-3.5-flash-lite": [0.3, 2.5, 0.03, 0], "google/gemini-3.1-flash-lite": [0.25, 1.5, 0.03, 0], - "xai/grok-4.5": [2, 6, 0.5, 0] + "sakana/fugu-ultra": [5, 30, 0.5, 0], + "meta/muse-spark-1.1": [1.25, 4.25, 0.15, 0], + "meta/muse-spark-1.2": [1.25, 4.25, 0.15, 0], + "meta/muse-spark-1.2-contributor": [0.1, 0.2, 0.002, 0], + "xai/grok-4.5": [2, 6, 0.5, 0], + "xai/grok-4.6": [2, 6, 0.5, 0] } } diff --git a/tests/test-model-metadata-check.ts b/tests/test-model-metadata-check.ts index ac1b61f..97fe30e 100644 --- a/tests/test-model-metadata-check.ts +++ b/tests/test-model-metadata-check.ts @@ -5,6 +5,7 @@ import { commandCodeModelMetadataFromContents, diffModelMetadata, hasModelMetadataDiff, + parseBundleModelCapabilities, parseKnownTextOnlyModelIds, parseModelsReference, parsePackageVersion, @@ -21,7 +22,7 @@ const MODELS_REFERENCE = ` ` const CLI_BUNDLE = - 'const catalog=new Set(["text-model"]),__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' + 'const V={id:"vision-model",inputModalities:["text","image"],reasoning:!0,reasoningEfforts:["low","high"],maxOutputTokens:32768},T={id:"text-model",inputModalities:["text"]},catalog=new Set(["text-model"]),__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' describe("Command Code model metadata checker", () => { it("parses model ids and reasoning efforts from the generated reference", () => { @@ -42,29 +43,39 @@ describe("Command Code model metadata checker", () => { assert.throws(() => parsePackageVersion("latest"), /one semantic version/) }) - it("derives image support by excluding known text-only models", () => { + it("derives image, reasoning, effort, and output-limit metadata", () => { + assert.deepEqual(parseBundleModelCapabilities(CLI_BUNDLE, ["text-model", "vision-model"]), { + reasoningModelIds: ["vision-model"], + maxOutputTokens: { "vision-model": 32_768 }, + }) assert.deepEqual(commandCodeModelMetadataFromContents(MODELS_REFERENCE, CLI_BUNDLE), { imageModelIds: ["vision-model"], + reasoningModelIds: ["vision-model"], reasoningEfforts: { "vision-model": ["low", "high"] }, + maxOutputTokens: { "vision-model": 32_768 }, }) }) it("reports additions, removals, and changed reasoning efforts", () => { const current: CommandCodeModelMetadata = { imageModelIds: ["removed-image", "stable-image"], + reasoningModelIds: ["removed-reasoning", "stable-reasoning"], reasoningEfforts: { - "changed-reasoning": ["low"], - "removed-reasoning": ["high"], - "stable-reasoning": ["low", "high"], + "changed-effort": ["low"], + "removed-effort": ["high"], + "stable-effort": ["low", "high"], }, + maxOutputTokens: { "changed-output": 1, "removed-output": 2, "stable-output": 3 }, } const upstream: CommandCodeModelMetadata = { imageModelIds: ["added-image", "stable-image"], + reasoningModelIds: ["added-reasoning", "stable-reasoning"], reasoningEfforts: { - "added-reasoning": ["max"], - "changed-reasoning": ["low", "high"], - "stable-reasoning": ["low", "high"], + "added-effort": ["max"], + "changed-effort": ["low", "high"], + "stable-effort": ["low", "high"], }, + maxOutputTokens: { "added-output": 4, "changed-output": 5, "stable-output": 3 }, } const diff = diffModelMetadata(current, upstream) @@ -75,7 +86,12 @@ describe("Command Code model metadata checker", () => { removedImageModelIds: ["removed-image"], addedReasoningModelIds: ["added-reasoning"], removedReasoningModelIds: ["removed-reasoning"], - changedReasoningModelIds: ["changed-reasoning"], + addedEffortModelIds: ["added-effort"], + removedEffortModelIds: ["removed-effort"], + changedEffortModelIds: ["changed-effort"], + addedMaxOutputModelIds: ["added-output"], + removedMaxOutputModelIds: ["removed-output"], + changedMaxOutputModelIds: ["changed-output"], }) assert.equal(hasModelMetadataDiff(diff), true) }) @@ -83,7 +99,9 @@ describe("Command Code model metadata checker", () => { it("reports CLI version drift even when model metadata is unchanged", () => { const metadata: CommandCodeModelMetadata = { imageModelIds: ["vision-model"], + reasoningModelIds: ["vision-model"], reasoningEfforts: { "vision-model": ["low"] }, + maxOutputTokens: { "vision-model": 32_768 }, } const diff = diffModelMetadata(metadata, metadata, "1.32.2", "1.33.0") @@ -96,10 +114,12 @@ describe("Command Code model metadata checker", () => { assert.equal( renderCommandCodeCatalog("1.33.0", { imageModelIds: ["b-model", "a-model"], + reasoningModelIds: ["c-model", "a-model"], reasoningEfforts: { "b-model": ["high", "max"], "a-model": ["low"], }, + maxOutputTokens: { "b-model": 32_768 }, }), `export const COMMAND_CODE_CLI_VERSION = "1.33.0" @@ -115,10 +135,19 @@ export const MODEL_INPUT_MODALITIES: Readonly> = { + "a-model": true, + "c-model": true, +} + export const MODEL_EFFORTS: Readonly> = { "a-model": ["low"], "b-model": ["high", "max"], } + +export const MODEL_MAX_OUTPUT_TOKENS: Readonly> = { + "b-model": 32_768, +} `, ) assert.equal( diff --git a/tests/test-models.ts b/tests/test-models.ts index 391e06b..c40d321 100644 --- a/tests/test-models.ts +++ b/tests/test-models.ts @@ -16,6 +16,8 @@ import { loadCommandCodeModels, MODEL_EFFORTS, MODEL_INPUT_MODALITIES, + MODEL_MAX_OUTPUT_TOKENS, + MODEL_REASONING, modelSupportsImageInput, thinkingLevelMapForEfforts, thinkingMetadataForModel, @@ -41,7 +43,7 @@ const EXPECTED_MODELS: readonly CommandCodeModel[] = [ id: "Qwen/Qwen3.7-Max", name: "Qwen 3.7 Max (CC)", api: "openai-completions", - reasoning: false, + reasoning: true, contextWindow: 1_000_000, maxTokens: 65_536, }, @@ -124,17 +126,55 @@ describe("commandCodeModelsFromApiResponse()", () => { } }) - it("marks only known reasoning models as reasoning-capable", () => { + it("tracks reasoning independently from selectable effort levels", () => { const models = commandCodeModelsFromApiResponse({ object: "list", data: [ { ...API_RESPONSE.data[0], id: "deepseek/deepseek-v4-flash" }, + { ...API_RESPONSE.data[0], id: "moonshotai/Kimi-K3" }, { ...API_RESPONSE.data[0], id: "new-model-without-metadata" }, ], }) assert.equal(models[0]?.reasoning, true) - assert.equal(models[1]?.reasoning, false) + assert.equal(models[1]?.reasoning, true) + assert.deepEqual(thinkingMetadataForModel("moonshotai/Kimi-K3"), { + thinkingLevelMap: { + minimal: null, + low: null, + medium: null, + high: null, + xhigh: null, + max: null, + }, + }) + assert.equal(models[2]?.reasoning, false) + assert.equal(Object.keys(MODEL_REASONING).length, 48) + }) + + it("uses model-specific output limits from the CLI catalog", () => { + const models = commandCodeModelsFromApiResponse({ + object: "list", + data: [ + { ...API_RESPONSE.data[0], id: "Qwen/Qwen3.8-27B", context_length: 262_144 }, + { ...API_RESPONSE.data[0], id: "stealth/ox-alpha", context_length: 1_048_576 }, + { + ...API_RESPONSE.data[0], + id: "poolside/laguna-s-2.1-free", + context_length: 256_000, + }, + ], + }) + + assert.deepEqual( + models.map(({ id, maxTokens }) => ({ id, maxTokens })), + [ + { id: "Qwen/Qwen3.8-27B", maxTokens: 32_768 }, + { id: "stealth/ox-alpha", maxTokens: 131_072 }, + { id: "poolside/laguna-s-2.1-free", maxTokens: 32_768 }, + ], + ) + assert.equal(Object.keys(MODEL_MAX_OUTPUT_TOKENS).length, 3) }) it(`uses the command-code@${COMMAND_CODE_CLI_VERSION} reasoning effort catalog`, () => { @@ -151,6 +191,7 @@ describe("commandCodeModelsFromApiResponse()", () => { for (const [modelId, efforts] of Object.entries(MODEL_EFFORTS)) { const metadata = thinkingMetadataForModel(modelId) assert.ok(metadata, `${modelId} should have reasoning metadata`) + assert.ok(metadata.thinking) assert.equal(metadata.thinking.mode, "effort") assert.deepEqual(metadata.thinking.efforts, efforts) assert.deepEqual( diff --git a/tests/test-pricing.ts b/tests/test-pricing.ts index 202d8aa..d7141a5 100644 --- a/tests/test-pricing.ts +++ b/tests/test-pricing.ts @@ -27,7 +27,7 @@ const fixtureUrl = new URL("./fixtures/commandcode-model-ids.json", import.meta. const fixture = JSON.parse(await readFile(fixtureUrl, "utf-8")) as ModelCatalogSnapshot const pricingFixtureUrl = new URL("./fixtures/commandcode-pricing.json", import.meta.url) const pricingFixture = JSON.parse(await readFile(pricingFixtureUrl, "utf-8")) as PricingSnapshot -const freeModels = new Set(["poolside/laguna-s-2.1-free", "inclusionai/ling-3.0-flash-free"]) +const freeModels = new Set(["poolside/laguna-s-2.1-free", "stealth/ox-alpha"]) function assertCost( modelId: string, @@ -50,7 +50,7 @@ function assertCost( describe("MODEL_COSTS pricing overlay", () => { it("covers the current Command Code model catalog snapshot", () => { assert.equal(fixture.source, "https://api.commandcode.ai/provider/v1/models") - assert.match(fixture.fetchedAt, /^2026-08-22T/) + assert.match(fixture.fetchedAt, /^2026-08-25T/) const catalogIds = [...fixture.modelIds].sort() const pricedIds = Object.keys(MODEL_COSTS).sort() @@ -138,6 +138,24 @@ describe("MODEL_COSTS pricing overlay", () => { cacheRead: 0.03, cacheWrite: 0, }) + assertCost("Qwen/Qwen3.8-27B", { + input: 0.4, + output: 3, + cacheRead: 0.04, + cacheWrite: 0, + }) + assertCost("google/gemini-3.7-flash", { + input: 0.75, + output: 3.75, + cacheRead: 0.075, + cacheWrite: 0.04167, + }) + assertCost("meta/muse-spark-1.2-contributor", { + input: 0.1, + output: 0.2, + cacheRead: 0.002, + cacheWrite: 0, + }) }) it("uses the documented base rates for context-dependent models", () => { @@ -165,11 +183,20 @@ describe("MODEL_COSTS pricing overlay", () => { cacheRead: 0.02, cacheWrite: 0.25, }) + assert.deepEqual(MODEL_COSTS["xai/grok-4.6"]?.tiers, [ + { + inputTokensAbove: 200_000, + input: 4, + output: 12, + cacheRead: 1, + cacheWrite: 0, + }, + ]) }) it("tracks pricing provenance", () => { assert.equal(PRICING_SOURCE_URL, "https://commandcode.ai/docs/resources/pricing-limits") - assert.equal(PRICING_LAST_VERIFIED, "2026-08-22") + assert.equal(PRICING_LAST_VERIFIED, "2026-08-25") }) it("fails once temporary pricing needs review", () => { From c51a7905301cf1ce5b10b7dac09718631f2d09f4 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 15:54:18 +0200 Subject: [PATCH 31/40] fix(stream): match Command Code CLI transport behavior --- index.ts | 7 +- src/auth-server.ts | 7 ++ src/converters.ts | 53 +++++++++--- src/core.ts | 71 ++++++++++++---- src/oauth.ts | 43 +++++++--- src/types.ts | 2 + tests/test-oauth.ts | 158 ++++++++++++++++++++++++++--------- tests/test-pure-functions.ts | 49 ++++++++++- tests/test-stream.ts | 121 ++++++++++++++++++++++++++- 9 files changed, 426 insertions(+), 85 deletions(-) diff --git a/index.ts b/index.ts index d3005d4..09222bd 100644 --- a/index.ts +++ b/index.ts @@ -26,6 +26,7 @@ import { getModelsTimeoutMs, inputModalitiesForModel, loadCommandCodeModels, + MODEL_EFFORTS, thinkingMetadataForModel, type CommandCodeModel, } from "./src/models.ts" @@ -37,7 +38,7 @@ import { createCommandCodeRuntime } from "./src/runtime.ts" import { createCommandCodeTransportRouter } from "./src/transport.ts" function commandCodeHeaders(): Record | undefined { - if (process.env.COMMANDCODE_ZDR === "1") { + if (process.env.CMD_ZDR === "1" || process.env.COMMANDCODE_ZDR === "1") { return { "x-cmd-zdr": "1" } } return undefined @@ -52,7 +53,7 @@ function createProviderConfig( return { name: "Command Code", baseUrl: apiBase, - apiKey: getConfiguredApiKey() ?? "$COMMANDCODE_API_KEY", + apiKey: getConfiguredApiKey() ?? "$COMMAND_CODE_API_KEY", api: "commandcode-custom", streamSimple: streamCommandCode, headers, @@ -79,7 +80,7 @@ function createProviderConfig( ? { supportsStore: false, supportsDeveloperRole: false, - supportsReasoningEffort: true, + supportsReasoningEffort: MODEL_EFFORTS[model.id] !== undefined, maxTokensField: "max_tokens", } : { diff --git a/src/auth-server.ts b/src/auth-server.ts index e815c10..ee9e6de 100644 --- a/src/auth-server.ts +++ b/src/auth-server.ts @@ -28,6 +28,7 @@ export interface AuthServer { export interface AuthServerOptions { startPort?: number portRange?: number + expectedState?: string } function listenOnAvailablePort( @@ -181,6 +182,12 @@ export async function startAuthServer(options: AuthServerOptions = {}): Promise< return } + if (options.expectedState !== undefined && state !== options.expectedState) { + res.writeHead(403) + res.end(JSON.stringify({ success: false, error: "Invalid state token" })) + return + } + res.writeHead(200) res.end(JSON.stringify({ success: true })) diff --git a/src/converters.ts b/src/converters.ts index bb60df2..1e7ea97 100644 --- a/src/converters.ts +++ b/src/converters.ts @@ -107,6 +107,7 @@ export function getApiKey( } = {}, ): string | undefined { const env = options.env ?? process.env + if (env.COMMAND_CODE_API_KEY) return env.COMMAND_CODE_API_KEY if (env.COMMANDCODE_API_KEY) return env.COMMANDCODE_API_KEY const home = options.homeDir?.() ?? homedir() @@ -138,10 +139,11 @@ export function getApiKey( return undefined } -// Hosts such as OMP may pass the literal env-var name "$COMMANDCODE_API_KEY" -// (or "COMMANDCODE_API_KEY") as the "resolved" registry key instead of the -// actual credential. Treat those as unresolved. +// Hosts such as OMP may pass a literal env-var name as the "resolved" registry +// key instead of the actual credential. Treat those as unresolved. export const COMMAND_CODE_PLACEHOLDER_KEYS = new Set([ + "$COMMAND_CODE_API_KEY", + "COMMAND_CODE_API_KEY", "$COMMANDCODE_API_KEY", "COMMANDCODE_API_KEY", ]) @@ -162,6 +164,16 @@ export function pickCommandCodeApiKey( } export function textContent(message: { content?: unknown }): string { + if (typeof message.content === "string") return message.content + if (message.content === null || message.content === undefined) return "" + if (!Array.isArray(message.content)) { + try { + return JSON.stringify(message.content) ?? String(message.content) + } catch { + return String(message.content) + } + } + return recordArray(message.content) .filter((part) => part.type === "text") .map((part) => stringValue(part.text) ?? "") @@ -182,7 +194,12 @@ export function toolsToJson(tools?: readonly ToolLike[]): unknown[] { })) } -function completeToolCallIds(messages?: readonly MessageLike[]): Set { +interface ToolCallState { + callIds: ReadonlySet + resultIds: ReadonlySet +} + +function toolCallState(messages?: readonly MessageLike[]): ToolCallState { const callIds = new Set() const resultIds = new Set() @@ -194,12 +211,12 @@ function completeToolCallIds(messages?: readonly MessageLike[]): Set { if (id) callIds.add(id) } } - } else if (message.role === "toolResult") { - if (message.toolCallId) resultIds.add(message.toolCallId) + } else if (message.role === "toolResult" && message.toolCallId) { + resultIds.add(message.toolCallId) } } - return new Set([...callIds].filter((id) => resultIds.has(id))) + return { callIds, resultIds } } export function messagesToCC( @@ -210,7 +227,7 @@ export function messagesToCC( if (!allowImages) assertTextOnlyMessages(messages) const out: unknown[] = [] - const pairedToolCallIds = completeToolCallIds(messages) + const { callIds, resultIds } = toolCallState(messages) for (const message of messages ?? []) { if (message.role === "user") { @@ -220,23 +237,37 @@ export function messagesToCC( }) } else if (message.role === "assistant") { const parts: unknown[] = [] + const missingResults: unknown[] = [] for (const content of recordArray(message.content)) { if (content.type === "text") { parts.push({ type: "text", text: stringValue(content.text) ?? "" }) } else if (content.type === "toolCall") { const toolCallId = stringValue(content.id) ?? "" - if (!pairedToolCallIds.has(toolCallId)) continue + const toolName = stringValue(content.name) ?? "" + if (!toolCallId) continue parts.push({ type: "tool-call", toolCallId, - toolName: stringValue(content.name) ?? "", + toolName, input: recordOrEmpty(content.arguments), }) + if (!resultIds.has(toolCallId)) { + missingResults.push({ + type: "tool-result", + toolCallId, + toolName, + output: { + type: "error-text", + value: "No result — the tool call did not complete (interrupted or lost).", + }, + }) + } } } if (parts.length > 0) out.push({ role: "assistant", content: parts }) + if (missingResults.length > 0) out.push({ role: "tool", content: missingResults }) } else if (message.role === "toolResult") { - if (!message.toolCallId || !pairedToolCallIds.has(message.toolCallId)) continue + if (!message.toolCallId || !callIds.has(message.toolCallId)) continue out.push({ role: "tool", content: [ diff --git a/src/core.ts b/src/core.ts index 57ed455..c88cef4 100644 --- a/src/core.ts +++ b/src/core.ts @@ -148,6 +148,10 @@ function mappedReasoningEffort(model: ModelLike, options?: StreamOptions): strin return typeof mapped === "string" && mapped !== "off" ? mapped : undefined } +function isUuid(value: string): boolean { + return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value) +} + export function projectSlugFromPath(pathName: string): string { const slug = pathName .toLowerCase() @@ -232,17 +236,15 @@ export function createStreamCommandCode(deps: CoreDependencies) { const stream = deps.createStream() async function run() { - // OMP may pass the legacy env-var name "COMMANDCODE_API_KEY" (old pi) - // or "$COMMANDCODE_API_KEY" (new pi) as the apiKey value instead of - // resolving it. Filter out these specific strings. - const LEGACY_API_KEY_REF = "$COMMANDCODE_API_KEY" - const OLD_API_KEY_REF = "COMMANDCODE_API_KEY" + // Some hosts pass a literal env-var reference instead of resolving it. + const PLACEHOLDER_API_KEYS = new Set([ + "$COMMAND_CODE_API_KEY", + "COMMAND_CODE_API_KEY", + "$COMMANDCODE_API_KEY", + "COMMANDCODE_API_KEY", + ]) const hostKey = - options?.apiKey && - options.apiKey !== LEGACY_API_KEY_REF && - options.apiKey !== OLD_API_KEY_REF - ? options.apiKey - : undefined + options?.apiKey && !PLACEHOLDER_API_KEYS.has(options.apiKey) ? options.apiKey : undefined const apiKey = hostKey ?? @@ -262,7 +264,7 @@ export function createStreamCommandCode(deps: CoreDependencies) { usage: defaultUsage(), stopReason: "error", errorMessage: - "No Command Code API key. Run /login and select Command Code, set the COMMANDCODE_API_KEY env var, or configure ~/.commandcode/auth.json, ~/.pi/agent/auth.json or ~/.omp/agent/auth.json", + "No Command Code API key. Run /login and select Command Code, set COMMAND_CODE_API_KEY (or legacy COMMANDCODE_API_KEY), or configure ~/.commandcode/auth.json, ~/.pi/agent/auth.json or ~/.omp/agent/auth.json", timestamp: now(), } stream.push({ type: "error", reason: "error", error: msg }) @@ -424,6 +426,15 @@ export function createStreamCommandCode(deps: CoreDependencies) { } case "finish": { + const rawFinishReason = stringValue(event.rawFinishReason) + if ( + rawFinishReason && + /^(?:network|connection|upstream)[-_\s]?error$/i.test(rawFinishReason) + ) { + throw new Error( + `Provider finished with reason "${rawFinishReason}" — upstream connection failed mid-stream`, + ) + } const usage = commandCodeUsage(event) if (usage) { const details = commandCodeInputTokenDetails(usage) @@ -447,6 +458,10 @@ export function createStreamCommandCode(deps: CoreDependencies) { break } + case "abort": { + throw abortError("Request aborted") + } + case "error": { const message = commandCodeErrorMessage(event.error) ?? @@ -464,7 +479,11 @@ export function createStreamCommandCode(deps: CoreDependencies) { if (controller.signal.aborted) throw abortError("Aborted") const workingDir = cwd() - const threadId = uuid() + const threadId = options?.sessionId + ? isUuid(options.sessionId) + ? options.sessionId + : undefined + : uuid() const reasoningEffort = mappedReasoningEffort(model, options) const timeoutMs = options?.timeoutMs @@ -492,8 +511,8 @@ export function createStreamCommandCode(deps: CoreDependencies) { tools: toolsToJson(context.tools), system: systemPromptToText(context.systemPrompt), max_tokens: generateMaxTokens(model, options), - temperature: 0.3, stream: true, + ...(options?.temperature !== undefined ? { temperature: options.temperature } : {}), ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}), }, threadId, @@ -523,7 +542,8 @@ export function createStreamCommandCode(deps: CoreDependencies) { "x-cli-environment": "production", "x-project-slug": projectSlugFromPath(workingDir), "x-taste-learning": "true", - "x-co-flag": "false", + ...(options?.sessionId ? { "x-session-id": options.sessionId } : {}), + "User-Agent": "cli", ...options?.headers, } const bodyStr = JSON.stringify(body) @@ -636,6 +656,11 @@ export function createStreamCommandCode(deps: CoreDependencies) { const { done, value } = await raceAbort(reader.read(), attemptController.signal) if (done) { if (buffer.trim()) handleEvent(parseStreamEventLine(buffer)) + if (!finished) { + throw new Error( + "Stream ended unexpectedly before completion (no finish event) — response was truncated", + ) + } break } if (controller.signal.aborted) throw abortError("Aborted") @@ -659,7 +684,12 @@ export function createStreamCommandCode(deps: CoreDependencies) { } catch {} reader = undefined - if (controller.signal.aborted) throw streamError + if ( + controller.signal.aborted || + (streamError instanceof Error && streamError.name === "AbortError") + ) { + throw streamError + } // Never retry after visible content was emitted (including timeout mid-stream). const canRetry = output.content.length === 0 && attempt < maxRetries @@ -679,6 +709,12 @@ export function createStreamCommandCode(deps: CoreDependencies) { throw streamError } + if (!finished) { + throw new Error( + "Stream ended unexpectedly before completion (no finish event) — response was truncated", + ) + } + // Stream completed successfully. endTextBlock() endThinking() @@ -696,7 +732,10 @@ export function createStreamCommandCode(deps: CoreDependencies) { } } } catch (error: unknown) { - const reason: ErrorReason = controller.signal.aborted ? "aborted" : "error" + const reason: ErrorReason = + controller.signal.aborted || (error instanceof Error && error.name === "AbortError") + ? "aborted" + : "error" output.stopReason = reason output.errorMessage = reason === "aborted" diff --git a/src/oauth.ts b/src/oauth.ts index 0a68ac5..470ebd3 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -18,7 +18,8 @@ import { startAuthServer } from "./auth-server.ts" const STUDIO_BASE_URL = "https://commandcode.ai" const TEN_YEARS_MS = 10 * 365 * 24 * 60 * 60 * 1000 // API keys don't expire -const DEFAULT_AUTH_TIMEOUT_MS = 15_000 +const DEFAULT_AUTH_TIMEOUT_MS = 120_000 +const DEFAULT_API_BASE = "https://api.commandcode.ai" export interface OAuthLoginCallbacks { onAuth(params: { url: string }): void @@ -95,9 +96,34 @@ export function sanitizeApiKey(input: string): string { .trim() } +export async function validateApiKey( + apiKey: string, + options: { fetchImpl?: typeof fetch; apiBase?: string } = {}, +): Promise { + let response: Response + try { + response = await (options.fetchImpl ?? fetch)( + `${options.apiBase ?? DEFAULT_API_BASE}/alpha/whoami`, + { + headers: { Authorization: `Bearer ${apiKey}` }, + }, + ) + } catch (error) { + throw new Error( + `Could not validate the Command Code API key: ${error instanceof Error ? error.message : String(error)}`, + ) + } + + if (response.status === 401) throw new Error("Invalid Command Code API key") + if (!response.ok) { + throw new Error(`Could not validate the Command Code API key (${response.status})`) + } +} + async function promptForApiKey(callbacks: OAuthLoginCallbacks, message: string) { const apiKey = sanitizeApiKey(await callbacks.onPrompt({ message })) if (!apiKey) throw new Error("No Command Code API key provided") + await validateApiKey(apiKey) return credentialsFromApiKey(apiKey) } @@ -130,9 +156,10 @@ async function chooseLoginFlow(callbacks: OAuthLoginCallbacks): Promise { + const stateToken = generateStateToken() let authServer try { - authServer = await startAuthServer() + authServer = await startAuthServer({ expectedState: stateToken }) } catch { return promptForApiKey( callbacks, @@ -140,7 +167,6 @@ async function browserLogin(callbacks: OAuthLoginCallbacks): Promise { const choice = await chooseLoginFlow(callbacks) - if (choice.type === "apiKey") return credentialsFromApiKey(choice.apiKey) + if (choice.type === "apiKey") { + await validateApiKey(choice.apiKey) + return credentialsFromApiKey(choice.apiKey) + } if (choice.type === "prompt") { return promptForApiKey(callbacks, "Paste your Command Code API key:") } diff --git a/src/types.ts b/src/types.ts index dc680ea..50066ef 100644 --- a/src/types.ts +++ b/src/types.ts @@ -112,6 +112,8 @@ export interface StreamOptions { headers?: Record fetch?: typeof fetch maxTokens?: number + temperature?: number + sessionId?: string /** Resolved pi thinking level; forwarded only through the model's map. */ reasoning?: string onPayload?: (payload: unknown, model: ModelLike) => unknown | Promise diff --git a/tests/test-oauth.ts b/tests/test-oauth.ts index 00d94a0..6a64381 100644 --- a/tests/test-oauth.ts +++ b/tests/test-oauth.ts @@ -9,7 +9,7 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" import { startAuthServer, type AuthCallback } from "../src/auth-server.ts" -import { getApiKey, login, refreshToken, sanitizeApiKey } from "../src/oauth.ts" +import { getApiKey, login, refreshToken, sanitizeApiKey, validateApiKey } from "../src/oauth.ts" /** * Helper: wait for an HTTP server to close, or resolve immediately if already closed. @@ -24,9 +24,27 @@ function waitForClose(server: { }) } +async function withValidApiKeyFetch(run: () => Promise): Promise { + const originalFetch = globalThis.fetch + globalThis.fetch = (input, init) => { + if (String(input).endsWith("/alpha/whoami")) { + return Promise.resolve(new Response(JSON.stringify({ user: {} }), { status: 200 })) + } + return originalFetch(input, init) + } + try { + return await run() + } finally { + globalThis.fetch = originalFetch + } +} + describe("startAuthServer()", () => { it("starts on a localhost port and accepts a valid callback POST", async () => { - const { server, port, waitForCallback } = await startAuthServer({ startPort: 0 }) + const { server, port, waitForCallback } = await startAuthServer({ + startPort: 0, + expectedState: "test-state-token", + }) const callbackData: AuthCallback = { apiKey: "user_testKey123", @@ -57,6 +75,42 @@ describe("startAuthServer()", () => { await waitForClose(server) }) + it("rejects a mismatched state without closing the callback server", async () => { + const { server, port, waitForCallback } = await startAuthServer({ + startPort: 0, + expectedState: "correct-state", + }) + + const invalidResponse = await fetch(`http://127.0.0.1:${port}/callback`, { + method: "POST", + headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" }, + body: JSON.stringify({ + apiKey: "user_badState", + state: "wrong-state", + userId: "user_789", + userName: "Attacker", + keyName: "evil-key", + }), + }) + assert.equal(invalidResponse.status, 403) + assert.equal(server.listening, true) + + const validResponse = await fetch(`http://127.0.0.1:${port}/callback`, { + method: "POST", + headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" }, + body: JSON.stringify({ + apiKey: "user_valid", + state: "correct-state", + userId: "user_123", + userName: "Valid User", + keyName: "valid-key", + }), + }) + assert.equal(validResponse.status, 200) + assert.equal((await waitForCallback).apiKey, "user_valid") + await waitForClose(server) + }) + it("rejects when the callback indicates access_denied", async () => { const { server, port, waitForCallback } = await startAuthServer({ startPort: 0 }) @@ -176,6 +230,18 @@ describe("OAuth functions", () => { it("sanitizeApiKey removes paste markers, control chars, and whitespace", () => { assert.equal(sanitizeApiKey("\u001b[200~ user_manualKey\n\u001b[201~"), "user_manualKey") }) + + it("validates manual API keys through whoami", async () => { + await validateApiKey("valid-key", { + fetchImpl: () => Promise.resolve(new Response(JSON.stringify({ user: {} }), { status: 200 })), + }) + await assert.rejects( + validateApiKey("invalid-key", { + fetchImpl: () => Promise.resolve(new Response("unauthorized", { status: 401 })), + }), + /Invalid Command Code API key/, + ) + }) }) describe("login()", () => { @@ -242,15 +308,17 @@ describe("login()", () => { const promptMessages: string[] = [] try { - const result = await login({ - onAuth(params: { url: string }) { - authUrl = params.url - }, - async onPrompt(params: { message: string }): Promise { - promptMessages.push(params.message) - return promptMessages.length === 1 ? "" : "\u001b[200~ user_manualApiKey\n\u001b[201~" - }, - }) + const result = await withValidApiKeyFetch(() => + login({ + onAuth(params: { url: string }) { + authUrl = params.url + }, + async onPrompt(params: { message: string }): Promise { + promptMessages.push(params.message) + return promptMessages.length === 1 ? "" : "\u001b[200~ user_manualApiKey\n\u001b[201~" + }, + }), + ) assert.match(authUrl, /^https:\/\/commandcode\.ai\/studio\/auth\/cli\?/) assert.match(promptMessages[1] ?? "", /Paste your Command Code API key/) @@ -265,14 +333,16 @@ describe("login()", () => { it("accepts a directly pasted API key", async () => { let authOpened = false - const result = await login({ - onAuth() { - authOpened = true - }, - onPrompt(): Promise { - return Promise.resolve("user_directApiKey") - }, - }) + const result = await withValidApiKeyFetch(() => + login({ + onAuth() { + authOpened = true + }, + onPrompt(): Promise { + return Promise.resolve("user_directApiKey") + }, + }), + ) assert.equal(authOpened, false) assert.equal(result.access, "user_directApiKey") @@ -280,21 +350,23 @@ describe("login()", () => { it("offers an explicit API key prompt", async () => { let promptCount = 0 - const result = await login({ - onAuth() { - throw new Error("browser should not open") - }, - onPrompt(): Promise { - promptCount += 1 - return Promise.resolve(promptCount === 1 ? "key" : "user_promptedApiKey") - }, - }) + const result = await withValidApiKeyFetch(() => + login({ + onAuth() { + throw new Error("browser should not open") + }, + onPrompt(): Promise { + promptCount += 1 + return Promise.resolve(promptCount === 1 ? "key" : "user_promptedApiKey") + }, + }), + ) assert.equal(result.access, "user_promptedApiKey") assert.equal(promptCount, 2) }) - it("rejects on state token mismatch", async () => { + it("keeps waiting after a state mismatch and accepts the legitimate callback", async () => { let authUrl = "" const callbacks = { onAuth(params: { url: string }) { @@ -305,12 +377,7 @@ describe("login()", () => { }, } - const loginPromise: Promise = login(callbacks).then( - () => { - throw new Error("Expected login to reject") - }, - (e: Error) => e.message, - ) + const loginPromise = login(callbacks) // Wait for onAuth to be called asynchronously while (!authUrl) await new Promise((resolve) => setTimeout(resolve, 10)) @@ -318,8 +385,8 @@ describe("login()", () => { const url = new URL(authUrl) const port = parseInt(url.searchParams.get("callback")?.match(/localhost:(\d+)/)?.[1] ?? "0") - // Post back with a wrong state token - await fetch(`http://127.0.0.1:${port}/callback`, { + // Post back with a wrong state token. + const invalidResponse = await fetch(`http://127.0.0.1:${port}/callback`, { method: "POST", headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" }, body: JSON.stringify({ @@ -331,7 +398,20 @@ describe("login()", () => { }), }) - const errorMsg = await loginPromise - assert.match(errorMsg, /State token mismatch/) + assert.equal(invalidResponse.status, 403) + + const validResponse = await fetch(`http://127.0.0.1:${port}/callback`, { + method: "POST", + headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" }, + body: JSON.stringify({ + apiKey: "user_goodState", + state: url.searchParams.get("state"), + userId: "user_123", + userName: "Real User", + keyName: "real-key", + }), + }) + assert.equal(validResponse.status, 200) + assert.equal((await loginPromise).access, "user_goodState") }) }) diff --git a/tests/test-pure-functions.ts b/tests/test-pure-functions.ts index d71dba0..17a4c78 100644 --- a/tests/test-pure-functions.ts +++ b/tests/test-pure-functions.ts @@ -27,8 +27,18 @@ import { redactCommandCodeErrorText } from "../src/overflow.ts" import { objectAt } from "./helpers.ts" describe("getApiKey()", () => { - it("uses COMMANDCODE_API_KEY from provided env", () => { - assert.equal(getApiKey({ env: { COMMANDCODE_API_KEY: "env-key" }, authPaths: [] }), "env-key") + it("uses the official API key env var before the legacy alias", () => { + assert.equal( + getApiKey({ + env: { COMMAND_CODE_API_KEY: "official-key", COMMANDCODE_API_KEY: "legacy-key" }, + authPaths: [], + }), + "official-key", + ) + assert.equal( + getApiKey({ env: { COMMANDCODE_API_KEY: "legacy-key" }, authPaths: [] }), + "legacy-key", + ) }) it("reads apiKey, commandcode, pi OAuth, and official CLI credential fields", () => { @@ -109,11 +119,14 @@ describe("error redaction", () => { describe("pickCommandCodeApiKey()", () => { it("falls back to the host key for a placeholder registry value", () => { + assert.equal(pickCommandCodeApiKey("$COMMAND_CODE_API_KEY", "file-key"), "file-key") + assert.equal(pickCommandCodeApiKey("COMMAND_CODE_API_KEY", "file-key"), "file-key") assert.equal(pickCommandCodeApiKey("$COMMANDCODE_API_KEY", "file-key"), "file-key") assert.equal(pickCommandCodeApiKey("COMMANDCODE_API_KEY", "file-key"), "file-key") }) it("returns undefined when only a placeholder is provided (no fallback)", () => { + assert.equal(pickCommandCodeApiKey("$COMMAND_CODE_API_KEY", undefined), undefined) assert.equal(pickCommandCodeApiKey("$COMMANDCODE_API_KEY", undefined), undefined) }) @@ -199,6 +212,12 @@ describe("textContent()", () => { ) }) + it("normalizes malformed string and object content", () => { + assert.equal(textContent({ content: "raw result" }), "raw result") + assert.equal(textContent({ content: { ok: true } }), '{"ok":true}') + assert.equal(textContent({ content: null }), "") + }) + it("handles empty or missing content", () => { assert.equal(textContent({ content: [] }), "") assert.equal(textContent({}), "") @@ -531,6 +550,23 @@ describe("messagesToCC()", () => { assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "hello\nworld") }) + it("preserves malformed string tool results instead of sending empty output", () => { + const result = messagesToCC([ + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: "raw result", + }, + ]) + + assert.equal(objectAt(result, ["1", "content", "0", "output", "value"]), "raw result") + }) + it("serializes image inputs in the current Command Code wire format", () => { assert.deepEqual( messagesToCC( @@ -632,7 +668,7 @@ describe("messagesToCC()", () => { ]) }) - it("drops orphaned tool calls that have no matching tool result", () => { + it("synthesizes missing results for orphaned tool calls", () => { const result = messagesToCC([ { role: "user", content: "edit a file" }, { @@ -651,7 +687,12 @@ describe("messagesToCC()", () => { assert.equal(objectAt(result, ["1", "role"]), "assistant") assert.equal(objectAt(result, ["1", "content", "0", "type"]), "text") - assert.equal(objectAt(result, ["1", "content", "1"]), undefined) + assert.equal(objectAt(result, ["1", "content", "1", "type"]), "tool-call") + assert.equal(objectAt(result, ["2", "role"]), "tool") + assert.match( + String(objectAt(result, ["2", "content", "0", "output", "value"])), + /did not complete/, + ) }) it("handles empty conversations", () => { diff --git a/tests/test-stream.ts b/tests/test-stream.ts index a015e75..fdca1c4 100644 --- a/tests/test-stream.ts +++ b/tests/test-stream.ts @@ -77,6 +77,23 @@ describe("streamCommandCode — auth", () => { ) }) + it("accepts the official CLI API key environment variable", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ + apiBase: server.baseUrl(), + env: { COMMAND_CODE_API_KEY: "official-env-key" }, + }) + + await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "$COMMAND_CODE_API_KEY" }), + ) + + assert.equal(server.lastRequestHeaders().authorization, "Bearer official-env-key") + }) + it("uses options.apiKey in the Authorization header", async () => { server.mockResponse({ type: "success", @@ -555,7 +572,7 @@ describe("streamCommandCode — request serialization", () => { assert.equal(objectAt(body, ["params", "stream"]), true) assert.equal(objectAt(body, ["params", "max_tokens"]), 500) assert.equal(objectAt(body, ["params", "reasoning_effort"]), undefined) - assert.equal(objectAt(body, ["params", "temperature"]), 0.3) + assert.equal(objectAt(body, ["params", "temperature"]), undefined) assert.equal(objectAt(body, ["params", "system"]), "You are a test assistant.") assert.equal(objectAt(body, ["memory"]), null) assert.equal(objectAt(body, ["taste"]), null) @@ -573,10 +590,53 @@ describe("streamCommandCode — request serialization", () => { assert.equal(headers["x-command-code-version"], COMMAND_CODE_CLI_VERSION) assert.equal(headers["x-project-slug"], "repo") assert.equal(headers["x-taste-learning"], "true") - assert.equal(headers["x-co-flag"], "false") + assert.equal(headers["user-agent"], "cli") + assert.equal(headers["x-co-flag"], undefined) assert.equal(headers["x-session-id"], undefined) }) + it("forwards explicit temperature and stable session metadata", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: "mock-key", + temperature: 0.7, + sessionId: "11111111-1111-4111-8111-111111111111", + }), + ) + + const body = server.lastRequestBody() + assert.equal(objectAt(body, ["params", "temperature"]), 0.7) + assert.equal(objectAt(body, ["threadId"]), "11111111-1111-4111-8111-111111111111") + assert.equal( + server.lastRequestHeaders()["x-session-id"], + "11111111-1111-4111-8111-111111111111", + ) + }) + + it("omits non-UUID session ids from the generate thread id", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: "mock-key", + sessionId: "human-readable-session", + }), + ) + + assert.equal(objectAt(server.lastRequestBody(), ["threadId"]), undefined) + assert.equal(server.lastRequestHeaders()["x-session-id"], "human-readable-session") + }) + it("accepts the legacy OMP nested reasoning map", async () => { server.mockResponse({ type: "success", @@ -819,6 +879,63 @@ describe("streamCommandCode — upstream errors and malformed streams", () => { assert.equal(error.error.errorMessage, "provider failed") }) + it("rejects a truncated stream without a finish event", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "text-delta", text: "truncated" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.match(error.error.errorMessage ?? "", /no finish event/i) + }) + + it("maps an upstream abort event to an aborted request", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "abort" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.equal(error.reason, "aborted") + }) + + it("rejects terminal upstream network failure reasons", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ + type: "finish", + finishReason: "stop", + rawFinishReason: "upstream_error", + }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.match(error.error.errorMessage ?? "", /upstream connection failed/i) + }) + it("handles SSE lines, malformed lines, split chunks, and final line without newline", async () => { const textEvent = `data: ${JSON.stringify({ type: "text-delta", text: "split" })}\n` const finishEvent = JSON.stringify({ From 6a7a71e20637c11ea5148af004a9203f9761b037 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 15:54:29 +0200 Subject: [PATCH 32/40] fix(auth): support official Command Code environment names --- scripts/live-e2e-profile.mjs | 3 ++- scripts/pi-authenticated.mjs | 1 + scripts/pi-isolated.mjs | 1 + src/api-key.ts | 1 + src/quota-command.ts | 2 +- tests/test-api-key.ts | 13 ++++++++++--- tests/test-live-e2e.mjs | 2 ++ tests/test-omp-compat.mjs | 2 +- tests/test-pi-authenticated.mjs | 5 +++-- tests/test-pi-isolated.mjs | 6 ++++-- tests/test-pi-local.mjs | 4 ++-- tests/test-quota-command.ts | 2 +- tests/test-smoke.mjs | 3 ++- 13 files changed, 31 insertions(+), 14 deletions(-) diff --git a/scripts/live-e2e-profile.mjs b/scripts/live-e2e-profile.mjs index 01e9bab..3271323 100644 --- a/scripts/live-e2e-profile.mjs +++ b/scripts/live-e2e-profile.mjs @@ -39,10 +39,11 @@ function runProfile(profile, apiKey) { const model = process.env[modelVariable] ?? "deepseek/deepseek-v4-flash" const env = { ...process.env, - COMMANDCODE_API_KEY: apiKey, + COMMAND_CODE_API_KEY: apiKey, COMMANDCODE_E2E_MODEL: model, COMMANDCODE_E2E_PROFILE: profile, } + delete env.COMMANDCODE_API_KEY delete env.COMMANDCODE_E2E_GO_API_KEY delete env.COMMANDCODE_E2E_PROVIDER_API_KEY diff --git a/scripts/pi-authenticated.mjs b/scripts/pi-authenticated.mjs index b0fe10b..974d590 100644 --- a/scripts/pi-authenticated.mjs +++ b/scripts/pi-authenticated.mjs @@ -11,6 +11,7 @@ const env = { ...process.env, PI_SKIP_VERSION_CHECK: "1", } +delete env.COMMAND_CODE_API_KEY delete env.COMMANDCODE_API_KEY const child = spawn( diff --git a/scripts/pi-isolated.mjs b/scripts/pi-isolated.mjs index a725f4e..b66c622 100644 --- a/scripts/pi-isolated.mjs +++ b/scripts/pi-isolated.mjs @@ -22,6 +22,7 @@ const env = { PI_CODING_AGENT_SESSION_DIR: sessionDir, PI_SKIP_VERSION_CHECK: "1", } +delete env.COMMAND_CODE_API_KEY delete env.COMMANDCODE_API_KEY let activeChild diff --git a/src/api-key.ts b/src/api-key.ts index 0ab52f5..f09e155 100644 --- a/src/api-key.ts +++ b/src/api-key.ts @@ -34,6 +34,7 @@ export function getConfiguredApiKey( } = {}, ): string | undefined { const env = options.env ?? process.env + if (env.COMMAND_CODE_API_KEY) return env.COMMAND_CODE_API_KEY if (env.COMMANDCODE_API_KEY) return env.COMMANDCODE_API_KEY const home = options.homeDir?.() ?? homedir() diff --git a/src/quota-command.ts b/src/quota-command.ts index 47f24d9..81a8779 100644 --- a/src/quota-command.ts +++ b/src/quota-command.ts @@ -45,7 +45,7 @@ export function registerCommandCodeQuota( const apiKey = pickCommandCodeApiKey(registryKey, getConfiguredKey()) if (!apiKey) { ctx.ui.notify( - "Command Code quota requires an API key. Run /login and select Command Code, or set COMMANDCODE_API_KEY.", + "Command Code quota requires an API key. Run /login and select Command Code, or set COMMAND_CODE_API_KEY.", "warning", ) return diff --git a/tests/test-api-key.ts b/tests/test-api-key.ts index a497978..f3029cb 100644 --- a/tests/test-api-key.ts +++ b/tests/test-api-key.ts @@ -21,10 +21,17 @@ async function withAuthFile( } describe("getConfiguredApiKey()", () => { - it("prefers the environment variable", () => { + it("prefers the official environment variable and keeps the legacy alias", () => { assert.equal( - getConfiguredApiKey({ env: { COMMANDCODE_API_KEY: "env-key" }, authPaths: [] }), - "env-key", + getConfiguredApiKey({ + env: { COMMAND_CODE_API_KEY: "official-key", COMMANDCODE_API_KEY: "legacy-key" }, + authPaths: [], + }), + "official-key", + ) + assert.equal( + getConfiguredApiKey({ env: { COMMANDCODE_API_KEY: "legacy-key" }, authPaths: [] }), + "legacy-key", ) }) diff --git a/tests/test-live-e2e.mjs b/tests/test-live-e2e.mjs index 88ca636..09d9d48 100644 --- a/tests/test-live-e2e.mjs +++ b/tests/test-live-e2e.mjs @@ -48,6 +48,7 @@ function findPiBinary() { function hasAuthMetadata() { return ( + Boolean(process.env.COMMAND_CODE_API_KEY) || Boolean(process.env.COMMANDCODE_API_KEY) || existsSync(join(homedir(), ".commandcode", "auth.json")) || existsSync(join(homedir(), ".pi", "agent", "auth.json")) @@ -70,6 +71,7 @@ function safeEnv(overrides = {}) { env.PI_CODING_AGENT_DIR = profileAgentDir env.COMMANDCODE_MODELS_CACHE = join(profileAgentDir, "commandcode-models.json") } else { + delete env.COMMAND_CODE_API_KEY delete env.COMMANDCODE_API_KEY } return env diff --git a/tests/test-omp-compat.mjs b/tests/test-omp-compat.mjs index eeb5793..ae64d1c 100644 --- a/tests/test-omp-compat.mjs +++ b/tests/test-omp-compat.mjs @@ -133,7 +133,7 @@ function runOmp(args, timeoutMs = 30_000) { HOME: tempHome, USERPROFILE: tempHome, PI_CODING_AGENT_DIR: join(tempHome, ".omp", "agent"), - COMMANDCODE_API_KEY: "mock-key", + COMMAND_CODE_API_KEY: "mock-key", COMMANDCODE_API_BASE: `${apiBase}/provider/v1`, COMMANDCODE_MODELS_URL: `${apiBase}/provider/v1/models`, }, diff --git a/tests/test-pi-authenticated.mjs b/tests/test-pi-authenticated.mjs index 7f992b3..39fa536 100644 --- a/tests/test-pi-authenticated.mjs +++ b/tests/test-pi-authenticated.mjs @@ -22,7 +22,7 @@ const { writeFileSync } = require("node:fs") writeFileSync(process.env.FAKE_PI_LOG, JSON.stringify({ args: process.argv.slice(2), agentDir: process.env.PI_CODING_AGENT_DIR ?? null, - apiKey: process.env.COMMANDCODE_API_KEY ?? null, + apiKey: process.env.COMMAND_CODE_API_KEY ?? process.env.COMMANDCODE_API_KEY ?? null, skipVersionCheck: process.env.PI_SKIP_VERSION_CHECK, })) NODE @@ -38,7 +38,8 @@ NODE PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ""}`, FAKE_PI_LOG: logPath, PI_CODING_AGENT_DIR: "/existing/pi-agent", - COMMANDCODE_API_KEY: "existing-key", + COMMAND_CODE_API_KEY: "official-existing-key", + COMMANDCODE_API_KEY: "legacy-existing-key", }, encoding: "utf8", }) diff --git a/tests/test-pi-isolated.mjs b/tests/test-pi-isolated.mjs index 0b809a6..3ca85c6 100644 --- a/tests/test-pi-isolated.mjs +++ b/tests/test-pi-isolated.mjs @@ -26,7 +26,8 @@ appendFileSync(process.env.FAKE_PI_LOG, JSON.stringify({ skipVersionCheck: process.env.PI_SKIP_VERSION_CHECK, home: process.env.HOME, userProfile: process.env.USERPROFILE, - inheritedApiKey: process.env.COMMANDCODE_API_KEY ?? null, + inheritedApiKey: + process.env.COMMAND_CODE_API_KEY ?? process.env.COMMANDCODE_API_KEY ?? null, }) + "\\n") NODE if [ "$1" = "install" ]; then exit 0; fi @@ -42,7 +43,8 @@ exit ${exitStatus} ...process.env, PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ""}`, FAKE_PI_LOG: logPath, - COMMANDCODE_API_KEY: "must-not-leak", + COMMAND_CODE_API_KEY: "must-not-leak-official", + COMMANDCODE_API_KEY: "must-not-leak-legacy", }, encoding: "utf8", }) diff --git a/tests/test-pi-local.mjs b/tests/test-pi-local.mjs index 2d0a999..3bce2c8 100644 --- a/tests/test-pi-local.mjs +++ b/tests/test-pi-local.mjs @@ -215,8 +215,8 @@ const env = { PI_CODING_AGENT_DIR: agentDir, PI_CODING_AGENT_SESSION_DIR: join(tempHome, "sessions"), COMMANDCODE_API_BASE: `${apiBase}/provider/v1`, - COMMANDCODE_API_KEY: "mock-key", - COMMANDCODE_ZDR: "1", + COMMAND_CODE_API_KEY: "mock-key", + CMD_ZDR: "1", COMMANDCODE_MODELS_URL: `${apiBase}/provider/v1/models`, } diff --git a/tests/test-quota-command.ts b/tests/test-quota-command.ts index 95692cd..3ae753c 100644 --- a/tests/test-quota-command.ts +++ b/tests/test-quota-command.ts @@ -68,7 +68,7 @@ describe("commandcode-quota command", () => { }) assert.ok(pi.handler) - const ctx = context("$COMMANDCODE_API_KEY") + const ctx = context("$COMMAND_CODE_API_KEY") await pi.handler("", ctx.value) assert.equal(ctx.waited(), true) assert.equal(requestKey, "fallback-key") diff --git a/tests/test-smoke.mjs b/tests/test-smoke.mjs index 2665710..483c00a 100644 --- a/tests/test-smoke.mjs +++ b/tests/test-smoke.mjs @@ -7,7 +7,7 @@ * 3. Can complete a simple prompt (requires Command Code auth) * * Run with: node tests/test-smoke.mjs - * Requires: pi on PATH plus COMMANDCODE_API_KEY or live pi auth files. + * Requires: pi on PATH plus COMMAND_CODE_API_KEY (or legacy COMMANDCODE_API_KEY) or live pi auth files. */ import { spawn } from "node:child_process" @@ -48,6 +48,7 @@ const RPC_QUERY_TIMEOUT = 60_000 function hasCommandCodeAuth() { return ( + !!process.env.COMMAND_CODE_API_KEY || !!process.env.COMMANDCODE_API_KEY || existsSync(join(homedir(), ".commandcode", "auth.json")) || existsSync(join(homedir(), ".pi", "agent", "auth.json")) From 804ba5862e2487a6d66206e1253c7c456ad0a31d Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 15:54:37 +0200 Subject: [PATCH 33/40] docs(core): document Command Code CLI parity --- CHANGELOG.md | 12 +++++++++--- README.md | 8 ++++---- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5730962..9188864 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,15 +2,21 @@ ## Unreleased -- Add a daily GitHub Actions synchronization job that opens or updates a pull request for CLI version, image capability, and reasoning-effort changes in the latest published Command Code catalog. -- Refresh static model capabilities from `command-code@1.32.2`, including new image and reasoning metadata. +- Add a daily GitHub Actions synchronization job that opens or updates a pull request for CLI version, image capability, reasoning, effort, and output-limit changes in the latest published Command Code catalog. +- Refresh static model capabilities from `command-code@1.32.2`, separating reasoning support from selectable effort levels and honoring model-specific output limits. +- Reject truncated, aborted, and network-failed generate streams instead of reporting partial responses as successful. +- Normalize malformed tool results and synthesize missing tool results so follow-up requests preserve valid tool-call history. +- Refresh display pricing for all 58 current models, including Gemini 3.7 Flash, Qwen 3.8 27B, Ox Alpha, Muse Spark 1.2, and Grok 4.6 long-context rates. +- Accept the official `COMMAND_CODE_API_KEY` and `CMD_ZDR` environment variables while retaining legacy aliases. +- Align generate request metadata with the CLI by forwarding stable session IDs, optional temperature, and the CLI user agent. +- Validate manually pasted API keys, use the CLI's two-minute browser timeout, and reject OAuth state mismatches without closing the callback server. - Add `/commandcode-quota` with live credits, plan, usage totals, and rolling-limit diagnostics from Command Code's alpha usage endpoints. - Add `zai-org/GLM-5.3` with its verified reasoning efforts and display pricing. - Prefer Command Code's Provider API (`/provider/v1/chat/completions` and `/provider/v1/messages`) and automatically fall back to the existing `/alpha/generate` transport only when the Provider API returns `403 upgrade_required` for a Go-plan account. - Remember the detected transport for the running process, re-detect it when credentials change, prevent stale in-flight requests from overwriting the new credential's transport, and never fall back for unrelated authentication, permission, rate-limit, network, or server failures. - Use Pi's native OpenAI- and Anthropic-compatible providers for Provider API streaming, including adaptive thinking for current reasoning-capable Claude models, while preserving the existing hardened generate transport, dynamic model discovery, offline cache, refresh/status commands, pricing, and OAuth credentials. - Let `/login` use browser authentication, an explicit API-key prompt, or a directly pasted API key. -- Add optional zero-data-retention headers through `COMMANDCODE_ZDR=1`. +- Add optional zero-data-retention headers through `CMD_ZDR=1` and the legacy `COMMANDCODE_ZDR=1` alias. - Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended, and display the current DeepSeek V4 off-peak rates for its time-dependent pricing. - Add isolated live E2E profiles for separate Go-plan and Provider-API credentials, including an explicit selected-transport assertion and packed-package validation. - Fix extension load failure on newer pi hosts that reject registering a custom API under a built-in name (`openai-completions`); register under `commandcode-custom` instead and restore the real wire API before native compat dispatch. diff --git a/README.md b/README.md index 83c8295..c85c829 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ If automatic transfer from the browser fails, copy the API key shown by Command ### Environment variable ```sh -export COMMANDCODE_API_KEY="user_..." +export COMMAND_CODE_API_KEY="user_..." ``` ### Auth file @@ -93,7 +93,7 @@ Open `/model` and select one of the models provided by Command Code. Model avail ### Reasoning support -Reasoning metadata is enriched only for models whose Command Code effort support is known. Those models register a model-specific `thinkingLevelMap`, so pi and OMP expose only supported levels. Pi's native OpenAI- and Anthropic-compatible providers translate the selected level for Provider API accounts; the existing Command Code generate transport sends the matching `reasoning_effort` for Go accounts. Unsupported levels and newly discovered models without metadata do not claim reasoning support. +Reasoning capability and selectable effort levels follow the official CLI catalog independently. Models can therefore be marked as reasoning-capable even when Command Code chooses their depth automatically. Models with explicit effort support also register a model-specific `thinkingLevelMap`, so pi and OMP expose only valid levels. Pi's native OpenAI- and Anthropic-compatible providers translate the selected level for Provider API accounts; the existing Command Code generate transport sends the matching `reasoning_effort` for Go accounts. List Command Code models from the terminal: @@ -133,7 +133,7 @@ While pi is running, use these provider commands without restarting: The `commandcode-quota` command reads from the Command Code alpha usage endpoints (the same ones the `cmd` CLI `/usage` command uses): `whoami`, `billing/credits`, `billing/subscriptions`, and `usage/summary`. It authenticates with the same API key the provider already uses. If the command cannot reach those endpoints or an endpoint schema changes, unavailable sections are reported explicitly instead of being displayed as zero usage. Output is plain text (via `ui.notify`) so it works across pi and compatible hosts such as OMP. -Set `COMMANDCODE_ZDR=1` to send Command Code's documented `x-cmd-zdr: 1` zero-data-retention header. +Set `CMD_ZDR=1` to send Command Code's documented `x-cmd-zdr: 1` zero-data-retention header. The legacy `COMMANDCODE_ZDR=1` alias remains supported. The following environment variables are intended for tests, local mocks, and compatible API endpoints: @@ -144,7 +144,7 @@ The following environment variables are intended for tests, local mocks, and com ## Image input -The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed. A daily GitHub Actions job synchronizes the CLI version, image capabilities, and reasoning efforts with the latest published CLI package and opens or updates a reviewable pull request when they change. Pricing remains manually reviewed because the CLI catalog does not expose every pricing tier and temporary promotion used by the provider. +The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed. A daily GitHub Actions job synchronizes the CLI version, image capabilities, reasoning flags, reasoning efforts, and model-specific output limits with the latest published CLI package and opens or updates a reviewable pull request when they change. Pricing remains manually reviewed because temporary promotions and long-context tiers require explicit review. For vision-capable models, Pi's native provider adapters forward image blocks from user messages and tool results using the documented OpenAI or Anthropic message schema. Unknown and text-only models remain marked text-only in Pi. From cee0d7cc963f6e52c3426ff49ff825104cc45168 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 16:37:24 +0200 Subject: [PATCH 34/40] test(e2e): cover live Go and GOAT accounts --- CHANGELOG.md | 2 +- CONTRIBUTING.md | 3 +- README.md | 11 ++- package.json | 3 +- scripts/live-e2e-profile.mjs | 22 ++++-- tests/test-live-e2e.mjs | 142 +++++++++++++++++++++++++++++++---- 6 files changed, 157 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9188864..db8014d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ - Let `/login` use browser authentication, an explicit API-key prompt, or a directly pasted API key. - Add optional zero-data-retention headers through `CMD_ZDR=1` and the legacy `COMMANDCODE_ZDR=1` alias. - Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended, and display the current DeepSeek V4 off-peak rates for its time-dependent pricing. -- Add isolated live E2E profiles for separate Go-plan and Provider-API credentials, including an explicit selected-transport assertion and packed-package validation. +- Add isolated live E2E profiles for separate Go-, GOAT-, and Provider-plan credentials, covering transport selection, reasoning across turns, quota identity, aborts, tools, GOAT vision, Go image rejection, and packed-package validation. - Fix extension load failure on newer pi hosts that reject registering a custom API under a built-in name (`openai-completions`); register under `commandcode-custom` instead and restore the real wire API before native compat dispatch. ## 0.5.1 - 2026-08-11 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6adb7bb..0a6b84f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,10 +46,11 @@ Run the transport-specific live tests with separate credentials: ```sh COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key npm run test:e2e:live:go +COMMANDCODE_E2E_GOAT_API_KEY_FILE=/path/to/goat-key npm run test:e2e:live:goat COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key npm run test:e2e:live:provider ``` -Use `npm run test:e2e:live:all` with both file variables to run them sequentially. Store the keys in a secret manager and export each one to a new mode-`0600` temporary file for the test; never add key files to the repository. The direct `COMMANDCODE_E2E_GO_API_KEY` and `COMMANDCODE_E2E_PROVIDER_API_KEY` variables are intended primarily for protected CI secrets. +Use `npm run test:e2e:live:all` with the Go and GOAT file variables to run both subscription transports sequentially. Store keys in a secret manager and export each one to a new mode-`0600` temporary file for the test; never add key files to the repository. Direct `*_API_KEY` variables are intended primarily for protected CI secrets. Before opening a PR, run: diff --git a/README.md b/README.md index c85c829..637e497 100644 --- a/README.md +++ b/README.md @@ -195,23 +195,26 @@ Both commands accept additional pi arguments after `--`, for example `npm run pi ### Live transport tests -Keep the Go-plan and Provider-API test keys in separate secret-manager entries. Pass them through protected files so the keys do not enter shell history: +Keep Go-, GOAT-, and optional Provider-plan test keys in separate secret-manager entries. Pass them through protected files so the keys do not enter shell history: ```sh COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \ npm run test:e2e:live:go +COMMANDCODE_E2E_GOAT_API_KEY_FILE=/path/to/goat-key \ + npm run test:e2e:live:goat + COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key \ npm run test:e2e:live:provider COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \ -COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key \ +COMMANDCODE_E2E_GOAT_API_KEY_FILE=/path/to/goat-key \ npm run test:e2e:live:all ``` -Each profile runs with an isolated Pi agent directory and asserts the selected transport through `/commandcode-status`: Go must select `generate`, while a Provider API account must select `provider`. The profile-specific `*_API_KEY` environment variables are also supported for CI secrets, but key files are preferred for local use. +Each profile runs with an isolated Pi agent directory and asserts transport selection, reasoning across turns, quota plan identity, abort handling, tool calls, and the packed npm artifact. Go must select `generate` and reject unsupported images; GOAT must select `provider` and complete a live vision request. The profile-specific `*_API_KEY` environment variables are also supported for CI secrets, but key files are preferred for local use. -Override the default DeepSeek test model with `COMMANDCODE_E2E_GO_MODEL` or `COMMANDCODE_E2E_PROVIDER_MODEL`. A successful live Anthropic `/provider/v1/messages` test requires a Provider API account whose plan includes the selected Claude model. +The Go profile defaults to DeepSeek V4 Flash; GOAT defaults to Grok 4.6 because its Provider API stream exposes reasoning consistently across consecutive turns. Override them with `COMMANDCODE_E2E_GO_MODEL`, `COMMANDCODE_E2E_GOAT_MODEL`, or `COMMANDCODE_E2E_PROVIDER_MODEL`. A successful live Anthropic `/provider/v1/messages` test requires a paid account whose plan includes the selected Claude model. See [CONTRIBUTING.md](CONTRIBUTING.md) for local setup and tests. See [RELEASE.md](RELEASE.md) for the release process. diff --git a/package.json b/package.json index 660b6c1..568cdb3 100644 --- a/package.json +++ b/package.json @@ -55,8 +55,9 @@ "test:smoke": "node tests/test-smoke.mjs", "test:e2e:live": "node tests/test-live-e2e.mjs", "test:e2e:live:go": "node scripts/live-e2e-profile.mjs go", + "test:e2e:live:goat": "node scripts/live-e2e-profile.mjs goat", "test:e2e:live:provider": "node scripts/live-e2e-profile.mjs provider", - "test:e2e:live:all": "node scripts/live-e2e-profile.mjs go provider", + "test:e2e:live:all": "node scripts/live-e2e-profile.mjs go goat", "test:cost": "tsx tests/test-cost.ts" }, "pi": { diff --git a/scripts/live-e2e-profile.mjs b/scripts/live-e2e-profile.mjs index 3271323..b2cca8d 100644 --- a/scripts/live-e2e-profile.mjs +++ b/scripts/live-e2e-profile.mjs @@ -11,14 +11,19 @@ const profiles = process.argv.slice(2) if ( profiles.length === 0 || - profiles.some((profile) => profile !== "go" && profile !== "provider") + profiles.some((profile) => profile !== "go" && profile !== "goat" && profile !== "provider") ) { - console.error("Usage: node scripts/live-e2e-profile.mjs [go|provider]") + console.error("Usage: node scripts/live-e2e-profile.mjs [go|goat|provider]") process.exit(2) } async function credentialFor(profile) { - const prefix = profile === "go" ? "COMMANDCODE_E2E_GO" : "COMMANDCODE_E2E_PROVIDER" + const prefix = + profile === "go" + ? "COMMANDCODE_E2E_GO" + : profile === "goat" + ? "COMMANDCODE_E2E_GOAT" + : "COMMANDCODE_E2E_PROVIDER" const direct = process.env[`${prefix}_API_KEY`]?.trim() const file = process.env[`${prefix}_API_KEY_FILE`] @@ -35,8 +40,14 @@ async function credentialFor(profile) { function runProfile(profile, apiKey) { const modelVariable = - profile === "go" ? "COMMANDCODE_E2E_GO_MODEL" : "COMMANDCODE_E2E_PROVIDER_MODEL" - const model = process.env[modelVariable] ?? "deepseek/deepseek-v4-flash" + profile === "go" + ? "COMMANDCODE_E2E_GO_MODEL" + : profile === "goat" + ? "COMMANDCODE_E2E_GOAT_MODEL" + : "COMMANDCODE_E2E_PROVIDER_MODEL" + const model = + process.env[modelVariable] ?? + (profile === "goat" ? "xai/grok-4.6" : "deepseek/deepseek-v4-flash") const env = { ...process.env, COMMAND_CODE_API_KEY: apiKey, @@ -45,6 +56,7 @@ function runProfile(profile, apiKey) { } delete env.COMMANDCODE_API_KEY delete env.COMMANDCODE_E2E_GO_API_KEY + delete env.COMMANDCODE_E2E_GOAT_API_KEY delete env.COMMANDCODE_E2E_PROVIDER_API_KEY return new Promise((resolveRun, reject) => { diff --git a/tests/test-live-e2e.mjs b/tests/test-live-e2e.mjs index 09d9d48..f17ffa5 100644 --- a/tests/test-live-e2e.mjs +++ b/tests/test-live-e2e.mjs @@ -27,7 +27,20 @@ const extensionPath = join(projectDir, "index.ts") const testModel = process.env.COMMANDCODE_E2E_MODEL ?? "deepseek/deepseek-v4-flash" const testProfile = process.env.COMMANDCODE_E2E_PROFILE const expectedTransport = - testProfile === "go" ? "generate" : testProfile === "provider" ? "provider" : undefined + testProfile === "go" + ? "generate" + : testProfile === "goat" || testProfile === "provider" + ? "provider" + : undefined +const expectedPlan = + testProfile === "go" + ? "go" + : testProfile === "goat" + ? "goat" + : testProfile === "provider" + ? "provider" + : undefined +const goatVisionModel = process.env.COMMANDCODE_E2E_GOAT_VISION_MODEL ?? "google/gemini-3.7-flash" const marker = "commandcode-live-e2e-ok" function findPiBinary() { @@ -104,7 +117,7 @@ function run(command, args, options = {}) { }) } -async function runRpc(extension, action, timeoutMs = 120_000) { +async function runRpc(extension, action, timeoutMs = 120_000, model = testModel) { const child = spawn( piBin, [ @@ -116,7 +129,9 @@ async function runRpc(extension, action, timeoutMs = 120_000) { "--provider", "commandcode", "--model", - testModel, + model, + "--thinking", + "high", ], { cwd: projectDir, env: safeEnv(), stdio: ["pipe", "pipe", "pipe"] }, ) @@ -224,8 +239,19 @@ try { await waitFor( (event) => event.type === "response" && event.id === "reasoning-turn-1" && event.success, ) - await waitFor((event) => event.type === "agent_settled") - const firstThinkingDeltas = countThinkingDeltas(firstStart) + const firstSettled = await waitFor( + (event) => event.type === "agent_settled" && events.indexOf(event) >= firstStart, + ) + const firstSettledIndex = events.indexOf(firstSettled) + const firstThinkingDeltas = events + .slice(firstStart, firstSettledIndex + 1) + .filter( + (event) => + event.type === "message_update" && + event.assistantMessageEvent?.type === "thinking_delta" && + typeof event.assistantMessageEvent.delta === "string" && + event.assistantMessageEvent.delta.length > 0, + ).length const secondStart = events.length send({ @@ -237,15 +263,16 @@ try { await waitFor( (event) => event.type === "response" && event.id === "reasoning-turn-2" && event.success, ) - await waitFor((event) => event.type === "agent_settled" && events.indexOf(event) >= secondStart) + const secondSettled = await waitFor( + (event) => event.type === "agent_settled" && events.indexOf(event) >= secondStart, + ) const secondThinkingDeltas = countThinkingDeltas(secondStart) + assert.ok(events.indexOf(secondSettled) >= secondStart) return { firstThinkingDeltas, secondThinkingDeltas, stderr: getStderr() } }) - if (testProfile !== "provider") { - assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning") - assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning") - } + assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning") + assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning") assert.doesNotMatch(multiTurn.stderr, /Bearer\s+\S+/i) console.log("[live-e2e] live runtime refresh/status commands") @@ -283,15 +310,66 @@ try { typeof event.message === "string" && event.message.includes("source:"), ) - return { names, refresh: refresh.message, status: status.message, stderr: getStderr() } + + send({ id: "quota", type: "prompt", message: "/commandcode-quota" }) + await waitFor((event) => event.type === "response" && event.id === "quota" && event.success) + const quota = await waitFor( + (event) => + event.type === "extension_ui_request" && + event.method === "notify" && + typeof event.message === "string" && + event.message.includes("Plan:"), + ) + return { + names, + refresh: refresh.message, + status: status.message, + quota: quota.message, + stderr: getStderr(), + } }) assert.ok(runtime.names.includes("commandcode-refresh")) assert.ok(runtime.names.includes("commandcode-status")) + assert.ok(runtime.names.includes("commandcode-quota")) assert.match(runtime.refresh, /model catalog (?:refreshed|unchanged)/) if (expectedTransport) assert.match(runtime.status, new RegExp(`transport: ${expectedTransport}`)) assert.match(runtime.status, /source: (?:live|cache)/) assert.match(runtime.status, /model count: [1-9][0-9]*/) - assert.doesNotMatch(`${runtime.refresh}\n${runtime.status}\n${runtime.stderr}`, /Bearer\s+\S+/i) + if (expectedPlan) assert.match(runtime.quota, new RegExp(`Plan:.*\\b${expectedPlan}\\b`, "i")) + assert.doesNotMatch( + `${runtime.refresh}\n${runtime.status}\n${runtime.quota}\n${runtime.stderr}`, + /Bearer\s+\S+/i, + ) + + console.log("[live-e2e] live abort through real RPC host") + const abortResult = await runRpc(extensionPath, async ({ send, waitFor, events, getStderr }) => { + const startIndex = events.length + send({ + id: "abort-turn", + type: "prompt", + message: "Write a very long detailed explanation of every integer from 1 to 10000.", + }) + await waitFor( + (event) => event.type === "response" && event.id === "abort-turn" && event.success, + ) + await waitFor((event) => event.type === "message_update" && events.indexOf(event) >= startIndex) + send({ id: "abort", type: "abort" }) + await waitFor((event) => event.type === "response" && event.id === "abort" && event.success) + await waitFor((event) => event.type === "agent_settled" && events.indexOf(event) >= startIndex) + return { + aborted: events + .slice(startIndex) + .some( + (event) => + event.type === "message_end" && + event.message?.role === "assistant" && + event.message?.stopReason === "aborted", + ), + stderr: getStderr(), + } + }) + assert.equal(abortResult.aborted, true) + assert.doesNotMatch(abortResult.stderr, /Bearer\s+\S+/i) console.log("[live-e2e] live tool-call round trip") const toolRoot = join(tempRoot, "tool-roundtrip") @@ -319,9 +397,45 @@ try { ) assert.equal(toolResult.code, 0, toolResult.stderr) assert.match(toolResult.stdout, new RegExp(marker)) - assert.equal(readFileSync(targetPath, "utf-8"), marker) + assert.equal(readFileSync(targetPath, "utf-8").trimEnd(), marker) - if (testProfile !== "provider") { + if (testProfile === "goat") { + console.log("[live-e2e] live vision request through Provider API") + const vision = await runRpc( + extensionPath, + async ({ send, waitFor, events, getStderr }) => { + const startIndex = events.length + send({ + id: "vision", + type: "prompt", + message: "Describe the attached image briefly.", + images: [ + { + type: "image", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", + mimeType: "image/png", + }, + ], + }) + await waitFor( + (event) => event.type === "response" && event.id === "vision" && event.success, + ) + await waitFor( + (event) => event.type === "agent_settled" && events.indexOf(event) >= startIndex, + ) + const messageEnd = events + .slice(startIndex) + .find((event) => event.type === "message_end" && event.message?.role === "assistant") + return { messageEnd, stderr: getStderr() } + }, + 180_000, + goatVisionModel, + ) + assert.notEqual(vision.messageEnd?.message?.stopReason, "error") + assert.doesNotMatch(vision.stderr, /Bearer\s+\S+/i) + } + + if (testProfile === "go") { console.log("[live-e2e] image rejection through real RPC host") const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => { send({ From f8976adb11863a7feffa7f4a42cc94109dfa9915 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 16:40:45 +0200 Subject: [PATCH 35/40] ci(security): allow documented Command Code env vars --- .semgrep/pi-extension-audit.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.semgrep/pi-extension-audit.yaml b/.semgrep/pi-extension-audit.yaml index b56c540..750ebf6 100644 --- a/.semgrep/pi-extension-audit.yaml +++ b/.semgrep/pi-extension-audit.yaml @@ -89,8 +89,6 @@ rules: - "**/*.ts" - "**/index.ts" - - # ──────────────────────────────────────────────────────────────────────── # Supply chain: dependency injection # ──────────────────────────────────────────────────────────────────────── @@ -209,8 +207,8 @@ rules: - id: pi-extension-unusual-import patterns: - pattern-either: - - pattern: "import $X from \"...\"" - - pattern: "const $X = require(\"...\")" + - pattern: 'import $X from "..."' + - pattern: 'const $X = require("...")' - metavariable-regex: metavariable: $X regex: "(compression|pako|zlib|tar|stream|archiver|request|axios|needle|got|superagent|node-fetch|undici)" @@ -256,10 +254,10 @@ rules: - pattern: process.env.$VAR - metavariable-regex: metavariable: $VAR - regex: "(?!COMMANDCODE_|NODE_|PATH|HOME|SHELL|USER|LANG|LC_|TERM|TMPDIR|NIX_).*" + regex: "(?!COMMANDCODE_|COMMAND_CODE_|CMD_ZDR|NODE_|PATH|HOME|SHELL|USER|LANG|LC_|TERM|TMPDIR|NIX_).*" message: > Reading unexpected environment variable $VAR. Provider should only - read COMMANDCODE_* variables. + read documented Command Code or standard runtime variables. severity: WARNING languages: [javascript, typescript] paths: @@ -270,4 +268,3 @@ rules: # ──────────────────────────────────────────────────────────────────────── # OAuth flow manipulation # ──────────────────────────────────────────────────────────────────────── - From dceec74e35fd6ee157a5bda3c77479ac7739abcd Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 16:41:23 +0200 Subject: [PATCH 36/40] style(ci): keep semgrep config focused --- .semgrep/pi-extension-audit.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.semgrep/pi-extension-audit.yaml b/.semgrep/pi-extension-audit.yaml index 750ebf6..120a0ec 100644 --- a/.semgrep/pi-extension-audit.yaml +++ b/.semgrep/pi-extension-audit.yaml @@ -89,6 +89,8 @@ rules: - "**/*.ts" - "**/index.ts" + + # ──────────────────────────────────────────────────────────────────────── # Supply chain: dependency injection # ──────────────────────────────────────────────────────────────────────── @@ -207,8 +209,8 @@ rules: - id: pi-extension-unusual-import patterns: - pattern-either: - - pattern: 'import $X from "..."' - - pattern: 'const $X = require("...")' + - pattern: "import $X from \"...\"" + - pattern: "const $X = require(\"...\")" - metavariable-regex: metavariable: $X regex: "(compression|pako|zlib|tar|stream|archiver|request|axios|needle|got|superagent|node-fetch|undici)" From ad307c93fdf3af2b747ce98eb648463ffd5be6eb Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 16:56:19 +0200 Subject: [PATCH 37/40] fix(stream): remove redundant completion guard --- src/core.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/core.ts b/src/core.ts index c88cef4..134ee2a 100644 --- a/src/core.ts +++ b/src/core.ts @@ -709,12 +709,6 @@ export function createStreamCommandCode(deps: CoreDependencies) { throw streamError } - if (!finished) { - throw new Error( - "Stream ended unexpectedly before completion (no finish event) — response was truncated", - ) - } - // Stream completed successfully. endTextBlock() endThinking() From a624643d900273445f058d9c7bded8e7b01af553 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 16:58:29 +0200 Subject: [PATCH 38/40] fix(stream): forward incremental tool-call arguments --- CHANGELOG.md | 1 + src/core.ts | 82 +++++++++++++++++++++++++++++++----- src/types.ts | 6 +++ tests/test-stream.ts | 98 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5730962..66b71dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Stream incremental tool-call arguments from the `/alpha/generate` transport instead of waiting for the final complete tool-call event. - Add a daily GitHub Actions synchronization job that opens or updates a pull request for CLI version, image capability, and reasoning-effort changes in the latest published Command Code catalog. - Refresh static model capabilities from `command-code@1.32.2`, including new image and reasoning metadata. - Add `/commandcode-quota` with live credits, plan, usage totals, and rolling-limit diagnostics from Command Code's alpha usage endpoints. diff --git a/src/core.ts b/src/core.ts index 57ed455..a988a5d 100644 --- a/src/core.ts +++ b/src/core.ts @@ -286,6 +286,10 @@ export function createStreamCommandCode(deps: CoreDependencies) { let textBlock: TextContent | undefined let currentTextIdx = -1 let thinkingIdx = -1 + const streamingToolCalls = new Map< + string, + { contentIndex: number; toolCall: ToolCallContent; partialArgs: string } + >() let finished = false const abortUpstream = () => { @@ -398,25 +402,81 @@ export function createStreamCommandCode(deps: CoreDependencies) { break } + case "tool-input-start": { + endTextBlock() + endThinking() + const id = stringValue(event.id) + if (!id || streamingToolCalls.has(id)) break + + const toolCall: ToolCallContent = { + type: "toolCall", + id, + name: stringValue(event.toolName) ?? "", + arguments: {}, + } + output.content.push(toolCall) + const contentIndex = output.content.length - 1 + streamingToolCalls.set(id, { contentIndex, toolCall, partialArgs: "" }) + stream.push({ + type: "toolcall_start", + contentIndex, + partial: output, + }) + break + } + + case "tool-input-delta": { + const id = stringValue(event.id) + const delta = stringValue(event.delta) + if (!id || delta === undefined) break + const active = streamingToolCalls.get(id) + if (!active) break + + active.partialArgs += delta + active.toolCall.arguments = recordOrEmpty(active.partialArgs) + stream.push({ + type: "toolcall_delta", + contentIndex: active.contentIndex, + delta, + partial: output, + }) + break + } + + case "tool-input-end": { + break + } + case "tool-call": { endTextBlock() endThinking() - const toolCall: ToolCallContent = { + const id = stringValue(event.toolCallId) ?? "" + const active = streamingToolCalls.get(id) + const toolCall: ToolCallContent = active?.toolCall ?? { type: "toolCall", - id: stringValue(event.toolCallId) ?? "", + id, name: stringValue(event.toolName) ?? "", - arguments: recordOrEmpty(event.input ?? event.args ?? event.arguments), + arguments: {}, + } + toolCall.name = stringValue(event.toolName) ?? toolCall.name + toolCall.arguments = recordOrEmpty(event.input ?? event.args ?? event.arguments) + + let contentIndex: number + if (active) { + contentIndex = active.contentIndex + streamingToolCalls.delete(id) + } else { + output.content.push(toolCall) + contentIndex = output.content.length - 1 + stream.push({ + type: "toolcall_start", + contentIndex, + partial: output, + }) } - output.content.push(toolCall) - const idx = output.content.length - 1 - stream.push({ - type: "toolcall_start", - contentIndex: idx, - partial: output, - }) stream.push({ type: "toolcall_end", - contentIndex: idx, + contentIndex, toolCall, partial: output, }) diff --git a/src/types.ts b/src/types.ts index dc680ea..2688976 100644 --- a/src/types.ts +++ b/src/types.ts @@ -172,6 +172,12 @@ export type AssistantMessageEvent = contentIndex: number partial: AssistantMessageLike } + | { + type: "toolcall_delta" + contentIndex: number + delta: string + partial: AssistantMessageLike + } | { type: "toolcall_end" contentIndex: number diff --git a/tests/test-stream.ts b/tests/test-stream.ts index a015e75..a54bd85 100644 --- a/tests/test-stream.ts +++ b/tests/test-stream.ts @@ -408,6 +408,104 @@ describe("streamCommandCode — successful streams", () => { assert.equal(toolCall?.type === "toolCall" ? toolCall.name : "", "read_file") }) + it("streams incremental tool-call arguments from generate events", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ + type: "tool-input-start", + id: "call_1", + toolName: "read_file", + }), + JSON.stringify({ type: "tool-input-delta", id: "call_1", delta: '{"path":"' }), + JSON.stringify({ type: "tool-input-delta", id: "call_1", delta: '/tmp/x"}' }), + JSON.stringify({ type: "tool-input-end", id: "call_1" }), + JSON.stringify({ + type: "tool-call", + toolCallId: "call_1", + toolName: "read_file", + input: { path: "/tmp/x" }, + }), + JSON.stringify({ type: "finish", finishReason: "tool-calls" }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + assert.deepEqual(eventTypes(events), [ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_delta", + "toolcall_end", + "done", + ]) + const deltas = events.flatMap((event) => (event.type === "toolcall_delta" ? [event.delta] : [])) + assert.deepEqual(deltas, ['{"path":"', '/tmp/x"}']) + + const done = events.at(-1) + if (done?.type !== "done") throw new Error("expected done") + assert.equal(done.reason, "toolUse") + const toolCall = done.message.content[0] + assert.equal(toolCall?.type, "toolCall") + if (toolCall?.type !== "toolCall") throw new Error("expected tool call") + assert.equal(toolCall.id, "call_1") + assert.equal(toolCall.name, "read_file") + assert.deepEqual(toolCall.arguments, { path: "/tmp/x" }) + }) + + it("keeps concurrent incremental tool calls separate", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ type: "tool-input-start", id: "call_1", toolName: "read_file" }), + JSON.stringify({ type: "tool-input-start", id: "call_2", toolName: "read_file" }), + JSON.stringify({ type: "tool-input-delta", id: "call_1", delta: '{"path":"/a"}' }), + JSON.stringify({ type: "tool-input-delta", id: "call_2", delta: '{"path":"/b"}' }), + JSON.stringify({ + type: "tool-call", + toolCallId: "call_2", + toolName: "read_file", + input: { path: "/b" }, + }), + JSON.stringify({ + type: "tool-call", + toolCallId: "call_1", + toolName: "read_file", + input: { path: "/a" }, + }), + JSON.stringify({ type: "finish", finishReason: "tool-calls" }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const starts = events.flatMap((event) => + event.type === "toolcall_start" ? [event.contentIndex] : [], + ) + const deltas = events.flatMap((event) => + event.type === "toolcall_delta" ? [[event.contentIndex, event.delta] as const] : [], + ) + const ends = events.flatMap((event) => + event.type === "toolcall_end" ? [[event.contentIndex, event.toolCall.id] as const] : [], + ) + assert.deepEqual(starts, [0, 1]) + assert.deepEqual(deltas, [ + [0, '{"path":"/a"}'], + [1, '{"path":"/b"}'], + ]) + assert.deepEqual(ends, [ + [1, "call_2"], + [0, "call_1"], + ]) + }) + it("flushes reasoning if finish arrives without reasoning-end", async () => { server.mockResponse({ type: "success", From 45281d693e6c08aa3ce9f4592cdece483cebdee6 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 17:46:41 +0200 Subject: [PATCH 39/40] fix(core): omit historical images for text models --- CHANGELOG.md | 1 + src/converters.ts | 18 +++++----- tests/test-pure-functions.ts | 65 ++++++++++++++++++++++++++++++------ tests/test-stream.ts | 30 ++++++++++++----- 4 files changed, 87 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fc4d81..c2030d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Allow switching from a vision-capable model to a text-only model by omitting historical image tool results while preserving their text output; direct image prompts still fail clearly. - Stream incremental tool-call arguments from the `/alpha/generate` transport instead of waiting for the final complete tool-call event. - Add a daily GitHub Actions synchronization job that opens or updates a pull request for CLI version, image capability, reasoning, effort, and output-limit changes in the latest published Command Code catalog. - Refresh static model capabilities from `command-code@1.32.2`, separating reasoning support from selectable effort levels and honoring model-specific output limits. diff --git a/src/converters.ts b/src/converters.ts index 1e7ea97..6cf71ac 100644 --- a/src/converters.ts +++ b/src/converters.ts @@ -66,9 +66,8 @@ function imageContentError(role: string): Error { export function assertTextOnlyMessages(messages?: readonly MessageLike[]): void { for (const message of messages ?? []) { - if (imageParts(message.content).length > 0) { - const role = message.role === "toolResult" ? "tool results" : `${message.role} messages` - throw imageContentError(role) + if (message.role !== "toolResult" && imageParts(message.content).length > 0) { + throw imageContentError(`${message.role} messages`) } } } @@ -268,6 +267,11 @@ export function messagesToCC( if (missingResults.length > 0) out.push({ role: "tool", content: missingResults }) } else if (message.role === "toolResult") { if (!message.toolCallId || !callIds.has(message.toolCallId)) continue + const images = imageParts(message.content) + const text = textContent(message) + const outputText = + text || + (images.length > 0 && !allowImages ? "[Image omitted: model does not support images]" : "") out.push({ role: "tool", content: [ @@ -276,15 +280,13 @@ export function messagesToCC( toolCallId: message.toolCallId, toolName: message.toolName, output: message.isError - ? { type: "error-text", value: textContent(message) } - : { type: "text", value: textContent(message) }, + ? { type: "error-text", value: outputText } + : { type: "text", value: outputText }, }, ], }) - const images = imageParts(message.content) - if (images.length > 0) { - if (!allowImages) throw imageContentError("tool results") + if (images.length > 0 && allowImages) { out.push({ role: "user", content: images.map(imageToCommandCode), diff --git a/tests/test-pure-functions.ts b/tests/test-pure-functions.ts index 17a4c78..96e742a 100644 --- a/tests/test-pure-functions.ts +++ b/tests/test-pure-functions.ts @@ -161,7 +161,7 @@ describe("projectSlugFromPath()", () => { }) describe("text-only image handling", () => { - it("rejects image content for models without image support", () => { + it("rejects direct image input for models without image support", () => { assert.throws( () => assertTextOnlyMessages([ @@ -172,16 +172,17 @@ describe("text-only image handling", () => { ]), /does not support image content/i, ) - assert.throws( - () => - assertTextOnlyMessages([ - { - role: "toolResult", - toolCallId: "c1", - content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], - }, - ]), - /does not support image content/i, + }) + + it("allows historical tool-result images to be omitted for text-only models", () => { + assert.doesNotThrow(() => + assertTextOnlyMessages([ + { + role: "toolResult", + toolCallId: "c1", + content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], + }, + ]), ) }) }) @@ -597,6 +598,48 @@ describe("messagesToCC()", () => { ) }) + it("omits tool-result images for text-only models while preserving their text", () => { + const result = messagesToCC([ + { role: "user", content: "read image" }, + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: [ + { type: "text", text: "image attached" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }, + ], + }, + ]) + + assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "image attached") + assert.equal(objectAt(result, ["3"]), undefined) + }) + + it("describes an omitted image-only tool result for text-only models", () => { + const result = messagesToCC([ + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }], + }, + ]) + + assert.equal( + objectAt(result, ["1", "content", "0", "output", "value"]), + "[Image omitted: model does not support images]", + ) + }) + it("preserves tool-result images as a following user image message", () => { const result = messagesToCC( [ diff --git a/tests/test-stream.ts b/tests/test-stream.ts index ea92508..a639980 100644 --- a/tests/test-stream.ts +++ b/tests/test-stream.ts @@ -240,12 +240,16 @@ describe("streamCommandCode — successful streams", () => { ) }) - it("rejects a tool-result image before network access for text-only models", async () => { + it("omits a historical tool-result image after switching to a text-only model", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) const events = await collectEvents( streamCommandCode( - makeModel({ id: "deepseek/deepseek-v4-pro" }), + makeModel({ id: "deepseek/deepseek-v4-flash" }), makeContext({ messages: [ { role: "user", content: "read the image" }, @@ -257,19 +261,29 @@ describe("streamCommandCode — successful streams", () => { role: "toolResult", toolCallId: "c1", toolName: "read", - content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }], + content: [ + { type: "text", text: "image attached" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, + ], }, + { role: "user", content: "continue without the image" }, ], }), { apiKey: "mock-key" }, ), ) - const lastEvent = events.at(-1) - assert.equal(lastEvent?.type, "error") - if (lastEvent?.type !== "error") throw new Error("expected error event") - assert.match(lastEvent.error.errorMessage ?? "", /does not support image content/i) - assert.equal(server.requestCount(), 0) + assert.equal(events.at(-1)?.type, "done") + assert.equal(server.requestCount(), 1) + const body = server.lastRequestBody() + assert.equal( + objectAt(body, ["params", "messages", "2", "content", "0", "output", "value"]), + "image attached", + ) + assert.equal( + objectAt(body, ["params", "messages", "3", "content"]), + "continue without the image", + ) }) it("rejects images before network access for text-only models", async () => { From f2bfb389967f40c142ac8ef999bc5f86949873c0 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 18:06:46 +0200 Subject: [PATCH 40/40] chore(release): prepare 0.6.0 --- CHANGELOG.md | 9 +++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2030d1..db107f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## Unreleased +## 0.6.0 - 2026-08-25 + - Allow switching from a vision-capable model to a text-only model by omitting historical image tool results while preserving their text output; direct image prompts still fail clearly. - Stream incremental tool-call arguments from the `/alpha/generate` transport instead of waiting for the final complete tool-call event. - Add a daily GitHub Actions synchronization job that opens or updates a pull request for CLI version, image capability, reasoning, effort, and output-limit changes in the latest published Command Code catalog. @@ -23,6 +25,13 @@ - Add isolated live E2E profiles for separate Go-, GOAT-, and Provider-plan credentials, covering transport selection, reasoning across turns, quota identity, aborts, tools, GOAT vision, Go image rejection, and packed-package validation. - Fix extension load failure on newer pi hosts that reject registering a custom API under a built-in name (`openai-completions`); register under `commandcode-custom` instead and restore the real wire API before native compat dispatch. +### Contributors + +- @jagaliano — added the live quota dashboard and hardened its integration. +- @omariqbalnaru — fixed custom API registration for Oh My Pi 17.4.0. +- @ThomasByr — added GLM-5.3 pricing and reasoning levels. +- @newCman1 — added DeepSeek V4 vision model support. + ## 0.5.1 - 2026-08-11 - Add model-specific image input capabilities from the `command-code@1.15.1` catalog and forward user and tool-result images using the current Command Code wire format. diff --git a/package-lock.json b/package-lock.json index e58b000..126d407 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-commandcode-provider", - "version": "0.5.1", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-commandcode-provider", - "version": "0.5.1", + "version": "0.6.0", "license": "MIT", "devDependencies": { "@types/node": "25.6.0", diff --git a/package.json b/package.json index 568cdb3..9293ffb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-commandcode-provider", - "version": "0.5.1", + "version": "0.6.0", "description": "pi custom provider for Command Code API (commandcode.ai)", "type": "module", "keywords": [