diff --git a/.github/scripts/check-commandcode-model-metadata.ts b/.github/scripts/check-commandcode-model-metadata.ts new file mode 100644 index 0000000..b83c386 --- /dev/null +++ b/.github/scripts/check-commandcode-model-metadata.ts @@ -0,0 +1,519 @@ +import { execFile } from "node:child_process" +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { pathToFileURL } from "node:url" +import { promisify } from "node:util" + +import { + COMMAND_CODE_CLI_VERSION, + MODEL_EFFORTS, + MODEL_INPUT_MODALITIES, + MODEL_MAX_OUTPUT_TOKENS, + MODEL_REASONING, +} from "../../src/commandcode-catalog.ts" + +const execFileAsync = promisify(execFile) +const MODELS_REFERENCE_PATH = "dist/bundled/command-code-knowledge/reference/models.md" +const CLI_BUNDLE_PATH = "dist/cli.mjs" +const TEXT_ONLY_MARKER = ',__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' +const VALID_EFFORTS = new Set(["minimal", "low", "medium", "high", "xhigh", "max"]) +const CATALOG_SOURCE_PATH = new URL("../../src/commandcode-catalog.ts", import.meta.url) +const README_PATH = new URL("../../README.md", import.meta.url) + +export interface CommandCodeModelMetadata { + imageModelIds: readonly string[] + reasoningModelIds: readonly string[] + reasoningEfforts: Readonly> + maxOutputTokens: Readonly> +} + +export interface ModelMetadataDiff { + versionChanged: boolean + addedImageModelIds: readonly string[] + removedImageModelIds: readonly string[] + addedReasoningModelIds: readonly string[] + removedReasoningModelIds: readonly string[] + addedEffortModelIds: readonly string[] + removedEffortModelIds: readonly string[] + changedEffortModelIds: readonly string[] + addedMaxOutputModelIds: readonly string[] + removedMaxOutputModelIds: readonly string[] + changedMaxOutputModelIds: readonly string[] +} + +interface PackedPackage { + filename: string +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((entry) => typeof entry === "string") +} + +function sorted(values: Iterable): string[] { + return [...values].sort((left, right) => left.localeCompare(right)) +} + +function parsePackedPackage(value: unknown): PackedPackage { + if (!Array.isArray(value) || value.length !== 1 || !isRecord(value[0])) { + throw new Error("Expected npm pack to return one package") + } + + const filename = value[0].filename + if (typeof filename !== "string" || filename.length === 0) { + throw new Error("Expected npm pack to return a tarball filename") + } + + return { filename } +} + +export function parsePackageVersion(value: unknown): string { + if (typeof value !== "string" || !/^\d+\.\d+\.\d+(?:[-+].+)?$/.test(value)) { + throw new Error("Expected npm view to return one semantic version") + } + return value +} + +export function parseModelsReference(markdown: string): { + modelIds: readonly string[] + reasoningEfforts: Readonly> +} { + const modelIds = new Set() + const reasoningEfforts: Record = {} + + for (const line of markdown.split("\n")) { + const match = /^\| `([^`]+)` \| [^|]* \| [^|]* \| ([^|]*) \|/.exec(line) + if (!match) continue + + const modelId = match[1] + const effortsColumn = match[2]?.trim() + if (!modelId || !effortsColumn) throw new Error(`Could not parse model row: ${line}`) + if (modelIds.has(modelId)) throw new Error(`Duplicate model id in reference: ${modelId}`) + modelIds.add(modelId) + + if (effortsColumn === "—") continue + + const efforts = effortsColumn.split(",").map((effort) => effort.trim()) + if (efforts.length === 0 || efforts.some((effort) => !VALID_EFFORTS.has(effort))) { + throw new Error(`Unexpected reasoning efforts for ${modelId}: ${effortsColumn}`) + } + reasoningEfforts[modelId] = efforts + } + + if (modelIds.size === 0) throw new Error("No model rows found in Command Code reference") + + return { + modelIds: sorted(modelIds), + reasoningEfforts: Object.fromEntries( + Object.entries(reasoningEfforts).sort(([left], [right]) => left.localeCompare(right)), + ), + } +} + +export function parseKnownTextOnlyModelIds(bundle: string): readonly string[] { + const markerIndex = bundle.indexOf(TEXT_ONLY_MARKER) + if (markerIndex < 0) { + throw new Error("Could not find Command Code's isKnownTextOnlyModel catalog") + } + + const setStart = bundle.lastIndexOf("new Set([", markerIndex) + if (setStart < 0) throw new Error("Could not find the text-only model set") + + const arrayStart = setStart + "new Set(".length + const arrayEnd = markerIndex - 1 + const literal = bundle.slice(arrayStart, arrayEnd) + const parsed: unknown = JSON.parse(literal) + if (!isStringArray(parsed)) throw new Error("Expected the text-only model catalog to be strings") + + return sorted(new Set(parsed)) +} + +function modelObject(bundle: string, modelId: string): string { + const start = bundle.indexOf(`{id:${JSON.stringify(modelId)},inputModalities:`) + if (start < 0) throw new Error(`Could not find model metadata for ${modelId}`) + + let depth = 0 + let quote = "" + let escaped = false + for (let index = start; index < bundle.length; index += 1) { + const character = bundle[index] ?? "" + if (quote) { + if (escaped) escaped = false + else if (character === "\\") escaped = true + else if (character === quote) quote = "" + continue + } + if (character === '"' || character === "'" || character === "`") { + quote = character + continue + } + if (character === "{") depth += 1 + else if (character === "}" && --depth === 0) return bundle.slice(start, index + 1) + } + + throw new Error(`Unterminated model metadata for ${modelId}`) +} + +export function parseBundleModelCapabilities( + bundle: string, + modelIds: readonly string[], +): { + reasoningModelIds: readonly string[] + maxOutputTokens: Readonly> +} { + const reasoningModelIds: string[] = [] + const maxOutputTokens: Record = {} + + for (const modelId of modelIds) { + const entry = modelObject(bundle, modelId) + if (entry.includes("reasoning:!0") || entry.includes("reasoningEfforts:[")) { + reasoningModelIds.push(modelId) + } + const maxOutput = /maxOutputTokens:([^,}]+)/.exec(entry)?.[1] + if (maxOutput) { + const value = Number(maxOutput) + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`Unexpected max output tokens for ${modelId}: ${maxOutput}`) + } + maxOutputTokens[modelId] = value + } + } + + return { + reasoningModelIds: sorted(reasoningModelIds), + maxOutputTokens: Object.fromEntries( + Object.entries(maxOutputTokens).sort(([left], [right]) => left.localeCompare(right)), + ), + } +} + +export function commandCodeModelMetadataFromContents( + modelsReference: string, + cliBundle: string, +): CommandCodeModelMetadata { + const reference = parseModelsReference(modelsReference) + const textOnlyModelIds = new Set(parseKnownTextOnlyModelIds(cliBundle)) + const capabilities = parseBundleModelCapabilities(cliBundle, reference.modelIds) + + return { + imageModelIds: reference.modelIds.filter((modelId) => !textOnlyModelIds.has(modelId)), + reasoningModelIds: capabilities.reasoningModelIds, + reasoningEfforts: reference.reasoningEfforts, + maxOutputTokens: capabilities.maxOutputTokens, + } +} + +export function currentModelMetadata(): CommandCodeModelMetadata { + return { + imageModelIds: sorted(Object.keys(MODEL_INPUT_MODALITIES)), + reasoningModelIds: sorted(Object.keys(MODEL_REASONING)), + reasoningEfforts: Object.fromEntries( + Object.entries(MODEL_EFFORTS) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([modelId, efforts]) => [modelId, [...efforts]]), + ), + maxOutputTokens: Object.fromEntries( + Object.entries(MODEL_MAX_OUTPUT_TOKENS).sort(([left], [right]) => left.localeCompare(right)), + ), + } +} + +export function diffModelMetadata( + current: CommandCodeModelMetadata, + upstream: CommandCodeModelMetadata, + currentVersion = COMMAND_CODE_CLI_VERSION, + upstreamVersion = COMMAND_CODE_CLI_VERSION, +): ModelMetadataDiff { + const currentImages = new Set(current.imageModelIds) + const upstreamImages = new Set(upstream.imageModelIds) + const currentReasoning = new Set(current.reasoningModelIds) + const upstreamReasoning = new Set(upstream.reasoningModelIds) + const currentEffortIds = Object.keys(current.reasoningEfforts) + const upstreamEffortIds = Object.keys(upstream.reasoningEfforts) + const currentEffortSet = new Set(currentEffortIds) + const upstreamEffortSet = new Set(upstreamEffortIds) + const currentMaxOutputIds = Object.keys(current.maxOutputTokens) + const upstreamMaxOutputIds = Object.keys(upstream.maxOutputTokens) + const currentMaxOutputSet = new Set(currentMaxOutputIds) + const upstreamMaxOutputSet = new Set(upstreamMaxOutputIds) + + return { + versionChanged: currentVersion !== upstreamVersion, + addedImageModelIds: sorted( + upstream.imageModelIds.filter((modelId) => !currentImages.has(modelId)), + ), + removedImageModelIds: sorted( + current.imageModelIds.filter((modelId) => !upstreamImages.has(modelId)), + ), + addedReasoningModelIds: sorted( + upstream.reasoningModelIds.filter((modelId) => !currentReasoning.has(modelId)), + ), + removedReasoningModelIds: sorted( + current.reasoningModelIds.filter((modelId) => !upstreamReasoning.has(modelId)), + ), + addedEffortModelIds: sorted( + upstreamEffortIds.filter((modelId) => !currentEffortSet.has(modelId)), + ), + removedEffortModelIds: sorted( + currentEffortIds.filter((modelId) => !upstreamEffortSet.has(modelId)), + ), + changedEffortModelIds: sorted( + upstreamEffortIds.filter( + (modelId) => + currentEffortSet.has(modelId) && + JSON.stringify(current.reasoningEfforts[modelId]) !== + JSON.stringify(upstream.reasoningEfforts[modelId]), + ), + ), + addedMaxOutputModelIds: sorted( + upstreamMaxOutputIds.filter((modelId) => !currentMaxOutputSet.has(modelId)), + ), + removedMaxOutputModelIds: sorted( + currentMaxOutputIds.filter((modelId) => !upstreamMaxOutputSet.has(modelId)), + ), + changedMaxOutputModelIds: sorted( + upstreamMaxOutputIds.filter( + (modelId) => + currentMaxOutputSet.has(modelId) && + current.maxOutputTokens[modelId] !== upstream.maxOutputTokens[modelId], + ), + ), + } +} + +export function hasModelMetadataDiff(diff: ModelMetadataDiff): boolean { + return ( + diff.versionChanged || + Object.entries(diff).some(([key, modelIds]) => key !== "versionChanged" && modelIds.length > 0) + ) +} + +function formatList(modelIds: readonly string[]): string { + return modelIds.length > 0 ? modelIds.map((modelId) => `\`${modelId}\``).join(", ") : "None" +} + +function formatReasoningChanges( + modelIds: readonly string[], + current: CommandCodeModelMetadata, + upstream: CommandCodeModelMetadata, +): string { + if (modelIds.length === 0) return "None" + return modelIds + .map( + (modelId) => + `\`${modelId}\`: \`${(current.reasoningEfforts[modelId] ?? []).join(", ")}\` → \`${( + upstream.reasoningEfforts[modelId] ?? [] + ).join(", ")}\``, + ) + .join("
") +} + +function quoted(value: string): string { + return JSON.stringify(value) +} + +function recordEntries( + values: Readonly>, +): readonly [string, readonly string[]][] { + return Object.entries(values).sort(([left], [right]) => left.localeCompare(right)) +} + +export function renderCommandCodeCatalog( + packageVersion: string, + metadata: CommandCodeModelMetadata, +): string { + const imageEntries = sorted(metadata.imageModelIds) + .map((modelId) => ` ${quoted(modelId)}: ["text", "image"],`) + .join("\n") + const reasoningEntries = sorted(metadata.reasoningModelIds) + .map((modelId) => ` ${quoted(modelId)}: true,`) + .join("\n") + const effortEntries = recordEntries(metadata.reasoningEfforts) + .map( + ([modelId, efforts]) => + ` ${quoted(modelId)}: [${efforts.map((effort) => quoted(effort)).join(", ")}],`, + ) + .join("\n") + const maxOutputEntries = Object.entries(metadata.maxOutputTokens) + .sort(([left], [right]) => left.localeCompare(right)) + .map( + ([modelId, value]) => + ` ${quoted(modelId)}: ${value.toLocaleString("en-US").replaceAll(",", "_")},`, + ) + .join("\n") + + return `export const COMMAND_CODE_CLI_VERSION = ${quoted(packageVersion)}\n\nexport type CommandCodeInputType = "text" | "image"\nexport type CommandCodeReasoningEffort = "minimal" | "low" | "medium" | "high" | "xhigh" | "max"\n\n/**\n * Generated from command-code@${packageVersion} by \`npm run sync:commandcode-catalog\`.\n * Do not edit manually.\n */\nexport const MODEL_INPUT_MODALITIES: Readonly> = {\n${imageEntries}\n}\n\nexport const MODEL_REASONING: Readonly> = {\n${reasoningEntries}\n}\n\nexport const MODEL_EFFORTS: Readonly> = {\n${effortEntries}\n}\n\nexport const MODEL_MAX_OUTPUT_TOKENS: Readonly> = {\n${maxOutputEntries}\n}\n` +} + +function updateDocumentedCatalogVersion( + contents: string, + packageVersion: string, + context: string, +): string { + const pattern = /command-code@\d+\.\d+\.\d+(?:[-+][^`\s,]+)?/ + if (!pattern.test(contents)) throw new Error(`Could not find the ${context} catalog version`) + return contents.replace(pattern, `command-code@${packageVersion}`) +} + +export function updateReadmeCatalogVersion(readme: string, packageVersion: string): string { + return updateDocumentedCatalogVersion(readme, packageVersion, "README") +} + +async function writeSynchronizedCatalog( + packageVersion: string, + metadata: CommandCodeModelMetadata, +): Promise { + const readme = await readFile(README_PATH, "utf-8") + await Promise.all([ + writeFile(CATALOG_SOURCE_PATH, renderCommandCodeCatalog(packageVersion, metadata), "utf-8"), + writeFile(README_PATH, updateReadmeCatalogVersion(readme, packageVersion), "utf-8"), + ]) +} + +function metadataReport( + packageVersion: string, + current: CommandCodeModelMetadata, + upstream: CommandCodeModelMetadata, + diff: ModelMetadataDiff, +): string { + const status = hasModelMetadataDiff(diff) ? "❌ Drift detected" : "✅ Metadata is current" + return [ + "## Command Code static model metadata", + "", + `**${status}**`, + "", + `- Repository snapshot: \`command-code@${COMMAND_CODE_CLI_VERSION}\``, + `- Inspected package: \`command-code@${packageVersion}\``, + `- Image-capable models: ${current.imageModelIds.length} repository / ${upstream.imageModelIds.length} upstream`, + `- Reasoning models: ${current.reasoningModelIds.length} repository / ${upstream.reasoningModelIds.length} upstream`, + `- Models with selectable efforts: ${Object.keys(current.reasoningEfforts).length} repository / ${Object.keys(upstream.reasoningEfforts).length} upstream`, + `- Model-specific output limits: ${Object.keys(current.maxOutputTokens).length} repository / ${Object.keys(upstream.maxOutputTokens).length} upstream`, + "", + "| Change | Models |", + "| --- | --- |", + `| CLI version | ${diff.versionChanged ? `\`${COMMAND_CODE_CLI_VERSION}\` → \`${packageVersion}\`` : "Current"} |`, + `| New image support | ${formatList(diff.addedImageModelIds)} |`, + `| Removed image support | ${formatList(diff.removedImageModelIds)} |`, + `| New reasoning models | ${formatList(diff.addedReasoningModelIds)} |`, + `| Removed reasoning models | ${formatList(diff.removedReasoningModelIds)} |`, + `| New effort metadata | ${formatList(diff.addedEffortModelIds)} |`, + `| Removed effort metadata | ${formatList(diff.removedEffortModelIds)} |`, + `| Changed reasoning efforts | ${formatReasoningChanges(diff.changedEffortModelIds, current, upstream)} |`, + `| New output limits | ${formatList(diff.addedMaxOutputModelIds)} |`, + `| Removed output limits | ${formatList(diff.removedMaxOutputModelIds)} |`, + `| Changed output limits | ${formatList(diff.changedMaxOutputModelIds)} |`, + "", + ].join("\n") +} + +async function resolvePackageSpec( + packageSpec: string, + directory: string, + npmCacheDirectory: string, +): Promise { + if (packageSpec !== "command-code@latest") return packageSpec + + const { stdout } = await execFileAsync( + "npm", + ["view", packageSpec, "version", "--json", "--prefer-online", "--cache", npmCacheDirectory], + { + cwd: directory, + encoding: "utf-8", + }, + ) + return `command-code@${parsePackageVersion(JSON.parse(stdout) as unknown)}` +} + +async function inspectPackedPackage(packageSpec: string): Promise<{ + packageVersion: string + metadata: CommandCodeModelMetadata +}> { + const directory = await mkdtemp(join(tmpdir(), "pi-commandcode-model-check-")) + const npmCacheDirectory = join(directory, "npm-cache") + + try { + const resolvedPackageSpec = await resolvePackageSpec(packageSpec, directory, npmCacheDirectory) + const { stdout } = await execFileAsync( + "npm", + ["pack", resolvedPackageSpec, "--json", "--prefer-online", "--cache", npmCacheDirectory], + { + cwd: directory, + encoding: "utf-8", + maxBuffer: 10 * 1024 * 1024, + }, + ) + const packed = parsePackedPackage(JSON.parse(stdout) as unknown) + await execFileAsync("tar", ["-xzf", packed.filename], { cwd: directory }) + + const packageDirectory = join(directory, "package") + const packageJsonContents = await readFile(join(packageDirectory, "package.json"), "utf-8") + const packageJson: unknown = JSON.parse(packageJsonContents) + if (!isRecord(packageJson) || typeof packageJson.version !== "string") { + throw new Error("Expected command-code package.json to contain a version") + } + + const [modelsReference, cliBundle] = await Promise.all([ + readFile(join(packageDirectory, MODELS_REFERENCE_PATH), "utf-8"), + readFile(join(packageDirectory, CLI_BUNDLE_PATH), "utf-8"), + ]) + + return { + packageVersion: packageJson.version, + metadata: commandCodeModelMetadataFromContents(modelsReference, cliBundle), + } + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +async function main(): Promise { + const write = process.argv.includes("--write") + const packageSpec = + process.argv.find((argument) => argument.startsWith("command-code@")) ?? "command-code@latest" + const current = currentModelMetadata() + const upstreamPackage = await inspectPackedPackage(packageSpec) + const diff = diffModelMetadata( + current, + upstreamPackage.metadata, + COMMAND_CODE_CLI_VERSION, + upstreamPackage.packageVersion, + ) + const report = metadataReport( + upstreamPackage.packageVersion, + current, + upstreamPackage.metadata, + diff, + ) + + console.log(report) + + if (write) { + await writeSynchronizedCatalog(upstreamPackage.packageVersion, upstreamPackage.metadata) + console.log(`Synchronized static metadata with command-code@${upstreamPackage.packageVersion}.`) + return + } + + if (hasModelMetadataDiff(diff)) { + throw new Error( + `Static model metadata differs from command-code@${upstreamPackage.packageVersion}. Update src/models.ts and the snapshot version.`, + ) + } +} + +function isMainModule(): boolean { + const entrypoint = process.argv[1] + return entrypoint !== undefined && pathToFileURL(resolve(entrypoint)).href === import.meta.url +} + +if (isMainModule()) { + try { + await main() + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 + } +} diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml new file mode 100644 index 0000000..88c9ca5 --- /dev/null +++ b/.github/workflows/model-metadata.yml @@ -0,0 +1,90 @@ +name: Command Code catalog sync + +on: + pull_request: + branches: [main] + paths: + - ".github/scripts/check-commandcode-model-metadata.ts" + - ".github/workflows/model-metadata.yml" + - "src/commandcode-catalog.ts" + - "src/core.ts" + - "src/models.ts" + - "tests/test-model-metadata-check.ts" + schedule: + - cron: "17 6 * * *" + workflow_dispatch: + +concurrency: + group: commandcode-catalog-${{ github.event_name == 'pull_request' && github.event.pull_request.number || 'sync' }} + cancel-in-progress: true + +jobs: + check: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + registry-url: https://registry.npmjs.org + - run: npm ci + - name: Compare with the latest Command Code CLI + run: npm run check:commandcode-catalog | tee commandcode-catalog-report.md + - name: Publish catalog report + if: always() + run: cat commandcode-catalog-report.md >> "$GITHUB_STEP_SUMMARY" + + sync: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + registry-url: https://registry.npmjs.org + - run: npm ci + - name: Synchronize with the latest Command Code CLI + run: npm run sync:commandcode-catalog | tee commandcode-catalog-report.md + - name: Format and verify synchronized files + run: | + npm run format -- src/commandcode-catalog.ts README.md + npm run typecheck + npm run test:models + npm run format:check + git diff --check + - name: Publish catalog report + if: always() + run: cat commandcode-catalog-report.md >> "$GITHUB_STEP_SUMMARY" + - name: Create or update synchronization PR + uses: peter-evans/create-pull-request@v8 + with: + branch: automation/commandcode-catalog + delete-branch: true + commit-message: "chore(models): sync Command Code catalog" + title: "chore(models): sync Command Code catalog" + body: | + Automated synchronization with the latest published `command-code` CLI package. + + This updates only machine-readable compatibility metadata: + - CLI version used in the `x-command-code-version` header + - image-input capabilities + - reasoning capability and selectable effort levels + - model-specific maximum output limits + - documented catalog snapshot version + + Pricing remains review-only because CLI documentation does not represent every pricing tier and temporary promotion used by the provider. + assignees: patlux + add-paths: | + src/commandcode-catalog.ts + README.md diff --git a/.gitleaks.toml b/.gitleaks.toml index e1c2d97..dcd3149 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -41,5 +41,5 @@ title = "pi-commandcode-provider secret scan" [[rules]] id = "pi-test-api-key" description = "Test API key value that looks real" - regex = '''(user_testKey|mock-key|fake-key|test-api-key)''' + regex = '''['"](user_testKey|mock-key|fake-key|test-api-key)['"]''' tags = ["pi-extension", "test"] diff --git a/.semgrep/pi-extension-audit.yaml b/.semgrep/pi-extension-audit.yaml index b56c540..120a0ec 100644 --- a/.semgrep/pi-extension-audit.yaml +++ b/.semgrep/pi-extension-audit.yaml @@ -256,10 +256,10 @@ rules: - pattern: process.env.$VAR - metavariable-regex: metavariable: $VAR - regex: "(?!COMMANDCODE_|NODE_|PATH|HOME|SHELL|USER|LANG|LC_|TERM|TMPDIR|NIX_).*" + regex: "(?!COMMANDCODE_|COMMAND_CODE_|CMD_ZDR|NODE_|PATH|HOME|SHELL|USER|LANG|LC_|TERM|TMPDIR|NIX_).*" message: > Reading unexpected environment variable $VAR. Provider should only - read COMMANDCODE_* variables. + read documented Command Code or standard runtime variables. severity: WARNING languages: [javascript, typescript] paths: @@ -270,4 +270,3 @@ rules: # ──────────────────────────────────────────────────────────────────────── # OAuth flow manipulation # ──────────────────────────────────────────────────────────────────────── - diff --git a/CHANGELOG.md b/CHANGELOG.md index 01bf07a..faf228e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,36 @@ - Stop silently dropping `role: "developer"` messages (for example OMP advisor steering notes, reminders, and nudges). `/alpha/generate` only accepts `user`, `assistant`, and `tool` roles, so developer messages are now forwarded as `user` messages with identical content in the same chronological position instead of disappearing from the request. +## 0.6.0 - 2026-08-25 + +- Allow switching from a vision-capable model to a text-only model by omitting historical image tool results while preserving their text output; direct image prompts still fail clearly. +- Stream incremental tool-call arguments from the `/alpha/generate` transport instead of waiting for the final complete tool-call event. +- Add a daily GitHub Actions synchronization job that opens or updates a pull request for CLI version, image capability, reasoning, effort, and output-limit changes in the latest published Command Code catalog. +- Refresh static model capabilities from `command-code@1.32.2`, separating reasoning support from selectable effort levels and honoring model-specific output limits. +- Reject truncated, aborted, and network-failed generate streams instead of reporting partial responses as successful. +- Normalize malformed tool results and synthesize missing tool results so follow-up requests preserve valid tool-call history. +- Refresh display pricing for all 58 current models, including Gemini 3.7 Flash, Qwen 3.8 27B, Ox Alpha, Muse Spark 1.2, and Grok 4.6 long-context rates. +- Accept the official `COMMAND_CODE_API_KEY` and `CMD_ZDR` environment variables while retaining legacy aliases. +- Align generate request metadata with the CLI by forwarding stable session IDs, optional temperature, and the CLI user agent. +- Validate manually pasted API keys, use the CLI's two-minute browser timeout, and reject OAuth state mismatches without closing the callback server. +- Add `/commandcode-quota` with live credits, plan, usage totals, and rolling-limit diagnostics from Command Code's alpha usage endpoints. +- Add `zai-org/GLM-5.3` with its verified reasoning efforts and display pricing. +- Prefer Command Code's Provider API (`/provider/v1/chat/completions` and `/provider/v1/messages`) and automatically fall back to the existing `/alpha/generate` transport only when the Provider API returns `403 upgrade_required` for a Go-plan account. +- Remember the detected transport for the running process, re-detect it when credentials change, prevent stale in-flight requests from overwriting the new credential's transport, and never fall back for unrelated authentication, permission, rate-limit, network, or server failures. +- Use Pi's native OpenAI- and Anthropic-compatible providers for Provider API streaming, including adaptive thinking for current reasoning-capable Claude models, while preserving the existing hardened generate transport, dynamic model discovery, offline cache, refresh/status commands, pricing, and OAuth credentials. +- Let `/login` use browser authentication, an explicit API-key prompt, or a directly pasted API key. +- Add optional zero-data-retention headers through `CMD_ZDR=1` and the legacy `COMMANDCODE_ZDR=1` alias. +- Refresh GPT-5.6 Terra and Luna display prices after their temporary 50% promotion ended, and display the current DeepSeek V4 off-peak rates for its time-dependent pricing. +- Add isolated live E2E profiles for separate Go-, GOAT-, and Provider-plan credentials, covering transport selection, reasoning across turns, quota identity, aborts, tools, GOAT vision, Go image rejection, and packed-package validation. +- Fix extension load failure on newer pi hosts that reject registering a custom API under a built-in name (`openai-completions`); register under `commandcode-custom` instead and restore the real wire API before native compat dispatch. + +### Contributors + +- @jagaliano — added the live quota dashboard and hardened its integration. +- @omariqbalnaru — fixed custom API registration for Oh My Pi 17.4.0. +- @ThomasByr — added GLM-5.3 pricing and reasoning levels. +- @newCman1 — added DeepSeek V4 vision model support. + ## 0.5.1 - 2026-08-11 - Add model-specific image input capabilities from the `command-code@1.15.1` catalog and forward user and tool-result images using the current Command Code wire format. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da4d0a4..0a6b84f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,16 @@ npm run pi:authenticated Both commands accept additional pi arguments after `--`, for example `npm run pi:authenticated -- --model claude-sonnet-4-6`. +Run the transport-specific live tests with separate credentials: + +```sh +COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key npm run test:e2e:live:go +COMMANDCODE_E2E_GOAT_API_KEY_FILE=/path/to/goat-key npm run test:e2e:live:goat +COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key npm run test:e2e:live:provider +``` + +Use `npm run test:e2e:live:all` with the Go and GOAT file variables to run both subscription transports sequentially. Store keys in a secret manager and export each one to a new mode-`0600` temporary file for the test; never add key files to the repository. Direct `*_API_KEY` variables are intended primarily for protected CI secrets. + Before opening a PR, run: ```sh diff --git a/README.md b/README.md index 91b8085..7256c32 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A custom provider for [pi](https://github.com/earendil-works/pi) that connects to the [Command Code](https://commandcode.ai) Provider API. -> **Disclaimer:** This is an unofficial, community-maintained integration. It is not affiliated with, endorsed by, or supported by Command Code. You need your own Command Code account and API key or subscription. Command Code's terms, availability, and pricing apply. +> **Disclaimer:** This is an unofficial, community-maintained integration. It is not affiliated with, endorsed by, or supported by Command Code. You need your own Command Code account, API key, and a plan with Provider API access. Command Code's terms, availability, and pricing apply. ## Install @@ -19,7 +19,7 @@ Start or reload pi, then authenticate: /login ``` -Select **Use a subscription**, then **Command Code**. Complete the browser flow and choose a model with `/model`. +Select **Use a subscription**, then **Command Code**. Choose browser login or paste an API key, then select a model with `/model`. ## Oh My Pi @@ -33,9 +33,9 @@ Restart OMP or run `/reload`, then use `/login` and select **Use a subscription* ## Authentication -### Browser login +### Login dialog -Run `/login` in pi or OMP. Select **Use a subscription**, then **Command Code**. The browser flow stores the returned credential in the host's auth file. +Run `/login` in pi or OMP. Select **Use a subscription**, then **Command Code**. Press Enter for browser login, type `key` to open a paste prompt, or paste the API key directly. The selected credential is stored in the host's auth file. Select Command Code in pi's login dialog @@ -44,7 +44,7 @@ If automatic transfer from the browser fails, copy the API key shown by Command ### Environment variable ```sh -export COMMANDCODE_API_KEY="user_..." +export COMMAND_CODE_API_KEY="user_..." ``` ### Auth file @@ -84,9 +84,7 @@ Open `/model` and select one of the models provided by Command Code. Model avail ### Reasoning support -Reasoning metadata is enriched only for models whose Command Code effort support is known. Those models register a model-specific `thinkingLevelMap`, so pi and OMP expose only supported levels. A selected supported level is sent as the documented `params.reasoning_effort` field; `off`, unsupported levels, and newly discovered models without metadata do not add reasoning fields to the request. No prompt instructions are injected. - -Reasoning blocks from completed assistant turns remain visible in pi's local session, but are not replayed to Command Code in later requests. Only the assistant's user-visible text and completed tool calls are sent back as history. This matches the current Command Code CLI behavior and prevents prior private reasoning traces from interfering with reasoning on follow-up turns. +Reasoning capability and selectable effort levels follow the official CLI catalog independently. Models can therefore be marked as reasoning-capable even when Command Code chooses their depth automatically. Models with explicit effort support also register a model-specific `thinkingLevelMap`, so pi and OMP expose only valid levels. Pi's native OpenAI- and Anthropic-compatible providers translate the selected level for Provider API accounts; the existing Command Code generate transport sends the matching `reasoning_effort` for Go accounts. List Command Code models from the terminal: @@ -122,6 +120,11 @@ While pi is running, use these provider commands without restarting: - `/commandcode-refresh` fetches and re-registers the current model catalog. Overlapping refreshes are coalesced, and a failed refresh keeps the last valid catalog active. - `/commandcode-status` shows redacted discovery diagnostics, including the source, model count, timestamps, cache path, endpoint, and warning. +- `/commandcode-quota` shows your Command Code account usage and quota in a dashboard-style layout: credits remaining and used with a percentage, monthly/purchased/free sources, the current plan, available usage totals, the API key name, and the 5-hour and weekly usage windows. + +The `commandcode-quota` command reads from the Command Code alpha usage endpoints (the same ones the `cmd` CLI `/usage` command uses): `whoami`, `billing/credits`, `billing/subscriptions`, and `usage/summary`. It authenticates with the same API key the provider already uses. If the command cannot reach those endpoints or an endpoint schema changes, unavailable sections are reported explicitly instead of being displayed as zero usage. Output is plain text (via `ui.notify`) so it works across pi and compatible hosts such as OMP. + +Set `CMD_ZDR=1` to send Command Code's documented `x-cmd-zdr: 1` zero-data-retention header. The legacy `COMMANDCODE_ZDR=1` alias remains supported. The following environment variables are intended for tests, local mocks, and compatible API endpoints: @@ -132,15 +135,15 @@ The following environment variables are intended for tests, local mocks, and com ## Image input -The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.15.1`; unknown models default to text-only until their upstream metadata is reviewed. +The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed. A daily GitHub Actions job synchronizes the CLI version, image capabilities, reasoning flags, reasoning efforts, and model-specific output limits with the latest published CLI package and opens or updates a reviewable pull request when they change. Pricing remains manually reviewed because temporary promotions and long-context tiers require explicit review. -For vision-capable models, image blocks from user messages and tool results are forwarded in Command Code's current data-URL wire format. Text-only models reject image content before making a network request instead of silently dropping it. +For vision-capable models, Pi's native provider adapters forward image blocks from user messages and tool results using the documented OpenAI or Anthropic message schema. Unknown and text-only models remain marked text-only in Pi. ## Pricing display -The Command Code Provider API does not currently include prices in its model catalog. This extension therefore keeps a static table for models with known prices so pi can display estimated request costs. +The Command Code Provider API does not currently include prices in its model catalog. This extension therefore keeps a static table for models with known prices so pi can display estimated request costs. DeepSeek V4 uses time-dependent rates; pi displays the documented off-peak rate, which applies for 17 hours per day. -Models missing from that table display zero cost in pi. This does **not** mean that Command Code will bill the request at zero. Check the current [Command Code pricing](https://commandcode.ai/docs/resources/pricing-limits) before relying on the displayed value. +Models missing from that table display zero cost in pi. This does **not** mean that Command Code will bill the request at zero. The Command Code Usage page remains authoritative for each request. Check the current [Command Code pricing](https://commandcode.ai/docs/resources/pricing-limits) before relying on the displayed value. ## Update and remove @@ -181,6 +184,29 @@ npm run pi:authenticated Both commands accept additional pi arguments after `--`, for example `npm run pi:authenticated -- --model claude-sonnet-4-6`. +### Live transport tests + +Keep Go-, GOAT-, and optional Provider-plan test keys in separate secret-manager entries. Pass them through protected files so the keys do not enter shell history: + +```sh +COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \ + npm run test:e2e:live:go + +COMMANDCODE_E2E_GOAT_API_KEY_FILE=/path/to/goat-key \ + npm run test:e2e:live:goat + +COMMANDCODE_E2E_PROVIDER_API_KEY_FILE=/path/to/provider-key \ + npm run test:e2e:live:provider + +COMMANDCODE_E2E_GO_API_KEY_FILE=/path/to/go-key \ +COMMANDCODE_E2E_GOAT_API_KEY_FILE=/path/to/goat-key \ + npm run test:e2e:live:all +``` + +Each profile runs with an isolated Pi agent directory and asserts transport selection, reasoning across turns, quota plan identity, abort handling, tool calls, and the packed npm artifact. Go must select `generate` and reject unsupported images; GOAT must select `provider` and complete a live vision request. The profile-specific `*_API_KEY` environment variables are also supported for CI secrets, but key files are preferred for local use. + +The Go profile defaults to DeepSeek V4 Flash; GOAT defaults to Grok 4.6 because its Provider API stream exposes reasoning consistently across consecutive turns. Override them with `COMMANDCODE_E2E_GO_MODEL`, `COMMANDCODE_E2E_GOAT_MODEL`, or `COMMANDCODE_E2E_PROVIDER_MODEL`. A successful live Anthropic `/provider/v1/messages` test requires a paid account whose plan includes the selected Claude model. + See [CONTRIBUTING.md](CONTRIBUTING.md) for local setup and tests. See [RELEASE.md](RELEASE.md) for the release process. ## License diff --git a/index.ts b/index.ts index 24a77b1..09222bd 100644 --- a/index.ts +++ b/index.ts @@ -1,12 +1,12 @@ /** * Command Code provider for pi. * - * Connects pi to Command Code's API (https://api.commandcode.ai/alpha/generate). - * The provider uses pi's legacy extension registration surface because the - * current pi host exposes `registerProvider(name, config)`, including OMP. + * Uses Command Code's documented Provider API: + * https://api.commandcode.ai/provider/v1 */ import { AssistantMessageEventStream } from "@earendil-works/pi-ai" +import { streamSimple as streamNativeProvider } from "@earendil-works/pi-ai/compat" import { getAgentDir, type ExtensionAPI, @@ -15,79 +15,109 @@ import { } from "@earendil-works/pi-coding-agent" import { join } from "node:path" -import { COMMAND_CODE_CLI_VERSION, createStreamCommandCode, DEFAULT_API_BASE } from "./src/core.ts" +import { getConfiguredApiKey } from "./src/api-key.ts" +import { createStreamCommandCode } from "./src/core.ts" import { calculateCommandCodeCost } from "./src/cost.ts" import { + apiForModelId, + baseUrlForModel, DEFAULT_MODELS_URL, + DEFAULT_PROVIDER_API_BASE, getModelsTimeoutMs, inputModalitiesForModel, loadCommandCodeModels, + MODEL_EFFORTS, thinkingMetadataForModel, type CommandCodeModel, } from "./src/models.ts" import { getApiKey as getOAuthApiKey, login, refreshToken } from "./src/oauth.ts" -import { MODEL_COSTS, ZERO_MODEL_COST } from "./src/pricing.ts" -import { createCommandCodeRuntime } from "./src/runtime.ts" import { normalizeCommandCodeMessage } from "./src/overflow.ts" +import { MODEL_COSTS, ZERO_MODEL_COST } from "./src/pricing.ts" +import { registerCommandCodeQuota } from "./src/quota-command.ts" +import { createCommandCodeRuntime } from "./src/runtime.ts" +import { createCommandCodeTransportRouter } from "./src/transport.ts" + +function commandCodeHeaders(): Record | undefined { + if (process.env.CMD_ZDR === "1" || process.env.COMMANDCODE_ZDR === "1") { + return { "x-cmd-zdr": "1" } + } + return undefined +} function createProviderConfig( models: readonly CommandCodeModel[], apiBase: string, streamCommandCode: ProviderConfig["streamSimple"], ): ProviderConfig { + const headers = commandCodeHeaders() return { name: "Command Code", baseUrl: apiBase, - // Keep environment authentication dynamic. OAuth credentials are resolved - // by pi's oauth registration, while the custom stream retains its own - // request-time legacy-file fallback for older compatible hosts. - apiKey: "$COMMANDCODE_API_KEY", - authHeader: true, + apiKey: getConfiguredApiKey() ?? "$COMMAND_CODE_API_KEY", api: "commandcode-custom", streamSimple: streamCommandCode, - headers: { - "x-command-code-version": COMMAND_CODE_CLI_VERSION, - "x-cli-environment": "production", - }, + headers, oauth: { name: "Command Code", login, refreshToken, getApiKey: getOAuthApiKey, }, - models: models.map(createProviderModel), + models: models.map((model) => ({ + id: model.id, + name: model.name, + api: "commandcode-custom", + baseUrl: baseUrlForModel(apiBase, model.api), + reasoning: model.reasoning, + ...(thinkingMetadataForModel(model.id) ?? {}), + input: [...inputModalitiesForModel(model.id)], + cost: MODEL_COSTS[model.id] ?? ZERO_MODEL_COST, + contextWindow: model.contextWindow, + maxTokens: model.maxTokens, + headers, + compat: + model.api === "openai-completions" + ? { + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: MODEL_EFFORTS[model.id] !== undefined, + maxTokensField: "max_tokens", + } + : { + supportsEagerToolInputStreaming: false, + supportsLongCacheRetention: false, + supportsCacheControlOnTools: false, + supportsToolReferences: false, + ...(model.reasoning ? { forceAdaptiveThinking: true } : {}), + }, + })), } } -function createProviderModel(model: { - id: string - name: string - reasoning: boolean - contextWindow: number - maxTokens: number -}) { - return { - id: model.id, - name: model.name, - reasoning: model.reasoning, - ...(thinkingMetadataForModel(model.id) ?? {}), - input: inputModalitiesForModel(model.id), - cost: MODEL_COSTS[model.id] ?? ZERO_MODEL_COST, - contextWindow: model.contextWindow, - maxTokens: model.maxTokens, - } as const +function legacyApiBase(providerApiBase: string): string { + return providerApiBase.replace(/\/provider\/v1\/?$/, "") } export default async function (pi: ExtensionAPI) { - const apiBase = process.env.COMMANDCODE_API_BASE ?? DEFAULT_API_BASE + const apiBase = process.env.COMMANDCODE_API_BASE ?? DEFAULT_PROVIDER_API_BASE const modelsUrl = process.env.COMMANDCODE_MODELS_URL ?? DEFAULT_MODELS_URL const modelsTimeoutMs = getModelsTimeoutMs() const modelsCachePath = process.env.COMMANDCODE_MODELS_CACHE ?? join(getAgentDir(), "commandcode-models.json") - const streamCommandCode = createStreamCommandCode({ + const streamGenerate = createStreamCommandCode({ createStream: () => new AssistantMessageEventStream(), calculateCost: calculateCommandCodeCost, - apiBase, + apiBase: legacyApiBase(apiBase), + }) + const transport = createCommandCodeTransportRouter({ + createStream: () => new AssistantMessageEventStream(), + streamProvider: (model, context, options) => + streamNativeProvider( + { ...model, api: apiForModelId(model.id), compat: model.compatConfig ?? model.compat }, + context, + options, + ), + streamGenerate, }) pi.on("message_end", async (event, ctx) => { @@ -96,6 +126,11 @@ export default async function (pi: ExtensionAPI) { return normalized ? { message: normalized.message } : undefined }) + registerCommandCodeQuota(pi, { + apiBase: legacyApiBase(apiBase), + headers: commandCodeHeaders(), + }) + const runtime = createCommandCodeRuntime(pi, { endpoint: modelsUrl, cachePath: modelsCachePath, @@ -105,7 +140,8 @@ export default async function (pi: ExtensionAPI) { cachePath: modelsCachePath, timeoutMs: modelsTimeoutMs, }), - createProviderConfig: (models) => createProviderConfig(models, apiBase, streamCommandCode), + createProviderConfig: (models) => createProviderConfig(models, apiBase, transport.stream), + getTransport: transport.getTransport, }) await runtime.initialize() diff --git a/package-lock.json b/package-lock.json index e58b000..e2df0a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-commandcode-provider", - "version": "0.5.1", + "version": "0.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-commandcode-provider", - "version": "0.5.1", + "version": "0.6.0", "license": "MIT", "devDependencies": { "@types/node": "25.6.0", @@ -473,7 +473,7 @@ "version": "25.6.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-25.6.0.tgz", "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==", - "devOptional": true, + "dev": true, "license": "MIT", "dependencies": { "undici-types": "~7.19.0" @@ -589,7 +589,7 @@ "version": "7.19.2", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.19.2.tgz", "integrity": "sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==", - "devOptional": true, + "dev": true, "license": "MIT" } } diff --git a/package.json b/package.json index 05a6ef0..9293ffb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-commandcode-provider", - "version": "0.5.1", + "version": "0.6.0", "description": "pi custom provider for Command Code API (commandcode.ai)", "type": "module", "keywords": [ @@ -29,14 +29,18 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", "pi:isolated": "node scripts/pi-isolated.mjs", "pi:authenticated": "node scripts/pi-authenticated.mjs", - "test:unit": "tsx tests/test-pure-functions.ts", - "test:models": "tsx tests/test-models.ts", + "check:commandcode-catalog": "tsx .github/scripts/check-commandcode-model-metadata.ts command-code@latest", + "sync:commandcode-catalog": "tsx .github/scripts/check-commandcode-model-metadata.ts command-code@latest --write", + "test:quota": "tsx tests/test-quota.ts && tsx tests/test-quota-command.ts", + "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", + "test:api-key": "tsx tests/test-api-key.ts", + "test:models": "tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts", "test:runtime": "tsx tests/test-runtime.ts", "test:pricing": "tsx tests/test-pricing.ts", "test:oauth": "tsx tests/test-oauth.ts", @@ -44,11 +48,16 @@ "test:overflow": "tsx tests/test-overflow.ts", "test:stream": "tsx tests/test-stream.ts", "test:retry": "tsx tests/test-retry.ts", + "test:transport": "tsx tests/test-transport.ts", "test:pi-isolated": "node tests/test-pi-isolated.mjs", "test:pi-authenticated": "node tests/test-pi-authenticated.mjs", "test:pi-local": "node tests/test-pi-local.mjs", "test:smoke": "node tests/test-smoke.mjs", "test:e2e:live": "node tests/test-live-e2e.mjs", + "test:e2e:live:go": "node scripts/live-e2e-profile.mjs go", + "test:e2e:live:goat": "node scripts/live-e2e-profile.mjs goat", + "test:e2e:live:provider": "node scripts/live-e2e-profile.mjs provider", + "test:e2e:live:all": "node scripts/live-e2e-profile.mjs go goat", "test:cost": "tsx tests/test-cost.ts" }, "pi": { diff --git a/scripts/live-e2e-profile.mjs b/scripts/live-e2e-profile.mjs new file mode 100644 index 0000000..b2cca8d --- /dev/null +++ b/scripts/live-e2e-profile.mjs @@ -0,0 +1,87 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process" +import { readFile } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +const projectDir = resolve(dirname(fileURLToPath(import.meta.url)), "..") +const liveTest = resolve(projectDir, "tests", "test-live-e2e.mjs") +const profiles = process.argv.slice(2) + +if ( + profiles.length === 0 || + profiles.some((profile) => profile !== "go" && profile !== "goat" && profile !== "provider") +) { + console.error("Usage: node scripts/live-e2e-profile.mjs [go|goat|provider]") + process.exit(2) +} + +async function credentialFor(profile) { + const prefix = + profile === "go" + ? "COMMANDCODE_E2E_GO" + : profile === "goat" + ? "COMMANDCODE_E2E_GOAT" + : "COMMANDCODE_E2E_PROVIDER" + const direct = process.env[`${prefix}_API_KEY`]?.trim() + const file = process.env[`${prefix}_API_KEY_FILE`] + + if (direct && file) + throw new Error(`${prefix}_API_KEY and ${prefix}_API_KEY_FILE are mutually exclusive`) + if (direct) return direct + if (file) { + const credential = (await readFile(file, "utf-8")).trim() + if (credential) return credential + } + + throw new Error(`Set ${prefix}_API_KEY_FILE (recommended) or ${prefix}_API_KEY`) +} + +function runProfile(profile, apiKey) { + const modelVariable = + profile === "go" + ? "COMMANDCODE_E2E_GO_MODEL" + : profile === "goat" + ? "COMMANDCODE_E2E_GOAT_MODEL" + : "COMMANDCODE_E2E_PROVIDER_MODEL" + const model = + process.env[modelVariable] ?? + (profile === "goat" ? "xai/grok-4.6" : "deepseek/deepseek-v4-flash") + const env = { + ...process.env, + COMMAND_CODE_API_KEY: apiKey, + COMMANDCODE_E2E_MODEL: model, + COMMANDCODE_E2E_PROFILE: profile, + } + delete env.COMMANDCODE_API_KEY + delete env.COMMANDCODE_E2E_GO_API_KEY + delete env.COMMANDCODE_E2E_GOAT_API_KEY + delete env.COMMANDCODE_E2E_PROVIDER_API_KEY + + return new Promise((resolveRun, reject) => { + console.log(`[live-e2e:${profile}] model ${model}`) + const child = spawn(process.execPath, [liveTest], { + cwd: projectDir, + env, + stdio: "inherit", + }) + child.on("error", reject) + child.on("close", (code, signal) => { + if (code === 0) { + resolveRun() + return + } + reject(new Error(`[live-e2e:${profile}] failed (${signal ?? `exit ${code}`})`)) + }) + }) +} + +try { + for (const profile of profiles) { + await runProfile(profile, await credentialFor(profile)) + } +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exit(1) +} diff --git a/scripts/pi-authenticated.mjs b/scripts/pi-authenticated.mjs index b0fe10b..974d590 100644 --- a/scripts/pi-authenticated.mjs +++ b/scripts/pi-authenticated.mjs @@ -11,6 +11,7 @@ const env = { ...process.env, PI_SKIP_VERSION_CHECK: "1", } +delete env.COMMAND_CODE_API_KEY delete env.COMMANDCODE_API_KEY const child = spawn( diff --git a/scripts/pi-isolated.mjs b/scripts/pi-isolated.mjs index a725f4e..b66c622 100644 --- a/scripts/pi-isolated.mjs +++ b/scripts/pi-isolated.mjs @@ -22,6 +22,7 @@ const env = { PI_CODING_AGENT_SESSION_DIR: sessionDir, PI_SKIP_VERSION_CHECK: "1", } +delete env.COMMAND_CODE_API_KEY delete env.COMMANDCODE_API_KEY let activeChild diff --git a/src/api-key.ts b/src/api-key.ts new file mode 100644 index 0000000..f09e155 --- /dev/null +++ b/src/api-key.ts @@ -0,0 +1,69 @@ +import { existsSync, readFileSync } from "node:fs" +import { homedir } from "node:os" +import { join } from "node:path" + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined +} + +function defaultAuthPaths(home: string): string[] { + return [ + join(home, ".commandcode", "auth.json"), + join(home, ".pi", "agent", "auth.json"), + join(home, ".omp", "agent", "auth.json"), + ] +} + +function apiKeyFromCredential(value: unknown): string | undefined { + if (!isRecord(value)) return undefined + + if (stringValue(value.type) === "oauth") return stringValue(value.access) + if (stringValue(value.type) === "api") return stringValue(value.key) + return stringValue(value.access) ?? stringValue(value.key) +} + +export function getConfiguredApiKey( + options: { + env?: NodeJS.ProcessEnv + authPaths?: readonly string[] + homeDir?: () => string + } = {}, +): string | undefined { + const env = options.env ?? process.env + if (env.COMMAND_CODE_API_KEY) return env.COMMAND_CODE_API_KEY + if (env.COMMANDCODE_API_KEY) return env.COMMANDCODE_API_KEY + + const home = options.homeDir?.() ?? homedir() + const authPaths = options.authPaths ?? defaultAuthPaths(home) + + for (const authPath of authPaths) { + try { + if (!existsSync(authPath)) continue + const parsed: unknown = JSON.parse(readFileSync(authPath, "utf-8")) + if (!isRecord(parsed)) continue + + const apiKey = stringValue(parsed.apiKey) + if (apiKey) return apiKey + + const commandcode = stringValue(parsed.commandcode) + if (commandcode) return commandcode + + const providerKey = apiKeyFromCredential(parsed.commandcode) + if (providerKey) return providerKey + + const commandCode = stringValue(parsed["command-code"]) + if (commandCode) return commandCode + + const commandCodeKey = apiKeyFromCredential(parsed["command-code"]) + if (commandCodeKey) return commandCodeKey + } catch { + // Ignore malformed or unreadable auth files. + } + } + + return undefined +} diff --git a/src/auth-server.ts b/src/auth-server.ts index e815c10..ee9e6de 100644 --- a/src/auth-server.ts +++ b/src/auth-server.ts @@ -28,6 +28,7 @@ export interface AuthServer { export interface AuthServerOptions { startPort?: number portRange?: number + expectedState?: string } function listenOnAvailablePort( @@ -181,6 +182,12 @@ export async function startAuthServer(options: AuthServerOptions = {}): Promise< return } + if (options.expectedState !== undefined && state !== options.expectedState) { + res.writeHead(403) + res.end(JSON.stringify({ success: false, error: "Invalid state token" })) + return + } + res.writeHead(200) res.end(JSON.stringify({ success: true })) diff --git a/src/commandcode-catalog.ts b/src/commandcode-catalog.ts new file mode 100644 index 0000000..6840f5f --- /dev/null +++ b/src/commandcode-catalog.ts @@ -0,0 +1,141 @@ +export const COMMAND_CODE_CLI_VERSION = "1.32.2" + +export type CommandCodeInputType = "text" | "image" +export type CommandCodeReasoningEffort = "minimal" | "low" | "medium" | "high" | "xhigh" | "max" + +/** + * Generated from command-code@1.32.2 by `npm run sync:commandcode-catalog`. + * Do not edit manually. + */ +export const MODEL_INPUT_MODALITIES: Readonly> = { + "claude-fable-5": ["text", "image"], + "claude-haiku-4-5-20251001": ["text", "image"], + "claude-opus-4-7": ["text", "image"], + "claude-opus-4-8": ["text", "image"], + "claude-opus-5": ["text", "image"], + "claude-sonnet-4-6": ["text", "image"], + "claude-sonnet-5": ["text", "image"], + "deepseek/deepseek-v4-flash-vision-exp": ["text", "image"], + "google/gemini-3.1-flash-lite": ["text", "image"], + "google/gemini-3.5-flash": ["text", "image"], + "google/gemini-3.5-flash-lite": ["text", "image"], + "google/gemini-3.6-flash": ["text", "image"], + "google/gemini-3.7-flash": ["text", "image"], + "gpt-5.3-codex": ["text", "image"], + "gpt-5.4": ["text", "image"], + "gpt-5.4-mini": ["text", "image"], + "gpt-5.5": ["text", "image"], + "gpt-5.6-luna": ["text", "image"], + "gpt-5.6-sol": ["text", "image"], + "gpt-5.6-terra": ["text", "image"], + "meta/muse-spark-1.1": ["text", "image"], + "meta/muse-spark-1.2": ["text", "image"], + "meta/muse-spark-1.2-contributor": ["text", "image"], + "MiniMaxAI/MiniMax-M3": ["text", "image"], + "moonshotai/Kimi-K2.5": ["text", "image"], + "moonshotai/Kimi-K2.6": ["text", "image"], + "moonshotai/Kimi-K2.7-Code": ["text", "image"], + "moonshotai/Kimi-K2.7-Code-Highspeed": ["text", "image"], + "moonshotai/Kimi-K3": ["text", "image"], + "Qwen/Qwen3.6-Plus": ["text", "image"], + "Qwen/Qwen3.7-Flash": ["text", "image"], + "Qwen/Qwen3.7-Plus": ["text", "image"], + "Qwen/Qwen3.8-27B": ["text", "image"], + "Qwen/Qwen3.8-Max": ["text", "image"], + "sakana/fugu-ultra": ["text", "image"], + "stealth/ox-alpha": ["text", "image"], + "stepfun/Step-3.7-Flash": ["text", "image"], + "thinkingmachines/inkling": ["text", "image"], + "thinkingmachines/inkling-small": ["text", "image"], + "xai/grok-4.5": ["text", "image"], + "xiaomi/mimo-v2.5": ["text", "image"], +} + +export const MODEL_REASONING: Readonly> = { + "claude-fable-5": true, + "claude-opus-4-7": true, + "claude-opus-4-8": true, + "claude-opus-5": true, + "claude-sonnet-4-6": true, + "claude-sonnet-5": true, + "deepseek/deepseek-v4-flash": true, + "deepseek/deepseek-v4-flash-vision-exp": true, + "deepseek/deepseek-v4-pro": true, + "google/gemini-3.1-flash-lite": true, + "google/gemini-3.5-flash": true, + "google/gemini-3.5-flash-lite": true, + "google/gemini-3.6-flash": true, + "google/gemini-3.7-flash": true, + "gpt-5.3-codex": true, + "gpt-5.4": true, + "gpt-5.4-mini": true, + "gpt-5.5": true, + "gpt-5.6-luna": true, + "gpt-5.6-sol": true, + "gpt-5.6-terra": true, + "meta/muse-spark-1.1": true, + "meta/muse-spark-1.2": true, + "meta/muse-spark-1.2-contributor": true, + "MiniMaxAI/MiniMax-M3": true, + "moonshotai/Kimi-K2.7-Code": true, + "moonshotai/Kimi-K2.7-Code-Highspeed": true, + "moonshotai/Kimi-K3": true, + "nvidia/nemotron-3-ultra-550b-a55b": true, + "poolside/laguna-s-2.1-free": true, + "Qwen/Qwen3.6-Max-Preview": true, + "Qwen/Qwen3.6-Plus": true, + "Qwen/Qwen3.7-Flash": true, + "Qwen/Qwen3.7-Max": true, + "Qwen/Qwen3.7-Plus": true, + "Qwen/Qwen3.8-27B": true, + "Qwen/Qwen3.8-Max": true, + "sakana/fugu-ultra": true, + "stealth/ox-alpha": true, + "stepfun/Step-3.5-Flash": true, + "stepfun/Step-3.7-Flash": true, + "tencent/hy3-paid": true, + "thinkingmachines/inkling": true, + "thinkingmachines/inkling-small": true, + "xai/grok-4.5": true, + "xai/grok-4.6": true, + "zai-org/GLM-5.2": true, + "zai-org/GLM-5.3": true, +} + +export const MODEL_EFFORTS: Readonly> = { + "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], + "claude-opus-4-7": ["low", "medium", "high", "xhigh", "max"], + "claude-opus-4-8": ["low", "medium", "high", "xhigh", "max"], + "claude-opus-5": ["low", "medium", "high", "xhigh", "max"], + "claude-sonnet-4-6": ["low", "medium", "high", "xhigh", "max"], + "claude-sonnet-5": ["low", "medium", "high", "xhigh", "max"], + "deepseek/deepseek-v4-flash": ["high", "max"], + "deepseek/deepseek-v4-flash-vision-exp": ["high", "max"], + "deepseek/deepseek-v4-pro": ["high", "max"], + "google/gemini-3.1-flash-lite": ["low", "medium", "high"], + "google/gemini-3.5-flash": ["low", "medium", "high"], + "google/gemini-3.5-flash-lite": ["low", "medium", "high"], + "google/gemini-3.6-flash": ["low", "medium", "high"], + "google/gemini-3.7-flash": ["low", "medium", "high"], + "gpt-5.3-codex": ["low", "medium", "high", "xhigh"], + "gpt-5.4": ["low", "medium", "high", "xhigh"], + "gpt-5.4-mini": ["low", "medium", "high"], + "gpt-5.5": ["low", "medium", "high", "xhigh"], + "gpt-5.6-luna": ["low", "medium", "high", "xhigh", "max"], + "gpt-5.6-sol": ["low", "medium", "high", "xhigh", "max"], + "gpt-5.6-terra": ["low", "medium", "high", "xhigh", "max"], + "Qwen/Qwen3.8-27B": ["low", "medium", "xhigh"], + "Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"], + "sakana/fugu-ultra": ["high", "xhigh"], + "stealth/ox-alpha": ["low", "high", "max"], + "xai/grok-4.5": ["low", "medium", "high"], + "xai/grok-4.6": ["low", "medium", "high", "xhigh"], + "zai-org/GLM-5.2": ["high", "max"], + "zai-org/GLM-5.3": ["low", "high", "max"], +} + +export const MODEL_MAX_OUTPUT_TOKENS: Readonly> = { + "poolside/laguna-s-2.1-free": 32_768, + "Qwen/Qwen3.8-27B": 32_768, + "stealth/ox-alpha": 131_072, +} diff --git a/src/converters.ts b/src/converters.ts index 6480263..0df5434 100644 --- a/src/converters.ts +++ b/src/converters.ts @@ -66,9 +66,8 @@ function imageContentError(role: string): Error { export function assertTextOnlyMessages(messages?: readonly MessageLike[]): void { for (const message of messages ?? []) { - if (imageParts(message.content).length > 0) { - const role = message.role === "toolResult" ? "tool results" : `${message.role} messages` - throw imageContentError(role) + if (message.role !== "toolResult" && imageParts(message.content).length > 0) { + throw imageContentError(`${message.role} messages`) } } } @@ -107,6 +106,7 @@ export function getApiKey( } = {}, ): string | undefined { const env = options.env ?? process.env + if (env.COMMAND_CODE_API_KEY) return env.COMMAND_CODE_API_KEY if (env.COMMANDCODE_API_KEY) return env.COMMANDCODE_API_KEY const home = options.homeDir?.() ?? homedir() @@ -138,7 +138,41 @@ export function getApiKey( return undefined } +// Hosts such as OMP may pass a literal env-var name as the "resolved" registry +// key instead of the actual credential. Treat those as unresolved. +export const COMMAND_CODE_PLACEHOLDER_KEYS = new Set([ + "$COMMAND_CODE_API_KEY", + "COMMAND_CODE_API_KEY", + "$COMMANDCODE_API_KEY", + "COMMANDCODE_API_KEY", +]) + +/** + * Pick the real API key from a host registry value and/or the env/auth-file + * fallback, never returning a literal placeholder or an empty/whitespace value. + * Pure/testable. + */ +export function pickCommandCodeApiKey( + registryKey: string | undefined, + hostKey: string | undefined, +): string | undefined { + const trimmed = typeof registryKey === "string" ? registryKey.trim() : undefined + if (!trimmed) return hostKey + if (COMMAND_CODE_PLACEHOLDER_KEYS.has(trimmed)) return hostKey + return trimmed +} + export function textContent(message: { content?: unknown }): string { + if (typeof message.content === "string") return message.content + if (message.content === null || message.content === undefined) return "" + if (!Array.isArray(message.content)) { + try { + return JSON.stringify(message.content) ?? String(message.content) + } catch { + return String(message.content) + } + } + return recordArray(message.content) .filter((part) => part.type === "text") .map((part) => stringValue(part.text) ?? "") @@ -159,7 +193,12 @@ export function toolsToJson(tools?: readonly ToolLike[]): unknown[] { })) } -function completeToolCallIds(messages?: readonly MessageLike[]): Set { +interface ToolCallState { + callIds: ReadonlySet + resultIds: ReadonlySet +} + +function toolCallState(messages?: readonly MessageLike[]): ToolCallState { const callIds = new Set() const resultIds = new Set() @@ -171,12 +210,12 @@ function completeToolCallIds(messages?: readonly MessageLike[]): Set { if (id) callIds.add(id) } } - } else if (message.role === "toolResult") { - if (message.toolCallId) resultIds.add(message.toolCallId) + } else if (message.role === "toolResult" && message.toolCallId) { + resultIds.add(message.toolCallId) } } - return new Set([...callIds].filter((id) => resultIds.has(id))) + return { callIds, resultIds } } export function messagesToCC( @@ -187,7 +226,7 @@ export function messagesToCC( if (!allowImages) assertTextOnlyMessages(messages) const out: unknown[] = [] - const pairedToolCallIds = completeToolCallIds(messages) + const { callIds, resultIds } = toolCallState(messages) for (const message of messages ?? []) { if (message.role === "user" || message.role === "developer") { @@ -202,23 +241,42 @@ export function messagesToCC( }) } else if (message.role === "assistant") { const parts: unknown[] = [] + const missingResults: unknown[] = [] for (const content of recordArray(message.content)) { if (content.type === "text") { parts.push({ type: "text", text: stringValue(content.text) ?? "" }) } else if (content.type === "toolCall") { const toolCallId = stringValue(content.id) ?? "" - if (!pairedToolCallIds.has(toolCallId)) continue + const toolName = stringValue(content.name) ?? "" + if (!toolCallId) continue parts.push({ type: "tool-call", toolCallId, - toolName: stringValue(content.name) ?? "", + toolName, input: recordOrEmpty(content.arguments), }) + if (!resultIds.has(toolCallId)) { + missingResults.push({ + type: "tool-result", + toolCallId, + toolName, + output: { + type: "error-text", + value: "No result — the tool call did not complete (interrupted or lost).", + }, + }) + } } } if (parts.length > 0) out.push({ role: "assistant", content: parts }) + if (missingResults.length > 0) out.push({ role: "tool", content: missingResults }) } else if (message.role === "toolResult") { - if (!message.toolCallId || !pairedToolCallIds.has(message.toolCallId)) continue + if (!message.toolCallId || !callIds.has(message.toolCallId)) continue + const images = imageParts(message.content) + const text = textContent(message) + const outputText = + text || + (images.length > 0 && !allowImages ? "[Image omitted: model does not support images]" : "") out.push({ role: "tool", content: [ @@ -227,15 +285,13 @@ export function messagesToCC( toolCallId: message.toolCallId, toolName: message.toolName, output: message.isError - ? { type: "error-text", value: textContent(message) } - : { type: "text", value: textContent(message) }, + ? { type: "error-text", value: outputText } + : { type: "text", value: outputText }, }, ], }) - const images = imageParts(message.content) - if (images.length > 0) { - if (!allowImages) throw imageContentError("tool results") + if (images.length > 0 && allowImages) { out.push({ role: "user", content: images.map(imageToCommandCode), diff --git a/src/core.ts b/src/core.ts index f1898dd..44d3fa7 100644 --- a/src/core.ts +++ b/src/core.ts @@ -7,6 +7,7 @@ import { randomUUID } from "node:crypto" +import { COMMAND_CODE_CLI_VERSION } from "./commandcode-catalog.ts" import { commandCodeErrorMessage, redactCommandCodeErrorText } from "./overflow.ts" import { modelSupportsImageInput } from "./models.ts" import { @@ -43,7 +44,7 @@ export * from "./overflow.ts" export * from "./types.ts" export const DEFAULT_API_BASE = "https://api.commandcode.ai" -export const COMMAND_CODE_CLI_VERSION = "1.15.1" +export { COMMAND_CODE_CLI_VERSION } const DEFAULT_GENERATE_MAX_TOKENS = 64_000 const DEFAULT_MAX_RETRIES = 0 @@ -147,6 +148,10 @@ function mappedReasoningEffort(model: ModelLike, options?: StreamOptions): strin return typeof mapped === "string" && mapped !== "off" ? mapped : undefined } +function isUuid(value: string): boolean { + return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value) +} + export function projectSlugFromPath(pathName: string): string { const slug = pathName .toLowerCase() @@ -231,17 +236,15 @@ export function createStreamCommandCode(deps: CoreDependencies) { const stream = deps.createStream() async function run() { - // OMP may pass the legacy env-var name "COMMANDCODE_API_KEY" (old pi) - // or "$COMMANDCODE_API_KEY" (new pi) as the apiKey value instead of - // resolving it. Filter out these specific strings. - const LEGACY_API_KEY_REF = "$COMMANDCODE_API_KEY" - const OLD_API_KEY_REF = "COMMANDCODE_API_KEY" + // Some hosts pass a literal env-var reference instead of resolving it. + const PLACEHOLDER_API_KEYS = new Set([ + "$COMMAND_CODE_API_KEY", + "COMMAND_CODE_API_KEY", + "$COMMANDCODE_API_KEY", + "COMMANDCODE_API_KEY", + ]) const hostKey = - options?.apiKey && - options.apiKey !== LEGACY_API_KEY_REF && - options.apiKey !== OLD_API_KEY_REF - ? options.apiKey - : undefined + options?.apiKey && !PLACEHOLDER_API_KEYS.has(options.apiKey) ? options.apiKey : undefined const apiKey = hostKey ?? @@ -261,7 +264,7 @@ export function createStreamCommandCode(deps: CoreDependencies) { usage: defaultUsage(), stopReason: "error", errorMessage: - "No Command Code API key. Run /login and select Command Code, set the COMMANDCODE_API_KEY env var, or configure ~/.commandcode/auth.json, ~/.pi/agent/auth.json or ~/.omp/agent/auth.json", + "No Command Code API key. Run /login and select Command Code, set COMMAND_CODE_API_KEY (or legacy COMMANDCODE_API_KEY), or configure ~/.commandcode/auth.json, ~/.pi/agent/auth.json or ~/.omp/agent/auth.json", timestamp: now(), } stream.push({ type: "error", reason: "error", error: msg }) @@ -285,6 +288,10 @@ export function createStreamCommandCode(deps: CoreDependencies) { let textBlock: TextContent | undefined let currentTextIdx = -1 let thinkingIdx = -1 + const streamingToolCalls = new Map< + string, + { contentIndex: number; toolCall: ToolCallContent; partialArgs: string } + >() let finished = false const abortUpstream = () => { @@ -397,25 +404,81 @@ export function createStreamCommandCode(deps: CoreDependencies) { break } + case "tool-input-start": { + endTextBlock() + endThinking() + const id = stringValue(event.id) + if (!id || streamingToolCalls.has(id)) break + + const toolCall: ToolCallContent = { + type: "toolCall", + id, + name: stringValue(event.toolName) ?? "", + arguments: {}, + } + output.content.push(toolCall) + const contentIndex = output.content.length - 1 + streamingToolCalls.set(id, { contentIndex, toolCall, partialArgs: "" }) + stream.push({ + type: "toolcall_start", + contentIndex, + partial: output, + }) + break + } + + case "tool-input-delta": { + const id = stringValue(event.id) + const delta = stringValue(event.delta) + if (!id || delta === undefined) break + const active = streamingToolCalls.get(id) + if (!active) break + + active.partialArgs += delta + active.toolCall.arguments = recordOrEmpty(active.partialArgs) + stream.push({ + type: "toolcall_delta", + contentIndex: active.contentIndex, + delta, + partial: output, + }) + break + } + + case "tool-input-end": { + break + } + case "tool-call": { endTextBlock() endThinking() - const toolCall: ToolCallContent = { + const id = stringValue(event.toolCallId) ?? "" + const active = streamingToolCalls.get(id) + const toolCall: ToolCallContent = active?.toolCall ?? { type: "toolCall", - id: stringValue(event.toolCallId) ?? "", + id, name: stringValue(event.toolName) ?? "", - arguments: recordOrEmpty(event.input ?? event.args ?? event.arguments), + arguments: {}, + } + toolCall.name = stringValue(event.toolName) ?? toolCall.name + toolCall.arguments = recordOrEmpty(event.input ?? event.args ?? event.arguments) + + let contentIndex: number + if (active) { + contentIndex = active.contentIndex + streamingToolCalls.delete(id) + } else { + output.content.push(toolCall) + contentIndex = output.content.length - 1 + stream.push({ + type: "toolcall_start", + contentIndex, + partial: output, + }) } - output.content.push(toolCall) - const idx = output.content.length - 1 - stream.push({ - type: "toolcall_start", - contentIndex: idx, - partial: output, - }) stream.push({ type: "toolcall_end", - contentIndex: idx, + contentIndex, toolCall, partial: output, }) @@ -423,6 +486,15 @@ export function createStreamCommandCode(deps: CoreDependencies) { } case "finish": { + const rawFinishReason = stringValue(event.rawFinishReason) + if ( + rawFinishReason && + /^(?:network|connection|upstream)[-_\s]?error$/i.test(rawFinishReason) + ) { + throw new Error( + `Provider finished with reason "${rawFinishReason}" — upstream connection failed mid-stream`, + ) + } const usage = commandCodeUsage(event) if (usage) { const details = commandCodeInputTokenDetails(usage) @@ -446,6 +518,10 @@ export function createStreamCommandCode(deps: CoreDependencies) { break } + case "abort": { + throw abortError("Request aborted") + } + case "error": { const message = commandCodeErrorMessage(event.error) ?? @@ -463,7 +539,11 @@ export function createStreamCommandCode(deps: CoreDependencies) { if (controller.signal.aborted) throw abortError("Aborted") const workingDir = cwd() - const threadId = uuid() + const threadId = options?.sessionId + ? isUuid(options.sessionId) + ? options.sessionId + : undefined + : uuid() const reasoningEffort = mappedReasoningEffort(model, options) const timeoutMs = options?.timeoutMs @@ -491,8 +571,8 @@ export function createStreamCommandCode(deps: CoreDependencies) { tools: toolsToJson(context.tools), system: systemPromptToText(context.systemPrompt), max_tokens: generateMaxTokens(model, options), - temperature: 0.3, stream: true, + ...(options?.temperature !== undefined ? { temperature: options.temperature } : {}), ...(reasoningEffort ? { reasoning_effort: reasoningEffort } : {}), }, threadId, @@ -522,7 +602,8 @@ export function createStreamCommandCode(deps: CoreDependencies) { "x-cli-environment": "production", "x-project-slug": projectSlugFromPath(workingDir), "x-taste-learning": "true", - "x-co-flag": "false", + ...(options?.sessionId ? { "x-session-id": options.sessionId } : {}), + "User-Agent": "cli", ...options?.headers, } const bodyStr = JSON.stringify(body) @@ -635,6 +716,11 @@ export function createStreamCommandCode(deps: CoreDependencies) { const { done, value } = await raceAbort(reader.read(), attemptController.signal) if (done) { if (buffer.trim()) handleEvent(parseStreamEventLine(buffer)) + if (!finished) { + throw new Error( + "Stream ended unexpectedly before completion (no finish event) — response was truncated", + ) + } break } if (controller.signal.aborted) throw abortError("Aborted") @@ -658,7 +744,12 @@ export function createStreamCommandCode(deps: CoreDependencies) { } catch {} reader = undefined - if (controller.signal.aborted) throw streamError + if ( + controller.signal.aborted || + (streamError instanceof Error && streamError.name === "AbortError") + ) { + throw streamError + } // Never retry after visible content was emitted (including timeout mid-stream). const canRetry = output.content.length === 0 && attempt < maxRetries @@ -695,7 +786,10 @@ export function createStreamCommandCode(deps: CoreDependencies) { } } } catch (error: unknown) { - const reason: ErrorReason = controller.signal.aborted ? "aborted" : "error" + const reason: ErrorReason = + controller.signal.aborted || (error instanceof Error && error.name === "AbortError") + ? "aborted" + : "error" output.stopReason = reason output.errorMessage = reason === "aborted" diff --git a/src/models.ts b/src/models.ts index cd470ef..be7c674 100644 --- a/src/models.ts +++ b/src/models.ts @@ -1,58 +1,26 @@ import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises" import { dirname } from "node:path" -export const DEFAULT_MODELS_URL = "https://api.commandcode.ai/provider/v1/models" +import { + MODEL_EFFORTS, + MODEL_INPUT_MODALITIES, + MODEL_MAX_OUTPUT_TOKENS, + MODEL_REASONING, + type CommandCodeInputType, + type CommandCodeReasoningEffort, +} from "./commandcode-catalog.ts" + +export { MODEL_EFFORTS, MODEL_INPUT_MODALITIES, MODEL_MAX_OUTPUT_TOKENS, MODEL_REASONING } +export type { CommandCodeInputType } + +export const DEFAULT_PROVIDER_API_BASE = "https://api.commandcode.ai/provider/v1" +export const DEFAULT_MODELS_URL = `${DEFAULT_PROVIDER_API_BASE}/models` export const DEFAULT_MODELS_TIMEOUT_MS = 10_000 const DEFAULT_MAX_OUTPUT_TOKENS = 65_536 const MODEL_CACHE_VERSION = 1 -export type CommandCodeInputType = "text" | "image" - -/** - * Model input modalities from the command-code@1.15.1 bundled catalog. - * Models omitted here remain text-only so newly discovered IDs never claim - * image support without upstream evidence. - */ -export const MODEL_INPUT_MODALITIES: Readonly> = { - "MiniMaxAI/MiniMax-M3": ["text", "image"], - "Qwen/Qwen3.6-Plus": ["text", "image"], - "Qwen/Qwen3.7-Flash": ["text", "image"], - "Qwen/Qwen3.7-Plus": ["text", "image"], - "Qwen/Qwen3.8-Max": ["text", "image"], - "claude-fable-5": ["text", "image"], - "claude-haiku-4-5-20251001": ["text", "image"], - "claude-opus-4-7": ["text", "image"], - "claude-opus-4-8": ["text", "image"], - "claude-opus-5": ["text", "image"], - "claude-sonnet-4-6": ["text", "image"], - "claude-sonnet-5": ["text", "image"], - "google/gemini-3.1-flash-lite": ["text", "image"], - "google/gemini-3.5-flash": ["text", "image"], - "google/gemini-3.5-flash-lite": ["text", "image"], - "google/gemini-3.6-flash": ["text", "image"], - "gpt-5.3-codex": ["text", "image"], - "gpt-5.4": ["text", "image"], - "gpt-5.4-mini": ["text", "image"], - "gpt-5.5": ["text", "image"], - "gpt-5.6-luna": ["text", "image"], - "gpt-5.6-sol": ["text", "image"], - "gpt-5.6-terra": ["text", "image"], - "meta/muse-spark-1.1": ["text", "image"], - "meta/muse-spark-1.2": ["text", "image"], - "meta/muse-spark-1.2-contributor": ["text", "image"], - "moonshotai/Kimi-K2.5": ["text", "image"], - "moonshotai/Kimi-K2.6": ["text", "image"], - "moonshotai/Kimi-K2.7-Code": ["text", "image"], - "moonshotai/Kimi-K2.7-Code-Highspeed": ["text", "image"], - "moonshotai/Kimi-K3": ["text", "image"], - "sakana/fugu-ultra": ["text", "image"], - "stepfun/Step-3.7-Flash": ["text", "image"], - "thinkingmachines/inkling": ["text", "image"], - "thinkingmachines/inkling-small": ["text", "image"], - "xai/grok-4.5": ["text", "image"], - "xiaomi/mimo-v2.5": ["text", "image"], -} +export type CommandCodeApi = "openai-completions" | "anthropic-messages" const TEXT_INPUT_ONLY = ["text"] as const @@ -66,43 +34,6 @@ export function modelSupportsImageInput(modelId: string): boolean { export type PiThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max" -type CommandCodeReasoningEffort = Exclude - -/** - * Per-model reasoning efforts supported by Command Code's generate endpoint. - * - * The Provider API does not expose reasoning metadata. This is an exact - * snapshot of `reasoningEfforts` from the command-code@1.15.1 model catalog - * (`packages/shared/src/model-catalog.ts`, also published in the generated - * `dist/bundled/command-code-knowledge/reference/models.md`). Models omitted - * here let Command Code choose their reasoning depth, matching the CLI. - */ -export const MODEL_EFFORTS: Readonly> = { - "Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"], - "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-4-7": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-4-8": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-5": ["low", "medium", "high", "xhigh", "max"], - "claude-sonnet-4-6": ["low", "medium", "high", "xhigh", "max"], - "claude-sonnet-5": ["low", "medium", "high", "xhigh", "max"], - "deepseek/deepseek-v4-flash": ["high", "max"], - "deepseek/deepseek-v4-pro": ["high", "max"], - "gpt-5.3-codex": ["low", "medium", "high", "xhigh"], - "gpt-5.4": ["low", "medium", "high", "xhigh"], - "gpt-5.4-mini": ["low", "medium", "high"], - "gpt-5.5": ["low", "medium", "high", "xhigh"], - "gpt-5.6-luna": ["low", "medium", "high", "xhigh", "max"], - "gpt-5.6-sol": ["low", "medium", "high", "xhigh", "max"], - "gpt-5.6-terra": ["low", "medium", "high", "xhigh", "max"], - "google/gemini-3.1-flash-lite": ["low", "medium", "high"], - "google/gemini-3.5-flash": ["low", "medium", "high"], - "google/gemini-3.5-flash-lite": ["low", "medium", "high"], - "google/gemini-3.6-flash": ["low", "medium", "high"], - "sakana/fugu-ultra": ["high", "xhigh"], - "xai/grok-4.5": ["low", "medium", "high"], - "zai-org/GLM-5.2": ["high", "max"], -} - const PI_THINKING_LEVELS: readonly PiThinkingLevel[] = [ "off", "minimal", @@ -126,7 +57,7 @@ export function thinkingLevelMapForEfforts( export interface ThinkingMetadata { thinkingLevelMap: Partial> - thinking: { + thinking?: { mode: "effort" effortMap: Partial> efforts: readonly CommandCodeReasoningEffort[] @@ -135,19 +66,26 @@ export interface ThinkingMetadata { export function thinkingMetadataForModel(modelId: string): ThinkingMetadata | undefined { const efforts = MODEL_EFFORTS[modelId] - if (!efforts) return undefined - return { - thinkingLevelMap: thinkingLevelMapForEfforts(efforts), - thinking: { - mode: "effort", - effortMap: Object.fromEntries(efforts.map((effort) => [effort, effort])), - efforts, - }, + if (efforts) { + return { + thinkingLevelMap: thinkingLevelMapForEfforts(efforts), + thinking: { + mode: "effort", + effortMap: Object.fromEntries(efforts.map((effort) => [effort, effort])), + efforts, + }, + } } + if (!isReasoningModel(modelId)) return undefined + return { thinkingLevelMap: thinkingLevelMapForEfforts([]) } } function isReasoningModel(modelId: string): boolean { - return MODEL_EFFORTS[modelId] !== undefined + return MODEL_REASONING[modelId] === true +} + +function maxOutputTokensForModel(modelId: string, contextLength: number): number { + return Math.min(contextLength, MODEL_MAX_OUTPUT_TOKENS[modelId] ?? DEFAULT_MAX_OUTPUT_TOKENS) } interface ApiModel { @@ -159,11 +97,22 @@ interface ApiModel { export interface CommandCodeModel { id: string name: string + api: CommandCodeApi reasoning: boolean contextWindow: number maxTokens: number } +export function apiForModelId(id: string): CommandCodeApi { + return id.startsWith("claude-") ? "anthropic-messages" : "openai-completions" +} + +export function baseUrlForModel(apiBase: string, api: CommandCodeApi): string { + const normalized = apiBase.replace(/\/+$/g, "") + if (api !== "anthropic-messages") return normalized + return normalized.endsWith("/v1") ? normalized.slice(0, -3) : normalized +} + interface FetchCommandCodeModelsOptions { url?: string fetchImpl?: typeof fetch @@ -222,12 +171,15 @@ function parseCachedModel(value: unknown): CommandCodeModel { const id = stringField(value, "id") booleanField(value, "reasoning") + positiveNumberField(value, "maxTokens") + const contextWindow = positiveNumberField(value, "contextWindow") return { id, name: stringField(value, "name"), + api: apiForModelId(id), reasoning: isReasoningModel(id), - contextWindow: positiveNumberField(value, "contextWindow"), - maxTokens: positiveNumberField(value, "maxTokens"), + contextWindow, + maxTokens: maxOutputTokensForModel(id, contextWindow), } } @@ -329,9 +281,10 @@ export function commandCodeModelsFromApiResponse(value: unknown): readonly Comma return data.map(parseApiModel).map((model) => ({ id: model.id, name: `${model.name} (CC)`, + api: apiForModelId(model.id), reasoning: isReasoningModel(model.id), contextWindow: model.contextLength, - maxTokens: Math.min(model.contextLength, DEFAULT_MAX_OUTPUT_TOKENS), + maxTokens: maxOutputTokensForModel(model.id, model.contextLength), })) } diff --git a/src/oauth.ts b/src/oauth.ts index be77391..470ebd3 100644 --- a/src/oauth.ts +++ b/src/oauth.ts @@ -1,13 +1,13 @@ /** * Command Code OAuth provider for pi's /login flow. * - * Implements a browser-assisted API key retrieval flow: - * 1. Starts a local HTTP server on a Command Code CLI-compatible port - * 2. Opens the Command Code Studio auth page in the browser - * 3. The user authenticates on the Command Code website - * 4. The website POSTs the API key back to the local server - * 5. If browser transfer fails, the user can paste the API key manually - * 6. The API key is stored in pi's auth.json as OAuth credentials + * Implements two API key retrieval flows: + * 1. Browser-assisted login opens Command Code Studio and waits for the + * website to POST the API key back to a local callback server. + * 2. Direct API key login prompts the user to paste a Studio API key. + * + * If browser transfer fails, the user can still paste the API key manually. + * The API key is stored in pi's auth.json as OAuth credentials. * * Since Command Code API keys don't expire, we store them as * OAuth credentials with a far-future expiry. @@ -18,7 +18,8 @@ import { startAuthServer } from "./auth-server.ts" const STUDIO_BASE_URL = "https://commandcode.ai" const TEN_YEARS_MS = 10 * 365 * 24 * 60 * 60 * 1000 // API keys don't expire -const DEFAULT_AUTH_TIMEOUT_MS = 15_000 +const DEFAULT_AUTH_TIMEOUT_MS = 120_000 +const DEFAULT_API_BASE = "https://api.commandcode.ai" export interface OAuthLoginCallbacks { onAuth(params: { url: string }): void @@ -95,22 +96,70 @@ export function sanitizeApiKey(input: string): string { .trim() } +export async function validateApiKey( + apiKey: string, + options: { fetchImpl?: typeof fetch; apiBase?: string } = {}, +): Promise { + let response: Response + try { + response = await (options.fetchImpl ?? fetch)( + `${options.apiBase ?? DEFAULT_API_BASE}/alpha/whoami`, + { + headers: { Authorization: `Bearer ${apiKey}` }, + }, + ) + } catch (error) { + throw new Error( + `Could not validate the Command Code API key: ${error instanceof Error ? error.message : String(error)}`, + ) + } + + if (response.status === 401) throw new Error("Invalid Command Code API key") + if (!response.ok) { + throw new Error(`Could not validate the Command Code API key (${response.status})`) + } +} + async function promptForApiKey(callbacks: OAuthLoginCallbacks, message: string) { const apiKey = sanitizeApiKey(await callbacks.onPrompt({ message })) if (!apiKey) throw new Error("No Command Code API key provided") + await validateApiKey(apiKey) return credentialsFromApiKey(apiKey) } -/** - * Starts the browser-based login flow for Command Code. - * - * Returns OAuth credentials where access == refresh == the user's API key. - * The keys don't expire, so we set a far-future expiry. - */ -export async function login(callbacks: OAuthLoginCallbacks): Promise { +type LoginChoice = { type: "browser" } | { type: "prompt" } | { type: "apiKey"; apiKey: string } + +async function chooseLoginFlow(callbacks: OAuthLoginCallbacks): Promise { + const input = sanitizeApiKey( + await callbacks.onPrompt({ + message: + "Command Code login: press Enter for browser login, type 'key' to paste an API key, or paste the API key directly:", + }), + ) + const normalized = input.toLowerCase() + + if (!input || normalized === "1" || normalized === "b" || normalized === "browser") { + return { type: "browser" } + } + + if ( + normalized === "2" || + normalized === "k" || + normalized === "key" || + normalized === "api" || + normalized === "paste" + ) { + return { type: "prompt" } + } + + return { type: "apiKey", apiKey: input } +} + +async function browserLogin(callbacks: OAuthLoginCallbacks): Promise { + const stateToken = generateStateToken() let authServer try { - authServer = await startAuthServer() + authServer = await startAuthServer({ expectedState: stateToken }) } catch { return promptForApiKey( callbacks, @@ -118,7 +167,6 @@ export async function login(callbacks: OAuthLoginCallbacks): Promise { + const choice = await chooseLoginFlow(callbacks) + + if (choice.type === "apiKey") { + await validateApiKey(choice.apiKey) + return credentialsFromApiKey(choice.apiKey) + } + if (choice.type === "prompt") { + return promptForApiKey(callbacks, "Paste your Command Code API key:") } - return credentialsFromApiKey(callback.apiKey) + return browserLogin(callbacks) } /** diff --git a/src/pricing.ts b/src/pricing.ts index aa801c0..fddd2a3 100644 --- a/src/pricing.ts +++ b/src/pricing.ts @@ -20,7 +20,7 @@ export interface TemporaryPricing { } export const PRICING_SOURCE_URL = "https://commandcode.ai/docs/resources/pricing-limits" -export const PRICING_LAST_VERIFIED = "2026-08-04" +export const PRICING_LAST_VERIFIED = "2026-08-25" export const ZERO_MODEL_COST: CommandCodeModelCost = { input: 0, @@ -40,7 +40,7 @@ export const ZERO_MODEL_COST: CommandCodeModelCost = { export const MODEL_COSTS: Readonly> = { // Free models "poolside/laguna-s-2.1-free": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - "inclusionai/ling-3.0-flash-free": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + "stealth/ox-alpha": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, // Open and open-weight models "tencent/hy3-paid": { input: 0.14, output: 0.58, cacheRead: 0.035, cacheWrite: 0 }, @@ -54,6 +54,7 @@ export const MODEL_COSTS: Readonly> = { }, "moonshotai/Kimi-K2.6": { input: 0.95, output: 4, cacheRead: 0.16, cacheWrite: 0 }, "moonshotai/Kimi-K2.5": { input: 0.6, output: 3, cacheRead: 0.1, cacheWrite: 0 }, + "zai-org/GLM-5.3": { input: 1.4, output: 4.4, cacheRead: 0.26, cacheWrite: 0 }, "zai-org/GLM-5.2": { input: 1.4, output: 4.4, cacheRead: 0.26, cacheWrite: 0 }, "zai-org/GLM-5.2-Fast": { input: 3, output: 10.25, cacheRead: 0.5, cacheWrite: 0 }, "zai-org/GLM-5.1": { input: 1.4, output: 4.4, cacheRead: 0.26, cacheWrite: 0 }, @@ -61,20 +62,28 @@ export const MODEL_COSTS: Readonly> = { "MiniMaxAI/MiniMax-M3": { input: 0.3, output: 1.2, cacheRead: 0.06, cacheWrite: 0 }, "MiniMaxAI/MiniMax-M2.7": { input: 0.3, output: 1.2, cacheRead: 0.06, cacheWrite: 0 }, "MiniMaxAI/MiniMax-M2.5": { input: 0.3, output: 1.2, cacheRead: 0.03, cacheWrite: 0 }, - // Permanent 75% discount. + // DeepSeek V4 uses time-dependent rates. Display the documented off-peak + // rates, which apply for 17 hours per day; the Usage page remains authoritative. "deepseek/deepseek-v4-pro": { - input: 0.435, - output: 0.87, - cacheRead: 0.003625, + input: 0.66, + output: 1.98, + cacheRead: 0.022, cacheWrite: 0, }, "deepseek/deepseek-v4-flash": { - input: 0.14, - output: 0.28, - cacheRead: 0.0028, + input: 0.22, + output: 0.66, + cacheRead: 0.007, + cacheWrite: 0, + }, + "deepseek/deepseek-v4-flash-vision-exp": { + input: 0.22, + output: 0.66, + cacheRead: 0.007, cacheWrite: 0, }, "Qwen/Qwen3.8-Max": { input: 2, output: 6, cacheRead: 0.25, cacheWrite: 2.5 }, + "Qwen/Qwen3.8-27B": { input: 0.4, output: 3, cacheRead: 0.04, cacheWrite: 0 }, "Qwen/Qwen3.7-Max": { input: 2.5, output: 7.5, cacheRead: 0.5, cacheWrite: 3.13 }, "Qwen/Qwen3.7-Plus": { input: 0.4, @@ -140,6 +149,13 @@ export const MODEL_COSTS: Readonly> = { cacheWrite: 0, }, "meta/muse-spark-1.1": { input: 1.25, output: 4.25, cacheRead: 0.15, cacheWrite: 0 }, + "meta/muse-spark-1.2": { input: 1.25, output: 4.25, cacheRead: 0.15, cacheWrite: 0 }, + "meta/muse-spark-1.2-contributor": { + input: 0.1, + output: 0.2, + cacheRead: 0.002, + cacheWrite: 0, + }, // Anthropic // Introductory pricing through 2026-08-31. @@ -158,43 +174,20 @@ export const MODEL_COSTS: Readonly> = { // OpenAI "gpt-5.6-sol": { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 6.25 }, - // Discounted rates through 2026-08-14. - "gpt-5.6-terra": { - input: 1, - output: 6, - cacheRead: 0.1, - cacheWrite: 1.25, - tiers: [ - { - inputTokensAbove: 272_000, - input: 2, - output: 9, - cacheRead: 0.2, - cacheWrite: 2.5, - }, - ], - }, - "gpt-5.6-luna": { - input: 0.1, - output: 0.6, - cacheRead: 0.01, - cacheWrite: 0.125, - tiers: [ - { - inputTokensAbove: 272_000, - input: 0.2, - output: 0.9, - cacheRead: 0.02, - cacheWrite: 0.25, - }, - ], - }, + "gpt-5.6-terra": { input: 2, output: 12, cacheRead: 0.2, cacheWrite: 2.5 }, + "gpt-5.6-luna": { input: 0.2, output: 1.2, cacheRead: 0.02, cacheWrite: 0.25 }, "gpt-5.5": { input: 5, output: 30, cacheRead: 0.5, cacheWrite: 0 }, "gpt-5.4": { input: 2.5, output: 15, cacheRead: 0.25, cacheWrite: 0 }, "gpt-5.3-codex": { input: 2, output: 8, cacheRead: 0.5, cacheWrite: 0 }, "gpt-5.4-mini": { input: 0.75, output: 4.5, cacheRead: 0.075, cacheWrite: 0 }, // Google and xAI + "google/gemini-3.7-flash": { + input: 0.75, + output: 3.75, + cacheRead: 0.075, + cacheWrite: 0.04167, + }, "google/gemini-3.6-flash": { input: 1.5, output: 7.5, cacheRead: 0.15, cacheWrite: 0 }, "google/gemini-3.5-flash": { input: 1.5, output: 9, cacheRead: 0.15, cacheWrite: 0 }, "google/gemini-3.5-flash-lite": { @@ -210,17 +203,32 @@ export const MODEL_COSTS: Readonly> = { cacheWrite: 0, }, "xai/grok-4.5": { input: 2, output: 6, cacheRead: 0.5, cacheWrite: 0 }, + "xai/grok-4.6": { + input: 2, + output: 6, + cacheRead: 0.5, + cacheWrite: 0, + tiers: [ + { + inputTokensAbove: 200_000, + input: 4, + output: 12, + cacheRead: 1, + cacheWrite: 0, + }, + ], + }, } export const TEMPORARY_PRICING: readonly TemporaryPricing[] = [ - { - models: ["gpt-5.6-terra", "gpt-5.6-luna"], - expiresOn: "2026-08-14", - description: "50% promotional rates", - }, { models: ["claude-sonnet-5"], expiresOn: "2026-08-31", description: "introductory pricing", }, + { + models: ["google/gemini-3.7-flash"], + expiresOn: "2026-12-31", + description: "50% promotional pricing", + }, ] diff --git a/src/quota-command.ts b/src/quota-command.ts new file mode 100644 index 0000000..81a8779 --- /dev/null +++ b/src/quota-command.ts @@ -0,0 +1,66 @@ +import { getConfiguredApiKey } from "./api-key.ts" +import { pickCommandCodeApiKey } from "./converters.ts" +import { fetchCommandCodeQuota, redactValue } from "./quota.ts" +import { formatQuota } from "./quota-format.ts" + +export interface QuotaCommandContext { + waitForIdle?: () => Promise + modelRegistry?: { + getApiKeyForProvider?: (provider: string) => Promise + } + ui: { + notify(message: string, type?: "info" | "warning" | "error"): void + } +} + +interface QuotaCommandApi { + registerCommand( + name: string, + options: { + description: string + handler: (args: string, ctx: QuotaCommandContext) => Promise + }, + ): void +} + +interface RegisterQuotaCommandOptions { + apiBase: string + headers?: Record + getConfiguredKey?: () => string | undefined + fetchQuota?: typeof fetchCommandCodeQuota +} + +export function registerCommandCodeQuota( + pi: QuotaCommandApi, + options: RegisterQuotaCommandOptions, +): void { + const getConfiguredKey = options.getConfiguredKey ?? getConfiguredApiKey + const fetchQuota = options.fetchQuota ?? fetchCommandCodeQuota + + pi.registerCommand("commandcode-quota", { + description: "Show Command Code account usage and quota", + handler: async (_args, ctx) => { + await ctx.waitForIdle?.() + const registryKey = await ctx.modelRegistry?.getApiKeyForProvider?.("commandcode") + const apiKey = pickCommandCodeApiKey(registryKey, getConfiguredKey()) + if (!apiKey) { + ctx.ui.notify( + "Command Code quota requires an API key. Run /login and select Command Code, or set COMMAND_CODE_API_KEY.", + "warning", + ) + return + } + + const result = await fetchQuota({ + apiKey, + baseUrl: options.apiBase, + extraHeaders: options.headers, + }) + if (!result.ok) { + ctx.ui.notify(redactValue(result.error.message), "error") + return + } + ctx.ui.notify(formatQuota(result.quota), "info") + }, + }) +} diff --git a/src/quota-format.ts b/src/quota-format.ts new file mode 100644 index 0000000..9bed62c --- /dev/null +++ b/src/quota-format.ts @@ -0,0 +1,111 @@ +import type { + CommandCodeCredits, + CommandCodeQuota, + CommandCodeSubscription, + CommandCodeWindowLimit, +} from "./quota-types.ts" + +export function formatWindowLimits( + limits: readonly CommandCodeWindowLimit[], + now: () => number = Date.now, +): string[] { + const labels: Record = { + fiveHour: "5-hour", + weekly: "Weekly", + } + + return limits.map((limit) => { + const used = limit.used.toFixed(2) + const cap = limit.cap.toFixed(2) + const percent = limit.cap > 0 ? Math.round((limit.used / limit.cap) * 100) : 0 + const reset = limit.resetAt === null ? "" : ` (resets ${formatResetClock(limit.resetAt, now)})` + return `${labels[limit.window]}: ${used} / ${cap} credits (${percent}% used)${reset}` + }) +} + +function formatResetClock(resetAtSeconds: number, now: () => number): string { + const date = new Date(resetAtSeconds * 1000) + if (Number.isNaN(date.getTime())) return "unknown" + const diffMs = date.getTime() - now() + if (diffMs <= 0) return "soon" + const minutes = Math.ceil(diffMs / 60_000) + if (minutes < 60) return `in ${minutes}m` + const hours = Math.floor(minutes / 60) + const remainingMinutes = minutes % 60 + if (hours < 24) { + return remainingMinutes > 0 ? `in ${hours}h ${remainingMinutes}m` : `in ${hours}h` + } + const days = Math.floor(hours / 24) + return days === 1 ? "in 1 day" : `in ${days} days` +} + +function creditsDetail(credits: CommandCodeCredits | null): string | undefined { + if (!credits) return undefined + const parts = [ + `monthly $${credits.monthlyCredits.toFixed(2)}`, + `purchased $${credits.purchasedCredits.toFixed(2)}`, + ] + if (credits.freeCredits > 0) parts.push(`free $${credits.freeCredits.toFixed(2)}`) + return `Sources: ${parts.join(" / ")}` +} + +function subscriptionLine(subscription: CommandCodeSubscription): string { + const plan = (subscription.planId ?? "Unknown").replace(/[_-]+/g, " ").trim() + const status = subscription.status ? ` (${subscription.status})` : "" + return `Plan: ${plan}${status}` +} + +function formatTokens(tokens: number): string { + if (tokens >= 1_000_000_000) return `${(tokens / 1_000_000_000).toFixed(1)}B` + if (tokens >= 1_000_000) return `${(tokens / 1_000_000).toFixed(1)}M` + if (tokens >= 1_000) return `${(tokens / 1_000).toFixed(1)}k` + return String(tokens) +} + +export function formatQuota(quota: CommandCodeQuota, now: () => number = Date.now): string { + const lines: string[] = [] + const remaining = quota.credits?.remainingCredits ?? 0 + const spent = quota.summary?.totalCost ?? 0 + const pool = remaining + spent + + if (quota.credits || quota.summary) { + lines.push("Credits") + lines.push(` Remaining: $${remaining.toFixed(2)} of $${pool.toFixed(2)}`) + lines.push(` Used: $${spent.toFixed(2)}`) + lines.push(` ${pool > 0 ? Math.round((spent / pool) * 100) : 0}% used`) + } + + const detail = creditsDetail(quota.credits) + if (detail) lines.push(detail) + if (quota.subscription) lines.push(subscriptionLine(quota.subscription)) + + if (quota.summary) { + lines.push("") + lines.push(quota.subscription?.currentPeriodStart ? "Usage (billing period)" : "Usage") + lines.push(` Cost: $${quota.summary.totalCost.toFixed(2)}`) + lines.push(` Requests: ${quota.summary.totalCount.toLocaleString("en-US")}`) + if (quota.summary.totalTokens !== undefined) { + lines.push(` Tokens: ${formatTokens(quota.summary.totalTokens)}`) + } + } + + lines.push("") + lines.push("Account") + lines.push(` ${quota.account.keyName ?? quota.account.login}`) + + const limits = quota.credits?.windowLimits ?? [] + if (limits.length > 0) { + lines.push("") + lines.push("Usage windows:") + lines.push(...formatWindowLimits(limits, now).map((line) => ` ${line}`)) + } + + if ((quota.unavailable?.length ?? 0) > 0) { + lines.push("") + lines.push(`Unavailable: ${quota.unavailable?.join(", ")}`) + } + + lines.push("") + lines.push("Full detail: https://commandcode.ai/usage") + return lines.join("\n") +} diff --git a/src/quota-types.ts b/src/quota-types.ts new file mode 100644 index 0000000..07f670e --- /dev/null +++ b/src/quota-types.ts @@ -0,0 +1,47 @@ +export interface CommandCodeWindowLimit { + window: "fiveHour" | "weekly" + used: number + cap: number + resetAt: number | null +} + +export interface CommandCodeCredits { + monthlyCredits: number + purchasedCredits: number + freeCredits: number + remainingCredits: number + windowLimits: CommandCodeWindowLimit[] +} + +export interface CommandCodeSubscription { + planId: string | null + status: string | null + currentPeriodStart: string | null + currentPeriodEnd: string | null +} + +export interface CommandCodeUsageSummary { + totalCost: number + totalCount: number + totalTokens?: number +} + +export type CommandCodeQuotaSection = "credits" | "subscription" | "usage" + +export interface CommandCodeQuota { + account: { + login: string + orgId: string | null + keyName?: string + } + credits: CommandCodeCredits | null + subscription: CommandCodeSubscription | null + summary: CommandCodeUsageSummary | null + unavailable?: readonly CommandCodeQuotaSection[] +} + +export type CommandCodeQuotaErrorKind = "config" | "http" | "network" | "timeout" + +export type CommandCodeQuotaResult = + | { ok: true; quota: CommandCodeQuota } + | { ok: false; error: { message: string; kind: CommandCodeQuotaErrorKind } } diff --git a/src/quota.ts b/src/quota.ts new file mode 100644 index 0000000..e595b08 --- /dev/null +++ b/src/quota.ts @@ -0,0 +1,335 @@ +import { redactCommandCodeErrorText } from "./overflow.ts" +import type { + CommandCodeCredits, + CommandCodeQuotaResult, + CommandCodeQuotaSection, + CommandCodeSubscription, + CommandCodeUsageSummary, + CommandCodeWindowLimit, +} from "./quota-types.ts" + +export const DEFAULT_API_BASE = "https://api.commandcode.ai" +export const QUOTA_TIMEOUT_MS = 15_000 + +interface FetchOptions { + apiKey: string + baseUrl?: string + fetchImpl?: typeof fetch + timeoutMs?: number + extraHeaders?: Record +} + +interface HttpErrorShape { + __httpError: true + message: string + status: number + body: string +} + +interface QuotaErrorShape { + __quotaError: true + kind: "timeout" | "network" +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function numberValue(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined +} + +function stringValue(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function normalizeResetAt(value: unknown): number | null { + let timestamp: number | undefined + if (typeof value === "number" && Number.isFinite(value)) timestamp = value + if (typeof value === "string" && value.length > 0) { + const trimmed = value.trim() + timestamp = /^\d+$/.test(trimmed) ? Number(trimmed) : Date.parse(trimmed) + } + if (timestamp === undefined || !Number.isFinite(timestamp) || timestamp < 0) return null + return timestamp >= 1e12 ? Math.round(timestamp / 1000) : timestamp +} + +export function windowLimitsFromCredits(value: unknown): CommandCodeWindowLimit[] { + if (!isRecord(value)) return [] + const limits: CommandCodeWindowLimit[] = [] + for (const [window, entry] of [ + ["fiveHour", value.fiveHour], + ["weekly", value.weekly], + ] as const) { + if (!isRecord(entry)) continue + const used = numberValue(entry.used) + const cap = numberValue(entry.cap) + if (used === undefined || cap === undefined || (used === 0 && cap === 0)) continue + limits.push({ window, used, cap, resetAt: normalizeResetAt(entry.resetAt) }) + } + return limits +} + +function parseCredits(value: unknown): CommandCodeCredits | null { + if (!isRecord(value) || !isRecord(value.credits)) return null + const credits = value.credits + const monthlyCredits = numberValue(credits.monthlyCredits) + const purchasedCredits = numberValue(credits.purchasedCredits) + const freeCredits = numberValue(credits.freeCredits) + if (monthlyCredits === undefined && purchasedCredits === undefined && freeCredits === undefined) { + return null + } + const monthly = monthlyCredits ?? 0 + const purchased = purchasedCredits ?? 0 + const free = freeCredits ?? 0 + return { + monthlyCredits: monthly, + purchasedCredits: purchased, + freeCredits: free, + remainingCredits: monthly + purchased + free, + windowLimits: windowLimitsFromCredits(value.windowLimits), + } +} + +function parseSubscription(value: unknown): CommandCodeSubscription | null { + if (!isRecord(value) || !isRecord(value.data)) return null + const data = value.data + const planId = stringValue(data.planId) + const status = stringValue(data.status) + const currentPeriodStart = stringValue(data.currentPeriodStart) + const currentPeriodEnd = stringValue(data.currentPeriodEnd) + if (!planId && !status && !currentPeriodStart && !currentPeriodEnd) return null + return { + planId: planId ?? null, + status: status ?? null, + currentPeriodStart: currentPeriodStart ?? null, + currentPeriodEnd: currentPeriodEnd ?? null, + } +} + +function parseSummary(value: unknown): CommandCodeUsageSummary | null { + if (!isRecord(value)) return null + const totalCost = numberValue(value.totalCost) + const totalCount = numberValue(value.totalCount) + if (totalCost === undefined || totalCount === undefined) return null + const totalTokens = numberValue(value.totalTokens) ?? numberValue(value.tokens) + return { totalCost, totalCount, ...(totalTokens === undefined ? {} : { totalTokens }) } +} + +function parseWhoami(value: unknown): { + login: string + orgId: string | null + keyName?: string +} | null { + if (!isRecord(value)) return null + const org = isRecord(value.org) ? value.org : undefined + const user = isRecord(value.user) ? value.user : undefined + const login = + (org ? stringValue(org.login) : undefined) ?? + (user ? (stringValue(user.userName) ?? stringValue(user.name)) : undefined) + if (!login) return null + const orgId = org ? stringValue(org.id) : undefined + const keyName = user ? (stringValue(user.keyName) ?? stringValue(user.displayName)) : undefined + return { login, orgId: orgId ?? null, ...(keyName ? { keyName } : {}) } +} + +function buildUrl(path: string, params: Record): string { + const search = new URLSearchParams() + for (const [key, value] of Object.entries(params)) { + if (value) search.set(key, value) + } + const query = search.toString() + return `${path}${query ? `?${query}` : ""}` +} + +function isHttpError(value: unknown): value is HttpErrorShape { + return ( + isRecord(value) && + value.__httpError === true && + typeof value.message === "string" && + typeof value.status === "number" && + typeof value.body === "string" + ) +} + +function isQuotaError(value: unknown): value is QuotaErrorShape { + return ( + isRecord(value) && + value.__quotaError === true && + (value.kind === "timeout" || value.kind === "network") + ) +} + +function isBlockingHttpError(error: HttpErrorShape): boolean { + return error.status === 401 || error.status === 403 +} + +function httpFailure(error: HttpErrorShape, context: string): CommandCodeQuotaResult { + const detail = error.body.trim().slice(0, 200) + return { + ok: false, + error: { + kind: "http", + message: redactValue( + `${context} request failed (${error.status}): ${detail || error.message}`, + ), + }, + } +} + +class QuotaTimeoutError extends Error {} + +export async function fetchCommandCodeQuota( + options: FetchOptions, +): Promise { + if (!options.apiKey) { + return { ok: false, error: { message: "No Command Code API key found", kind: "config" } } + } + + const baseUrl = options.baseUrl ?? DEFAULT_API_BASE + const fetchImpl = options.fetchImpl ?? fetch + const timeoutMs = options.timeoutMs ?? QUOTA_TIMEOUT_MS + const overallController = new AbortController() + const overallTimer = setTimeout(() => overallController.abort(), timeoutMs) + const headers = { + accept: "application/json", + Authorization: `Bearer ${options.apiKey}`, + ...options.extraHeaders, + } + + const request = async (path: string): Promise => { + if (overallController.signal.aborted) throw new QuotaTimeoutError() + try { + const response = await fetchImpl(`${baseUrl}${path}`, { + method: "GET", + headers, + signal: overallController.signal, + }) + if (!response.ok) { + return { + __httpError: true, + message: + response.status === 401 || response.status === 403 + ? "Command Code rejected the API key" + : response.statusText, + status: response.status, + body: await response.text().catch(() => ""), + } satisfies HttpErrorShape + } + return await response.json() + } catch (error) { + if (overallController.signal.aborted) throw new QuotaTimeoutError() + throw error + } + } + + const safeRequest = async (path: string): Promise => { + try { + return await request(path) + } catch (error) { + return { + __quotaError: true, + kind: error instanceof QuotaTimeoutError ? "timeout" : "network", + } satisfies QuotaErrorShape + } + } + + try { + const whoamiRaw = await request("/alpha/whoami") + if (isHttpError(whoamiRaw)) return httpFailure(whoamiRaw, "whoami") + const account = parseWhoami(whoamiRaw) + if (!account) { + return { + ok: false, + error: { kind: "http", message: "Command Code returned an unrecognized account response" }, + } + } + + const orgId = account.orgId ?? undefined + const [creditsRaw, subscriptionRaw] = await Promise.all([ + safeRequest(buildUrl("/alpha/billing/credits", { orgId })), + safeRequest(buildUrl("/alpha/billing/subscriptions", { orgId })), + ]) + if (isHttpError(creditsRaw) && isBlockingHttpError(creditsRaw)) { + return httpFailure(creditsRaw, "credits") + } + if (isHttpError(subscriptionRaw) && isBlockingHttpError(subscriptionRaw)) { + return httpFailure(subscriptionRaw, "subscription") + } + + const unavailable: CommandCodeQuotaSection[] = [] + const credits = + isHttpError(creditsRaw) || isQuotaError(creditsRaw) ? null : parseCredits(creditsRaw) + if (!credits) unavailable.push("credits") + const subscription = + isHttpError(subscriptionRaw) || isQuotaError(subscriptionRaw) + ? null + : parseSubscription(subscriptionRaw) + if (!subscription) unavailable.push("subscription") + + const summaryRaw = await safeRequest( + buildUrl("/alpha/usage/summary", { + orgId, + since: subscription?.currentPeriodStart ?? undefined, + }), + ) + if (isHttpError(summaryRaw) && isBlockingHttpError(summaryRaw)) { + return httpFailure(summaryRaw, "summary") + } + const summary = + isHttpError(summaryRaw) || isQuotaError(summaryRaw) ? null : parseSummary(summaryRaw) + if (!summary) unavailable.push("usage") + + if (!credits && !subscription && !summary) { + return { + ok: false, + error: { + kind: overallController.signal.aborted ? "timeout" : "http", + message: overallController.signal.aborted + ? "Command Code quota request timed out" + : "Command Code returned no recognized usage data for the account", + }, + } + } + + return { + ok: true, + quota: { + account, + credits, + subscription, + summary, + ...(unavailable.length > 0 ? { unavailable } : {}), + }, + } + } catch (error) { + if (error instanceof QuotaTimeoutError || overallController.signal.aborted) { + return { + ok: false, + error: { message: "Command Code quota request timed out", kind: "timeout" }, + } + } + return { + ok: false, + error: { + message: redactValue(`Failed to fetch Command Code quota: ${errorMessage(error)}`), + kind: "network", + }, + } + } finally { + clearTimeout(overallTimer) + } +} + +export function redactValue(value: string): string { + return redactCommandCodeErrorText(value) + .replace( + /("\s*(?:api[-_ ]?key|apikey|access[-_ ]?token|refresh[-_ ]?token|token|secret|password|authorization)\s*"\s*:\s*")([^"]{8,})/gi, + "$1[redacted]", + ) + .trim() +} diff --git a/src/runtime.ts b/src/runtime.ts index 34c034a..e103f96 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -28,11 +28,13 @@ export interface CommandCodeRuntimeOptions { cachePath: string loadModels: () => Promise createProviderConfig: (models: readonly CommandCodeModel[]) => TProviderConfig + getTransport?: () => "unknown" | "provider" | "generate" now?: () => number logWarning?: (message: string) => void } export interface CommandCodeRuntimeStatus { + transport: "unknown" | "provider" | "generate" source: LoadCommandCodeModelsResult["source"] modelCount: number lastSuccess?: number @@ -86,6 +88,7 @@ function formatTimestamp(timestamp: number | undefined): string { export function formatCommandCodeStatus(status: CommandCodeRuntimeStatus): string { const lines = [ + `transport: ${status.transport}`, `source: ${status.source}`, `model count: ${status.modelCount}`, `last success: ${formatTimestamp(status.lastSuccess)}`, @@ -113,6 +116,7 @@ export class CommandCodeRuntime console.warn(`[commandcode] ${message}`)) const initialStatus: CommandCodeRuntimeStatus = { + transport: "unknown", source: "empty", modelCount: 0, cachePath: options.cachePath, @@ -123,7 +127,10 @@ export class CommandCodeRuntime { @@ -259,10 +266,8 @@ export class CommandCodeRuntime { - ctx.ui.notify( - formatCommandCodeStatus(this.status), - this.status.warning ? "warning" : "info", - ) + const status = this.getStatus() + ctx.ui.notify(formatCommandCodeStatus(status), status.warning ? "warning" : "info") }, }) } diff --git a/src/transport.ts b/src/transport.ts new file mode 100644 index 0000000..c30171c --- /dev/null +++ b/src/transport.ts @@ -0,0 +1,140 @@ +import type { + AssistantMessageEvent, + AssistantMessageEventStreamLike, + ContextLike, + ModelLike, + StreamOptions, +} from "./types.ts" + +export type CommandCodeTransport = "unknown" | "provider" | "generate" + +interface TransportDependencies { + createStream: () => AssistantMessageEventStreamLike + streamProvider: ( + model: ModelLike, + context: ContextLike, + options?: StreamOptions, + ) => AssistantMessageEventStreamLike + streamGenerate: ( + model: ModelLike, + context: ContextLike, + options?: StreamOptions, + ) => AssistantMessageEventStreamLike +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +async function isUpgradeRequired(response: Response): Promise { + if (response.status !== 403) return false + + try { + const body: unknown = await response.clone().json() + if (!isRecord(body)) return false + const error = isRecord(body.error) ? body.error : body + return error.code === "upgrade_required" + } catch { + return false + } +} + +export function createCommandCodeTransportRouter(deps: TransportDependencies) { + let transport: CommandCodeTransport = "unknown" + let apiKey: string | undefined + + function pipe( + source: AssistantMessageEventStreamLike, + target: AssistantMessageEventStreamLike, + ): Promise { + return (async () => { + for await (const event of source) target.push(event) + })() + } + + return { + getTransport(): CommandCodeTransport { + return transport + }, + + reset(): void { + transport = "unknown" + apiKey = undefined + }, + + stream( + model: ModelLike, + context: ContextLike, + options?: StreamOptions, + ): AssistantMessageEventStreamLike { + if (options?.apiKey !== apiKey) { + apiKey = options?.apiKey + transport = "unknown" + } + const requestApiKey = options?.apiKey + if (transport === "generate") return deps.streamGenerate(model, context, options) + + const output = deps.createStream() + let upgradeRequired = false + const fetchImpl = options?.fetch ?? fetch + const providerOptions: StreamOptions = { + ...options, + fetch: async (input, init) => { + const response = await fetchImpl(input, init) + if (await isUpgradeRequired(response)) upgradeRequired = true + return response + }, + onResponse: async (response, responseModel) => { + if (upgradeRequired) return + await options?.onResponse?.(response, responseModel) + }, + } + + const run = async () => { + const providerStream = deps.streamProvider(model, context, providerOptions) + + for await (const event of providerStream) { + if (!upgradeRequired) { + if (apiKey === requestApiKey) transport = "provider" + output.push(event) + } + } + + if (upgradeRequired) { + if (apiKey === requestApiKey) transport = "generate" + await pipe(deps.streamGenerate(model, context, options), output) + } + output.end() + } + + run().catch((error: unknown) => { + const message = error instanceof Error ? error.message : String(error) + output.push({ + type: "error", + reason: "error", + error: { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "error", + errorMessage: message, + timestamp: Date.now(), + }, + }) + output.end() + }) + + return output + }, + } +} diff --git a/src/types.ts b/src/types.ts index c520543..d10f823 100644 --- a/src/types.ts +++ b/src/types.ts @@ -110,7 +110,10 @@ export interface StreamOptions { apiKey?: string signal?: AbortSignal headers?: Record + fetch?: typeof fetch maxTokens?: number + temperature?: number + sessionId?: string /** Resolved pi thinking level; forwarded only through the model's map. */ reasoning?: string onPayload?: (payload: unknown, model: ModelLike) => unknown | Promise @@ -171,6 +174,12 @@ export type AssistantMessageEvent = contentIndex: number partial: AssistantMessageLike } + | { + type: "toolcall_delta" + contentIndex: number + delta: string + partial: AssistantMessageLike + } | { type: "toolcall_end" contentIndex: number diff --git a/tests/fixtures/commandcode-model-ids.json b/tests/fixtures/commandcode-model-ids.json index f0974cd..2a69168 100644 --- a/tests/fixtures/commandcode-model-ids.json +++ b/tests/fixtures/commandcode-model-ids.json @@ -1,5 +1,5 @@ { - "fetchedAt": "2026-08-04T10:12:57.953Z", + "fetchedAt": "2026-08-25T13:32:11.631Z", "source": "https://api.commandcode.ai/provider/v1/models", "modelIds": [ "claude-sonnet-5", @@ -18,11 +18,13 @@ "gpt-5.4-mini", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-flash", + "deepseek/deepseek-v4-flash-vision-exp", "moonshotai/Kimi-K3", "moonshotai/Kimi-K2.7-Code", "moonshotai/Kimi-K2.7-Code-Highspeed", "moonshotai/Kimi-K2.6", "moonshotai/Kimi-K2.5", + "zai-org/GLM-5.3", "zai-org/GLM-5.2", "zai-org/GLM-5.2-Fast", "zai-org/GLM-5.1", @@ -33,6 +35,7 @@ "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.5", "Qwen/Qwen3.8-Max", + "Qwen/Qwen3.8-27B", "Qwen/Qwen3.7-Max", "Qwen/Qwen3.7-Plus", "Qwen/Qwen3.7-Flash", @@ -41,6 +44,7 @@ "stepfun/Step-3.7-Flash", "stepfun/Step-3.5-Flash", "tencent/hy3-paid", + "google/gemini-3.7-flash", "google/gemini-3.6-flash", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", @@ -49,9 +53,12 @@ "nvidia/nemotron-3-ultra-550b-a55b", "thinkingmachines/inkling", "thinkingmachines/inkling-small", + "stealth/ox-alpha", "poolside/laguna-s-2.1-free", - "inclusionai/ling-3.0-flash-free", "meta/muse-spark-1.1", - "xai/grok-4.5" + "meta/muse-spark-1.2", + "meta/muse-spark-1.2-contributor", + "xai/grok-4.5", + "xai/grok-4.6" ] } diff --git a/tests/fixtures/commandcode-pricing.json b/tests/fixtures/commandcode-pricing.json index deaee75..c65ae0c 100644 --- a/tests/fixtures/commandcode-pricing.json +++ b/tests/fixtures/commandcode-pricing.json @@ -1,5 +1,5 @@ { - "verifiedAt": "2026-08-04", + "verifiedAt": "2026-08-25", "source": "https://commandcode.ai/docs/resources/pricing-limits", "tierPolicy": "Use request-wide input tiers; the highest threshold exceeded by input plus cache tokens applies to the full request.", "tiers": { @@ -8,18 +8,18 @@ [32000, 0.1, 0.4, 0.02, 0.125], [256000, 0.2, 0.8, 0.04, 0.25] ], - "gpt-5.6-terra": [[272000, 2, 9, 0.2, 2.5]], - "gpt-5.6-luna": [[272000, 0.2, 0.9, 0.02, 0.25]] + "xai/grok-4.6": [[200000, 4, 12, 1, 0]] }, "costs": { - "poolside/laguna-s-2.1-free": [0, 0, 0, 0], - "inclusionai/ling-3.0-flash-free": [0, 0, 0, 0], - "tencent/hy3-paid": [0.14, 0.58, 0.035, 0], + "deepseek/deepseek-v4-pro": [0.66, 1.98, 0.022, 0], + "deepseek/deepseek-v4-flash": [0.22, 0.66, 0.007, 0], + "deepseek/deepseek-v4-flash-vision-exp": [0.22, 0.66, 0.007, 0], "moonshotai/Kimi-K3": [3, 15, 0.3, 0], "moonshotai/Kimi-K2.7-Code": [0.95, 4, 0.19, 0], "moonshotai/Kimi-K2.7-Code-Highspeed": [1.9, 8, 0.38, 0], "moonshotai/Kimi-K2.6": [0.95, 4, 0.16, 0], "moonshotai/Kimi-K2.5": [0.6, 3, 0.1, 0], + "zai-org/GLM-5.3": [1.4, 4.4, 0.26, 0], "zai-org/GLM-5.2": [1.4, 4.4, 0.26, 0], "zai-org/GLM-5.2-Fast": [3, 10.25, 0.5, 0], "zai-org/GLM-5.1": [1.4, 4.4, 0.26, 0], @@ -27,9 +27,10 @@ "MiniMaxAI/MiniMax-M3": [0.3, 1.2, 0.06, 0], "MiniMaxAI/MiniMax-M2.7": [0.3, 1.2, 0.06, 0], "MiniMaxAI/MiniMax-M2.5": [0.3, 1.2, 0.03, 0], - "deepseek/deepseek-v4-pro": [0.435, 0.87, 0.003625, 0], - "deepseek/deepseek-v4-flash": [0.14, 0.28, 0.0028, 0], + "xiaomi/mimo-v2.5-pro": [0.435, 0.87, 0.0036, 0], + "xiaomi/mimo-v2.5": [0.14, 0.28, 0.0028, 0], "Qwen/Qwen3.8-Max": [2, 6, 0.25, 2.5], + "Qwen/Qwen3.8-27B": [0.4, 3, 0.04, 0], "Qwen/Qwen3.7-Max": [2.5, 7.5, 0.5, 3.13], "Qwen/Qwen3.7-Plus": [0.4, 1.6, 0.08, 0.5], "Qwen/Qwen3.7-Flash": [0.03, 0.13, 0.006, 0.038], @@ -37,13 +38,12 @@ "Qwen/Qwen3.6-Plus": [0.5, 3, 0.1, 0], "stepfun/Step-3.7-Flash": [0.2, 1.15, 0.04, 0], "stepfun/Step-3.5-Flash": [0.1, 0.3, 0.02, 0], - "xiaomi/mimo-v2.5-pro": [0.435, 0.87, 0.0036, 0], - "xiaomi/mimo-v2.5": [0.14, 0.28, 0.0028, 0], + "tencent/hy3-paid": [0.14, 0.58, 0.035, 0], "nvidia/nemotron-3-ultra-550b-a55b": [0.6, 2.4, 0.12, 0], - "sakana/fugu-ultra": [5, 30, 0.5, 0], "thinkingmachines/inkling": [1, 4.05, 0.17, 0], "thinkingmachines/inkling-small": [0.5, 1.2, 0.1, 0], - "meta/muse-spark-1.1": [1.25, 4.25, 0.15, 0], + "poolside/laguna-s-2.1-free": [0, 0, 0, 0], + "stealth/ox-alpha": [0, 0, 0, 0], "claude-sonnet-5": [2, 10, 0.2, 2.5], "claude-sonnet-4-6": [3, 15, 0.3, 3.75], "claude-fable-5": [10, 50, 1, 12.5], @@ -52,16 +52,22 @@ "claude-opus-4-7": [5, 25, 0.5, 6.25], "claude-haiku-4-5-20251001": [1, 5, 0.1, 1.25], "gpt-5.6-sol": [5, 30, 0.5, 6.25], - "gpt-5.6-terra": [1, 6, 0.1, 1.25], - "gpt-5.6-luna": [0.1, 0.6, 0.01, 0.125], + "gpt-5.6-terra": [2, 12, 0.2, 2.5], + "gpt-5.6-luna": [0.2, 1.2, 0.02, 0.25], "gpt-5.5": [5, 30, 0.5, 0], "gpt-5.4": [2.5, 15, 0.25, 0], "gpt-5.3-codex": [2, 8, 0.5, 0], "gpt-5.4-mini": [0.75, 4.5, 0.075, 0], + "google/gemini-3.7-flash": [0.75, 3.75, 0.075, 0.04167], "google/gemini-3.6-flash": [1.5, 7.5, 0.15, 0], "google/gemini-3.5-flash": [1.5, 9, 0.15, 0], "google/gemini-3.5-flash-lite": [0.3, 2.5, 0.03, 0], "google/gemini-3.1-flash-lite": [0.25, 1.5, 0.03, 0], - "xai/grok-4.5": [2, 6, 0.5, 0] + "sakana/fugu-ultra": [5, 30, 0.5, 0], + "meta/muse-spark-1.1": [1.25, 4.25, 0.15, 0], + "meta/muse-spark-1.2": [1.25, 4.25, 0.15, 0], + "meta/muse-spark-1.2-contributor": [0.1, 0.2, 0.002, 0], + "xai/grok-4.5": [2, 6, 0.5, 0], + "xai/grok-4.6": [2, 6, 0.5, 0] } } diff --git a/tests/test-api-key.ts b/tests/test-api-key.ts new file mode 100644 index 0000000..f3029cb --- /dev/null +++ b/tests/test-api-key.ts @@ -0,0 +1,66 @@ +import assert from "node:assert/strict" +import { mkdtemp, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "node:test" + +import { getConfiguredApiKey } from "../src/api-key.ts" + +async function withAuthFile( + value: unknown, + run: (authPath: string) => Promise, +): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-commandcode-auth-")) + const authPath = join(directory, "auth.json") + try { + await writeFile(authPath, JSON.stringify(value), "utf-8") + await run(authPath) + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +describe("getConfiguredApiKey()", () => { + it("prefers the official environment variable and keeps the legacy alias", () => { + assert.equal( + getConfiguredApiKey({ + env: { COMMAND_CODE_API_KEY: "official-key", COMMANDCODE_API_KEY: "legacy-key" }, + authPaths: [], + }), + "official-key", + ) + assert.equal( + getConfiguredApiKey({ env: { COMMANDCODE_API_KEY: "legacy-key" }, authPaths: [] }), + "legacy-key", + ) + }) + + it("reads pi OAuth and API credentials", async () => { + const cases: readonly { credential: unknown; expected: string }[] = [ + { + credential: { commandcode: { type: "oauth", access: "oauth-key" } }, + expected: "oauth-key", + }, + { credential: { commandcode: { type: "api", key: "api-key" } }, expected: "api-key" }, + { credential: { "command-code": { type: "api", key: "cli-key" } }, expected: "cli-key" }, + { credential: { apiKey: "legacy-key" }, expected: "legacy-key" }, + ] + + for (const testCase of cases) { + await withAuthFile(testCase.credential, async (authPath) => { + assert.equal(getConfiguredApiKey({ env: {}, authPaths: [authPath] }), testCase.expected) + }) + } + }) + + it("ignores malformed files", async () => { + const directory = await mkdtemp(join(tmpdir(), "pi-commandcode-auth-")) + const authPath = join(directory, "auth.json") + try { + await writeFile(authPath, "not json", "utf-8") + assert.equal(getConfiguredApiKey({ env: {}, authPaths: [authPath] }), undefined) + } finally { + await rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/test-live-e2e.mjs b/tests/test-live-e2e.mjs index 160882a..f17ffa5 100644 --- a/tests/test-live-e2e.mjs +++ b/tests/test-live-e2e.mjs @@ -25,6 +25,22 @@ import { fileURLToPath } from "node:url" const projectDir = resolve(dirname(fileURLToPath(import.meta.url)), "..") const extensionPath = join(projectDir, "index.ts") const testModel = process.env.COMMANDCODE_E2E_MODEL ?? "deepseek/deepseek-v4-flash" +const testProfile = process.env.COMMANDCODE_E2E_PROFILE +const expectedTransport = + testProfile === "go" + ? "generate" + : testProfile === "goat" || testProfile === "provider" + ? "provider" + : undefined +const expectedPlan = + testProfile === "go" + ? "go" + : testProfile === "goat" + ? "goat" + : testProfile === "provider" + ? "provider" + : undefined +const goatVisionModel = process.env.COMMANDCODE_E2E_GOAT_VISION_MODEL ?? "google/gemini-3.7-flash" const marker = "commandcode-live-e2e-ok" function findPiBinary() { @@ -45,6 +61,7 @@ function findPiBinary() { function hasAuthMetadata() { return ( + Boolean(process.env.COMMAND_CODE_API_KEY) || Boolean(process.env.COMMANDCODE_API_KEY) || existsSync(join(homedir(), ".commandcode", "auth.json")) || existsSync(join(homedir(), ".pi", "agent", "auth.json")) @@ -57,9 +74,19 @@ if (!piBin || !hasAuthMetadata()) { process.exit(0) } +const profileAgentDir = testProfile + ? mkdtempSync(join(tmpdir(), `pi-commandcode-live-${testProfile}-agent-`)) + : undefined + function safeEnv(overrides = {}) { const env = { ...process.env, PI_SKIP_VERSION_CHECK: "1", ...overrides } - delete env.COMMANDCODE_API_KEY + if (testProfile && profileAgentDir) { + env.PI_CODING_AGENT_DIR = profileAgentDir + env.COMMANDCODE_MODELS_CACHE = join(profileAgentDir, "commandcode-models.json") + } else { + delete env.COMMAND_CODE_API_KEY + delete env.COMMANDCODE_API_KEY + } return env } @@ -90,7 +117,7 @@ function run(command, args, options = {}) { }) } -async function runRpc(extension, action, timeoutMs = 120_000) { +async function runRpc(extension, action, timeoutMs = 120_000, model = testModel) { const child = spawn( piBin, [ @@ -102,7 +129,9 @@ async function runRpc(extension, action, timeoutMs = 120_000) { "--provider", "commandcode", "--model", - testModel, + model, + "--thinking", + "high", ], { cwd: projectDir, env: safeEnv(), stdio: ["pipe", "pipe", "pipe"] }, ) @@ -210,8 +239,19 @@ try { await waitFor( (event) => event.type === "response" && event.id === "reasoning-turn-1" && event.success, ) - await waitFor((event) => event.type === "agent_settled") - const firstThinkingDeltas = countThinkingDeltas(firstStart) + const firstSettled = await waitFor( + (event) => event.type === "agent_settled" && events.indexOf(event) >= firstStart, + ) + const firstSettledIndex = events.indexOf(firstSettled) + const firstThinkingDeltas = events + .slice(firstStart, firstSettledIndex + 1) + .filter( + (event) => + event.type === "message_update" && + event.assistantMessageEvent?.type === "thinking_delta" && + typeof event.assistantMessageEvent.delta === "string" && + event.assistantMessageEvent.delta.length > 0, + ).length const secondStart = events.length send({ @@ -223,8 +263,11 @@ try { await waitFor( (event) => event.type === "response" && event.id === "reasoning-turn-2" && event.success, ) - await waitFor((event) => event.type === "agent_settled" && events.indexOf(event) >= secondStart) + const secondSettled = await waitFor( + (event) => event.type === "agent_settled" && events.indexOf(event) >= secondStart, + ) const secondThinkingDeltas = countThinkingDeltas(secondStart) + assert.ok(events.indexOf(secondSettled) >= secondStart) return { firstThinkingDeltas, secondThinkingDeltas, stderr: getStderr() } }) @@ -234,6 +277,14 @@ try { console.log("[live-e2e] live runtime refresh/status commands") const runtime = await runRpc(extensionPath, async ({ send, waitFor, getStderr }) => { + if (expectedTransport) { + send({ id: "transport-probe", type: "prompt", message: `Reply exactly: ${marker}` }) + await waitFor( + (event) => event.type === "response" && event.id === "transport-probe" && event.success, + ) + await waitFor((event) => event.type === "agent_settled") + } + send({ id: "commands", type: "get_commands" }) const commands = await waitFor( (event) => event.type === "response" && event.id === "commands" && event.success, @@ -259,14 +310,66 @@ try { typeof event.message === "string" && event.message.includes("source:"), ) - return { names, refresh: refresh.message, status: status.message, stderr: getStderr() } + + send({ id: "quota", type: "prompt", message: "/commandcode-quota" }) + await waitFor((event) => event.type === "response" && event.id === "quota" && event.success) + const quota = await waitFor( + (event) => + event.type === "extension_ui_request" && + event.method === "notify" && + typeof event.message === "string" && + event.message.includes("Plan:"), + ) + return { + names, + refresh: refresh.message, + status: status.message, + quota: quota.message, + stderr: getStderr(), + } }) assert.ok(runtime.names.includes("commandcode-refresh")) assert.ok(runtime.names.includes("commandcode-status")) + assert.ok(runtime.names.includes("commandcode-quota")) assert.match(runtime.refresh, /model catalog (?:refreshed|unchanged)/) + if (expectedTransport) assert.match(runtime.status, new RegExp(`transport: ${expectedTransport}`)) assert.match(runtime.status, /source: (?:live|cache)/) assert.match(runtime.status, /model count: [1-9][0-9]*/) - assert.doesNotMatch(`${runtime.refresh}\n${runtime.status}\n${runtime.stderr}`, /Bearer\s+\S+/i) + if (expectedPlan) assert.match(runtime.quota, new RegExp(`Plan:.*\\b${expectedPlan}\\b`, "i")) + assert.doesNotMatch( + `${runtime.refresh}\n${runtime.status}\n${runtime.quota}\n${runtime.stderr}`, + /Bearer\s+\S+/i, + ) + + console.log("[live-e2e] live abort through real RPC host") + const abortResult = await runRpc(extensionPath, async ({ send, waitFor, events, getStderr }) => { + const startIndex = events.length + send({ + id: "abort-turn", + type: "prompt", + message: "Write a very long detailed explanation of every integer from 1 to 10000.", + }) + await waitFor( + (event) => event.type === "response" && event.id === "abort-turn" && event.success, + ) + await waitFor((event) => event.type === "message_update" && events.indexOf(event) >= startIndex) + send({ id: "abort", type: "abort" }) + await waitFor((event) => event.type === "response" && event.id === "abort" && event.success) + await waitFor((event) => event.type === "agent_settled" && events.indexOf(event) >= startIndex) + return { + aborted: events + .slice(startIndex) + .some( + (event) => + event.type === "message_end" && + event.message?.role === "assistant" && + event.message?.stopReason === "aborted", + ), + stderr: getStderr(), + } + }) + assert.equal(abortResult.aborted, true) + assert.doesNotMatch(abortResult.stderr, /Bearer\s+\S+/i) console.log("[live-e2e] live tool-call round trip") const toolRoot = join(tempRoot, "tool-roundtrip") @@ -294,32 +397,70 @@ try { ) assert.equal(toolResult.code, 0, toolResult.stderr) assert.match(toolResult.stdout, new RegExp(marker)) - assert.equal(readFileSync(targetPath, "utf-8"), marker) + assert.equal(readFileSync(targetPath, "utf-8").trimEnd(), marker) - console.log("[live-e2e] image rejection through real RPC host") - const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => { - send({ - id: "image", - type: "prompt", - message: "Describe this image", - images: [{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }], - }) - await waitFor((event) => event.type === "response" && event.id === "image") - await waitFor( - (event) => - event.type === "message_end" && - event.message?.role === "assistant" && - event.message?.stopReason === "error", + if (testProfile === "goat") { + console.log("[live-e2e] live vision request through Provider API") + const vision = await runRpc( + extensionPath, + async ({ send, waitFor, events, getStderr }) => { + const startIndex = events.length + send({ + id: "vision", + type: "prompt", + message: "Describe the attached image briefly.", + images: [ + { + type: "image", + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", + mimeType: "image/png", + }, + ], + }) + await waitFor( + (event) => event.type === "response" && event.id === "vision" && event.success, + ) + await waitFor( + (event) => event.type === "agent_settled" && events.indexOf(event) >= startIndex, + ) + const messageEnd = events + .slice(startIndex) + .find((event) => event.type === "message_end" && event.message?.role === "assistant") + return { messageEnd, stderr: getStderr() } + }, + 180_000, + goatVisionModel, ) - return events - }) - assert.ok( - image.some( - (event) => - event.type === "message_end" && - /does not support image content/i.test(event.message?.errorMessage ?? ""), - ), - ) + assert.notEqual(vision.messageEnd?.message?.stopReason, "error") + assert.doesNotMatch(vision.stderr, /Bearer\s+\S+/i) + } + + if (testProfile === "go") { + console.log("[live-e2e] image rejection through real RPC host") + const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => { + send({ + id: "image", + type: "prompt", + message: "Describe this image", + images: [{ type: "image", data: "iVBORw0KGgo=", mimeType: "image/png" }], + }) + await waitFor((event) => event.type === "response" && event.id === "image") + await waitFor( + (event) => + event.type === "message_end" && + event.message?.role === "assistant" && + event.message?.stopReason === "error", + ) + return events + }) + assert.ok( + image.some( + (event) => + event.type === "message_end" && + /does not support image content/i.test(event.message?.errorMessage ?? ""), + ), + ) + } console.log("[live-e2e] packed artifact with existing authentication") const packDir = join(tempRoot, "pack") @@ -361,4 +502,5 @@ try { console.log("[live-e2e] PASS") } finally { rmSync(tempRoot, { recursive: true, force: true }) + if (profileAgentDir) rmSync(profileAgentDir, { recursive: true, force: true }) } diff --git a/tests/test-model-metadata-check.ts b/tests/test-model-metadata-check.ts new file mode 100644 index 0000000..97fe30e --- /dev/null +++ b/tests/test-model-metadata-check.ts @@ -0,0 +1,170 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { + commandCodeModelMetadataFromContents, + diffModelMetadata, + hasModelMetadataDiff, + parseBundleModelCapabilities, + parseKnownTextOnlyModelIds, + parseModelsReference, + parsePackageVersion, + renderCommandCodeCatalog, + updateReadmeCatalogVersion, + type CommandCodeModelMetadata, +} from "../.github/scripts/check-commandcode-model-metadata.ts" + +const MODELS_REFERENCE = ` +| Id (use EXACTLY this) | Name | Context | Efforts | $/1M in/out · cache read | Min plan | Best for | +|---|---|---|---|---|---|---| +| \`vision-model\` | Vision | 1M | low, high | $1/$2 | Go | images | +| \`text-model\` | Text | 200K | — | $1/$2 | Go | text | +` + +const CLI_BUNDLE = + 'const V={id:"vision-model",inputModalities:["text","image"],reasoning:!0,reasoningEfforts:["low","high"],maxOutputTokens:32768},T={id:"text-model",inputModalities:["text"]},catalog=new Set(["text-model"]),__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' + +describe("Command Code model metadata checker", () => { + it("parses model ids and reasoning efforts from the generated reference", () => { + assert.deepEqual(parseModelsReference(MODELS_REFERENCE), { + modelIds: ["text-model", "vision-model"], + reasoningEfforts: { "vision-model": ["low", "high"] }, + }) + }) + + it("extracts the text-only set from the bundled CLI catalog", () => { + assert.deepEqual(parseKnownTextOnlyModelIds(CLI_BUNDLE), ["text-model"]) + }) + + it("accepts one exact npm registry version and rejects stale-looking output shapes", () => { + assert.equal(parsePackageVersion("1.32.2"), "1.32.2") + assert.equal(parsePackageVersion("2.0.0-beta.1"), "2.0.0-beta.1") + assert.throws(() => parsePackageVersion(["1.32.1", "1.32.2"]), /one semantic version/) + assert.throws(() => parsePackageVersion("latest"), /one semantic version/) + }) + + it("derives image, reasoning, effort, and output-limit metadata", () => { + assert.deepEqual(parseBundleModelCapabilities(CLI_BUNDLE, ["text-model", "vision-model"]), { + reasoningModelIds: ["vision-model"], + maxOutputTokens: { "vision-model": 32_768 }, + }) + assert.deepEqual(commandCodeModelMetadataFromContents(MODELS_REFERENCE, CLI_BUNDLE), { + imageModelIds: ["vision-model"], + reasoningModelIds: ["vision-model"], + reasoningEfforts: { "vision-model": ["low", "high"] }, + maxOutputTokens: { "vision-model": 32_768 }, + }) + }) + + it("reports additions, removals, and changed reasoning efforts", () => { + const current: CommandCodeModelMetadata = { + imageModelIds: ["removed-image", "stable-image"], + reasoningModelIds: ["removed-reasoning", "stable-reasoning"], + reasoningEfforts: { + "changed-effort": ["low"], + "removed-effort": ["high"], + "stable-effort": ["low", "high"], + }, + maxOutputTokens: { "changed-output": 1, "removed-output": 2, "stable-output": 3 }, + } + const upstream: CommandCodeModelMetadata = { + imageModelIds: ["added-image", "stable-image"], + reasoningModelIds: ["added-reasoning", "stable-reasoning"], + reasoningEfforts: { + "added-effort": ["max"], + "changed-effort": ["low", "high"], + "stable-effort": ["low", "high"], + }, + maxOutputTokens: { "added-output": 4, "changed-output": 5, "stable-output": 3 }, + } + + const diff = diffModelMetadata(current, upstream) + + assert.deepEqual(diff, { + versionChanged: false, + addedImageModelIds: ["added-image"], + removedImageModelIds: ["removed-image"], + addedReasoningModelIds: ["added-reasoning"], + removedReasoningModelIds: ["removed-reasoning"], + addedEffortModelIds: ["added-effort"], + removedEffortModelIds: ["removed-effort"], + changedEffortModelIds: ["changed-effort"], + addedMaxOutputModelIds: ["added-output"], + removedMaxOutputModelIds: ["removed-output"], + changedMaxOutputModelIds: ["changed-output"], + }) + assert.equal(hasModelMetadataDiff(diff), true) + }) + + it("reports CLI version drift even when model metadata is unchanged", () => { + const metadata: CommandCodeModelMetadata = { + imageModelIds: ["vision-model"], + reasoningModelIds: ["vision-model"], + reasoningEfforts: { "vision-model": ["low"] }, + maxOutputTokens: { "vision-model": 32_768 }, + } + + const diff = diffModelMetadata(metadata, metadata, "1.32.2", "1.33.0") + + assert.equal(diff.versionChanged, true) + assert.equal(hasModelMetadataDiff(diff), true) + }) + + it("renders a deterministic generated catalog and updates the README version", () => { + assert.equal( + renderCommandCodeCatalog("1.33.0", { + imageModelIds: ["b-model", "a-model"], + reasoningModelIds: ["c-model", "a-model"], + reasoningEfforts: { + "b-model": ["high", "max"], + "a-model": ["low"], + }, + maxOutputTokens: { "b-model": 32_768 }, + }), + `export const COMMAND_CODE_CLI_VERSION = "1.33.0" + +export type CommandCodeInputType = "text" | "image" +export type CommandCodeReasoningEffort = "minimal" | "low" | "medium" | "high" | "xhigh" | "max" + +/** + * Generated from command-code@1.33.0 by \`npm run sync:commandcode-catalog\`. + * Do not edit manually. + */ +export const MODEL_INPUT_MODALITIES: Readonly> = { + "a-model": ["text", "image"], + "b-model": ["text", "image"], +} + +export const MODEL_REASONING: Readonly> = { + "a-model": true, + "c-model": true, +} + +export const MODEL_EFFORTS: Readonly> = { + "a-model": ["low"], + "b-model": ["high", "max"], +} + +export const MODEL_MAX_OUTPUT_TOKENS: Readonly> = { + "b-model": 32_768, +} +`, + ) + assert.equal( + updateReadmeCatalogVersion( + "The capability snapshot currently follows `command-code@1.32.2`.", + "1.33.0", + ), + "The capability snapshot currently follows `command-code@1.33.0`.", + ) + }) + + it("rejects unexpected upstream structures instead of silently passing", () => { + assert.throws(() => parseModelsReference("# no catalog"), /No model rows/) + assert.throws( + () => parseModelsReference(MODELS_REFERENCE.replace("low, high", "low, turbo")), + /Unexpected reasoning efforts/, + ) + assert.throws(() => parseKnownTextOnlyModelIds("const unrelated = true"), /Could not find/) + }) +}) diff --git a/tests/test-models.ts b/tests/test-models.ts index af96ebb..c40d321 100644 --- a/tests/test-models.ts +++ b/tests/test-models.ts @@ -4,7 +4,10 @@ import { tmpdir } from "node:os" import { join } from "node:path" import { describe, it } from "node:test" +import { COMMAND_CODE_CLI_VERSION } from "../src/commandcode-catalog.ts" import { + apiForModelId, + baseUrlForModel, commandCodeModelsFromApiResponse, commandCodeModelsFromCache, DEFAULT_MODELS_TIMEOUT_MS, @@ -13,6 +16,8 @@ import { loadCommandCodeModels, MODEL_EFFORTS, MODEL_INPUT_MODALITIES, + MODEL_MAX_OUTPUT_TOKENS, + MODEL_REASONING, modelSupportsImageInput, thinkingLevelMapForEfforts, thinkingMetadataForModel, @@ -37,7 +42,8 @@ const EXPECTED_MODELS: readonly CommandCodeModel[] = [ { id: "Qwen/Qwen3.7-Max", name: "Qwen 3.7 Max (CC)", - reasoning: false, + api: "openai-completions", + reasoning: true, contextWindow: 1_000_000, maxTokens: 65_536, }, @@ -84,61 +90,108 @@ describe("commandCodeModelsFromApiResponse()", () => { assert.deepEqual(commandCodeModelsFromApiResponse(API_RESPONSE), EXPECTED_MODELS) }) - it("matches command-code@1.15.1 image input capabilities", () => { - assert.deepEqual(inputModalitiesForModel("gpt-5.6-luna"), ["text", "image"]) - assert.deepEqual(inputModalitiesForModel("meta/muse-spark-1.2"), ["text", "image"]) - assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-pro"), ["text"]) - assert.deepEqual(inputModalitiesForModel("unknown-new-model"), ["text"]) - assert.equal(modelSupportsImageInput("gpt-5.6-luna"), true) - assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-pro"), false) - assert.equal(Object.keys(MODEL_INPUT_MODALITIES).length, 37) + it("routes Claude models to Anthropic Messages and all others to Chat Completions", () => { + assert.equal(apiForModelId("claude-sonnet-4-6"), "anthropic-messages") + assert.equal(apiForModelId("gpt-5.6-sol"), "openai-completions") + assert.equal( + baseUrlForModel("https://api.commandcode.ai/provider/v1/", "openai-completions"), + "https://api.commandcode.ai/provider/v1", + ) + assert.equal( + baseUrlForModel("https://api.commandcode.ai/provider/v1/", "anthropic-messages"), + "https://api.commandcode.ai/provider", + ) }) - it("marks only known reasoning models as reasoning-capable", () => { + it(`uses the command-code@${COMMAND_CODE_CLI_VERSION} image capability catalog`, () => { + assert.deepEqual(inputModalitiesForModel("gpt-5.6-luna"), ["text", "image"]) + assert.deepEqual(inputModalitiesForModel("meta/muse-spark-1.2"), ["text", "image"]) + assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-flash-vision-exp"), [ + "text", + "image", + ]) + assert.deepEqual(inputModalitiesForModel("Qwen/Qwen3.8-27B"), ["text", "image"]) + assert.deepEqual(inputModalitiesForModel("google/gemini-3.7-flash"), ["text", "image"]) + assert.deepEqual(inputModalitiesForModel("stealth/ox-alpha"), ["text", "image"]) + assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-pro"), ["text"]) + assert.deepEqual(inputModalitiesForModel("zai-org/GLM-5.3"), ["text"]) + assert.deepEqual(inputModalitiesForModel("unknown-new-model"), ["text"]) + assert.equal(modelSupportsImageInput("gpt-5.6-luna"), true) + assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-flash-vision-exp"), true) + assert.equal(modelSupportsImageInput("stealth/ox-alpha"), true) + assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-pro"), false) + assert.ok(Object.keys(MODEL_INPUT_MODALITIES).length > 0) + for (const modalities of Object.values(MODEL_INPUT_MODALITIES)) { + assert.deepEqual(modalities, ["text", "image"]) + } + }) + + it("tracks reasoning independently from selectable effort levels", () => { const models = commandCodeModelsFromApiResponse({ object: "list", data: [ { ...API_RESPONSE.data[0], id: "deepseek/deepseek-v4-flash" }, + { ...API_RESPONSE.data[0], id: "moonshotai/Kimi-K3" }, { ...API_RESPONSE.data[0], id: "new-model-without-metadata" }, ], }) assert.equal(models[0]?.reasoning, true) - assert.equal(models[1]?.reasoning, false) + assert.equal(models[1]?.reasoning, true) + assert.deepEqual(thinkingMetadataForModel("moonshotai/Kimi-K3"), { + thinkingLevelMap: { + minimal: null, + low: null, + medium: null, + high: null, + xhigh: null, + max: null, + }, + }) + assert.equal(models[2]?.reasoning, false) + assert.equal(Object.keys(MODEL_REASONING).length, 48) }) - it("matches the exact command-code@1.15.1 reasoning effort catalog", () => { - assert.deepEqual(MODEL_EFFORTS, { - "Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"], - "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-4-7": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-4-8": ["low", "medium", "high", "xhigh", "max"], - "claude-opus-5": ["low", "medium", "high", "xhigh", "max"], - "claude-sonnet-4-6": ["low", "medium", "high", "xhigh", "max"], - "claude-sonnet-5": ["low", "medium", "high", "xhigh", "max"], - "deepseek/deepseek-v4-flash": ["high", "max"], - "deepseek/deepseek-v4-pro": ["high", "max"], - "gpt-5.3-codex": ["low", "medium", "high", "xhigh"], - "gpt-5.4": ["low", "medium", "high", "xhigh"], - "gpt-5.4-mini": ["low", "medium", "high"], - "gpt-5.5": ["low", "medium", "high", "xhigh"], - "gpt-5.6-luna": ["low", "medium", "high", "xhigh", "max"], - "gpt-5.6-sol": ["low", "medium", "high", "xhigh", "max"], - "gpt-5.6-terra": ["low", "medium", "high", "xhigh", "max"], - "google/gemini-3.1-flash-lite": ["low", "medium", "high"], - "google/gemini-3.5-flash": ["low", "medium", "high"], - "google/gemini-3.5-flash-lite": ["low", "medium", "high"], - "google/gemini-3.6-flash": ["low", "medium", "high"], - "sakana/fugu-ultra": ["high", "xhigh"], - "xai/grok-4.5": ["low", "medium", "high"], - "zai-org/GLM-5.2": ["high", "max"], + it("uses model-specific output limits from the CLI catalog", () => { + const models = commandCodeModelsFromApiResponse({ + object: "list", + data: [ + { ...API_RESPONSE.data[0], id: "Qwen/Qwen3.8-27B", context_length: 262_144 }, + { ...API_RESPONSE.data[0], id: "stealth/ox-alpha", context_length: 1_048_576 }, + { + ...API_RESPONSE.data[0], + id: "poolside/laguna-s-2.1-free", + context_length: 256_000, + }, + ], }) + + assert.deepEqual( + models.map(({ id, maxTokens }) => ({ id, maxTokens })), + [ + { id: "Qwen/Qwen3.8-27B", maxTokens: 32_768 }, + { id: "stealth/ox-alpha", maxTokens: 131_072 }, + { id: "poolside/laguna-s-2.1-free", maxTokens: 32_768 }, + ], + ) + assert.equal(Object.keys(MODEL_MAX_OUTPUT_TOKENS).length, 3) + }) + + it(`uses the command-code@${COMMAND_CODE_CLI_VERSION} reasoning effort catalog`, () => { + const validEfforts = new Set(["minimal", "low", "medium", "high", "xhigh", "max"]) + assert.ok(Object.keys(MODEL_EFFORTS).length > 0) + for (const efforts of Object.values(MODEL_EFFORTS)) { + assert.ok(efforts.length > 0) + assert.equal(new Set(efforts).size, efforts.length) + assert.ok(efforts.every((effort) => validEfforts.has(effort))) + } }) it("builds separate canonical pi and OMP metadata", () => { for (const [modelId, efforts] of Object.entries(MODEL_EFFORTS)) { const metadata = thinkingMetadataForModel(modelId) assert.ok(metadata, `${modelId} should have reasoning metadata`) + assert.ok(metadata.thinking) assert.equal(metadata.thinking.mode, "effort") assert.deepEqual(metadata.thinking.efforts, efforts) assert.deepEqual( diff --git a/tests/test-oauth.ts b/tests/test-oauth.ts index 0ff0fab..6a64381 100644 --- a/tests/test-oauth.ts +++ b/tests/test-oauth.ts @@ -9,7 +9,7 @@ import assert from "node:assert/strict" import { describe, it } from "node:test" import { startAuthServer, type AuthCallback } from "../src/auth-server.ts" -import { getApiKey, login, refreshToken, sanitizeApiKey } from "../src/oauth.ts" +import { getApiKey, login, refreshToken, sanitizeApiKey, validateApiKey } from "../src/oauth.ts" /** * Helper: wait for an HTTP server to close, or resolve immediately if already closed. @@ -24,9 +24,27 @@ function waitForClose(server: { }) } +async function withValidApiKeyFetch(run: () => Promise): Promise { + const originalFetch = globalThis.fetch + globalThis.fetch = (input, init) => { + if (String(input).endsWith("/alpha/whoami")) { + return Promise.resolve(new Response(JSON.stringify({ user: {} }), { status: 200 })) + } + return originalFetch(input, init) + } + try { + return await run() + } finally { + globalThis.fetch = originalFetch + } +} + describe("startAuthServer()", () => { it("starts on a localhost port and accepts a valid callback POST", async () => { - const { server, port, waitForCallback } = await startAuthServer({ startPort: 0 }) + const { server, port, waitForCallback } = await startAuthServer({ + startPort: 0, + expectedState: "test-state-token", + }) const callbackData: AuthCallback = { apiKey: "user_testKey123", @@ -57,6 +75,42 @@ describe("startAuthServer()", () => { await waitForClose(server) }) + it("rejects a mismatched state without closing the callback server", async () => { + const { server, port, waitForCallback } = await startAuthServer({ + startPort: 0, + expectedState: "correct-state", + }) + + const invalidResponse = await fetch(`http://127.0.0.1:${port}/callback`, { + method: "POST", + headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" }, + body: JSON.stringify({ + apiKey: "user_badState", + state: "wrong-state", + userId: "user_789", + userName: "Attacker", + keyName: "evil-key", + }), + }) + assert.equal(invalidResponse.status, 403) + assert.equal(server.listening, true) + + const validResponse = await fetch(`http://127.0.0.1:${port}/callback`, { + method: "POST", + headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" }, + body: JSON.stringify({ + apiKey: "user_valid", + state: "correct-state", + userId: "user_123", + userName: "Valid User", + keyName: "valid-key", + }), + }) + assert.equal(validResponse.status, 200) + assert.equal((await waitForCallback).apiKey, "user_valid") + await waitForClose(server) + }) + it("rejects when the callback indicates access_denied", async () => { const { server, port, waitForCallback } = await startAuthServer({ startPort: 0 }) @@ -176,6 +230,18 @@ describe("OAuth functions", () => { it("sanitizeApiKey removes paste markers, control chars, and whitespace", () => { assert.equal(sanitizeApiKey("\u001b[200~ user_manualKey\n\u001b[201~"), "user_manualKey") }) + + it("validates manual API keys through whoami", async () => { + await validateApiKey("valid-key", { + fetchImpl: () => Promise.resolve(new Response(JSON.stringify({ user: {} }), { status: 200 })), + }) + await assert.rejects( + validateApiKey("invalid-key", { + fetchImpl: () => Promise.resolve(new Response("unauthorized", { status: 401 })), + }), + /Invalid Command Code API key/, + ) + }) }) describe("login()", () => { @@ -186,7 +252,7 @@ describe("login()", () => { authUrl = params.url }, onPrompt(_params: { message: string }): Promise { - throw new Error("onPrompt should not be called in browser flow") + return Promise.resolve("") }, } @@ -239,21 +305,23 @@ describe("login()", () => { process.env.COMMANDCODE_AUTH_TIMEOUT_MS = "1" let authUrl = "" - let promptMessage = "" + const promptMessages: string[] = [] try { - const result = await login({ - onAuth(params: { url: string }) { - authUrl = params.url - }, - async onPrompt(params: { message: string }): Promise { - promptMessage = params.message - return "\u001b[200~ user_manualApiKey\n\u001b[201~" - }, - }) + const result = await withValidApiKeyFetch(() => + login({ + onAuth(params: { url: string }) { + authUrl = params.url + }, + async onPrompt(params: { message: string }): Promise { + promptMessages.push(params.message) + return promptMessages.length === 1 ? "" : "\u001b[200~ user_manualApiKey\n\u001b[201~" + }, + }), + ) assert.match(authUrl, /^https:\/\/commandcode\.ai\/studio\/auth\/cli\?/) - assert.match(promptMessage, /Paste your Command Code API key/) + assert.match(promptMessages[1] ?? "", /Paste your Command Code API key/) assert.equal(result.access, "user_manualApiKey") assert.equal(result.refresh, "user_manualApiKey") assert.ok(result.expires > Date.now(), "expiry should be far in the future") @@ -263,23 +331,53 @@ describe("login()", () => { } }) - it("rejects on state token mismatch", async () => { + it("accepts a directly pasted API key", async () => { + let authOpened = false + const result = await withValidApiKeyFetch(() => + login({ + onAuth() { + authOpened = true + }, + onPrompt(): Promise { + return Promise.resolve("user_directApiKey") + }, + }), + ) + + assert.equal(authOpened, false) + assert.equal(result.access, "user_directApiKey") + }) + + it("offers an explicit API key prompt", async () => { + let promptCount = 0 + const result = await withValidApiKeyFetch(() => + login({ + onAuth() { + throw new Error("browser should not open") + }, + onPrompt(): Promise { + promptCount += 1 + return Promise.resolve(promptCount === 1 ? "key" : "user_promptedApiKey") + }, + }), + ) + + assert.equal(result.access, "user_promptedApiKey") + assert.equal(promptCount, 2) + }) + + it("keeps waiting after a state mismatch and accepts the legitimate callback", async () => { let authUrl = "" const callbacks = { onAuth(params: { url: string }) { authUrl = params.url }, onPrompt(_params: { message: string }): Promise { - throw new Error("should not prompt") + return Promise.resolve("") }, } - const loginPromise: Promise = login(callbacks).then( - () => { - throw new Error("Expected login to reject") - }, - (e: Error) => e.message, - ) + const loginPromise = login(callbacks) // Wait for onAuth to be called asynchronously while (!authUrl) await new Promise((resolve) => setTimeout(resolve, 10)) @@ -287,8 +385,8 @@ describe("login()", () => { const url = new URL(authUrl) const port = parseInt(url.searchParams.get("callback")?.match(/localhost:(\d+)/)?.[1] ?? "0") - // Post back with a wrong state token - await fetch(`http://127.0.0.1:${port}/callback`, { + // Post back with a wrong state token. + const invalidResponse = await fetch(`http://127.0.0.1:${port}/callback`, { method: "POST", headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" }, body: JSON.stringify({ @@ -300,7 +398,20 @@ describe("login()", () => { }), }) - const errorMsg = await loginPromise - assert.match(errorMsg, /State token mismatch/) + assert.equal(invalidResponse.status, 403) + + const validResponse = await fetch(`http://127.0.0.1:${port}/callback`, { + method: "POST", + headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" }, + body: JSON.stringify({ + apiKey: "user_goodState", + state: url.searchParams.get("state"), + userId: "user_123", + userName: "Real User", + keyName: "real-key", + }), + }) + assert.equal(validResponse.status, 200) + assert.equal((await loginPromise).access, "user_goodState") }) }) diff --git a/tests/test-omp-compat.mjs b/tests/test-omp-compat.mjs index 04f8c78..ced7c03 100644 --- a/tests/test-omp-compat.mjs +++ b/tests/test-omp-compat.mjs @@ -50,6 +50,9 @@ let modelListRequestCount = 0 let lastRequestBody let requestBodies = [] let lastRequestHeaders = {} +// When true the mock Provider API answers 403 upgrade_required so the +// transport router falls back to the legacy /alpha/generate transport. +let providerUpgradeRequired = false const server = createServer((req, res) => { if (req.method === "GET" && req.url === "/provider/v1/models") { @@ -81,6 +84,51 @@ const server = createServer((req, res) => { return } + if (req.method === "POST" && req.url === "/provider/v1/chat/completions") { + requestCount += 1 + lastRequestHeaders = Object.fromEntries( + Object.entries(req.headers).map(([key, value]) => [ + key, + Array.isArray(value) ? value.join(", ") : (value ?? ""), + ]), + ) + + let body = "" + req.on("data", (chunk) => { + body += chunk.toString("utf-8") + }) + req.on("end", () => { + try { + lastRequestBody = JSON.parse(body) + requestBodies.push(lastRequestBody) + } catch { + lastRequestBody = undefined + } + + if (providerUpgradeRequired) { + res.writeHead(403, { "Content-Type": "application/json; charset=utf-8" }) + res.end(JSON.stringify({ error: { code: "upgrade_required" } })) + return + } + + res.writeHead(200, { + "Content-Type": "text/event-stream; charset=utf-8", + "Transfer-Encoding": "chunked", + }) + res.write( + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: "mock-omp-ok" }, finish_reason: null }] })}\n\n`, + ) + res.write( + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n`, + ) + res.write( + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })}\n\n`, + ) + res.end("data: [DONE]\n\n") + }) + return + } + if (req.method !== "POST" || req.url !== "/alpha/generate") { res.writeHead(404) res.end("Not found") @@ -95,13 +143,13 @@ const server = createServer((req, res) => { ]), ) - let body = "" + let generateBody = "" req.on("data", (chunk) => { - body += chunk.toString("utf-8") + generateBody += chunk.toString("utf-8") }) req.on("end", () => { try { - lastRequestBody = JSON.parse(body) + lastRequestBody = JSON.parse(generateBody) requestBodies.push(lastRequestBody) } catch { lastRequestBody = undefined @@ -133,8 +181,8 @@ function runOmp(args, timeoutMs = 30_000) { HOME: tempHome, USERPROFILE: tempHome, PI_CODING_AGENT_DIR: join(tempHome, ".omp", "agent"), - COMMANDCODE_API_KEY: "mock-key", - COMMANDCODE_API_BASE: apiBase, + COMMAND_CODE_API_KEY: "mock-key", + COMMANDCODE_API_BASE: `${apiBase}/provider/v1`, COMMANDCODE_MODELS_URL: `${apiBase}/provider/v1/models`, }, stdio: ["ignore", "pipe", "pipe"], @@ -165,29 +213,25 @@ function runOmp(args, timeoutMs = 30_000) { try { console.log("[omp-compat] list models through real extension") modelListRequestCount = 0 - const list = await runOmp(["models", "--json", "-e", EXT_PATH, "--no-extensions"]) - if (list.code !== 0 && /unknown|unrecognized/i.test(list.stderr + list.stdout)) { - console.log("[omp-compat] SKIP models phase - omp models subcommand unavailable") - } else { - assert.equal(list.code, 0, list.stderr) - let listed = null - try { - listed = JSON.parse(list.stdout) - } catch { - listed = null - } - const models = Array.isArray(listed?.models) ? listed.models : [] - assert.ok( - models.some((model) => model.provider === "commandcode"), - "commandcode provider should be listed", - ) - assert.ok( - models.some((model) => model.id === TEST_MODEL), - "mock catalog model should be listed", - ) - assert.ok(modelListRequestCount >= 1) - assert.doesNotMatch(list.stdout + list.stderr, /Failed to load extension/) + // Prefer the flag form `omp -e EXT --list-models`; Homebrew's `omp` + // distribution only exposes the `omp models` subcommand, so fall back to + // that form when the flag invocation is not recognized. + let result = await runOmp(["-e", EXT_PATH, "--list-models"]) + if (result.code !== 0) { + result = await runOmp(["models", "-e", EXT_PATH]) } + assert.equal(result.code, 0, result.stderr) + const listOutput = result.stdout || result.stderr + assert.match(listOutput, /commandcode/) + assert.match(listOutput, /deepseek\/deepseek-v4-flash/) + // The failed flag attempt may already load the extension and fetch the + // catalog once before the subcommand fallback runs, so only assert that + // the mock catalog was actually consulted. + assert.ok(modelListRequestCount >= 1) + assert.doesNotThrow(() => + accessSync(join(tempHome, ".omp", "agent", "commandcode-models.json"), constants.R_OK), + ) + assert.doesNotMatch(result.stdout + result.stderr, /Failed to load extension/) console.log("[omp-compat] print mode through real extension and mock API") requestCount = 0 @@ -204,15 +248,13 @@ try { "Bearer mock-key", "should send the resolved env-var value, not the literal var name", ) - assert.equal(lastRequestBody?.params?.model, TEST_MODEL) - assert.equal(typeof lastRequestBody?.params?.system, "string") - assert.doesNotThrow(() => - accessSync(join(tempHome, ".omp", "agent", "commandcode-models.json"), constants.R_OK), - ) + assert.equal(lastRequestBody?.model, TEST_MODEL) + assert.ok(Array.isArray(lastRequestBody?.messages)) - console.log("[omp-compat] developer advisory reaches the provider request body") + console.log("[omp-compat] developer advisory reaches the legacy generate request body") requestCount = 0 requestBodies = [] + providerUpgradeRequired = true const advisoryRun = await runOmp( [ "-e", diff --git a/tests/test-pi-authenticated.mjs b/tests/test-pi-authenticated.mjs index 7f992b3..39fa536 100644 --- a/tests/test-pi-authenticated.mjs +++ b/tests/test-pi-authenticated.mjs @@ -22,7 +22,7 @@ const { writeFileSync } = require("node:fs") writeFileSync(process.env.FAKE_PI_LOG, JSON.stringify({ args: process.argv.slice(2), agentDir: process.env.PI_CODING_AGENT_DIR ?? null, - apiKey: process.env.COMMANDCODE_API_KEY ?? null, + apiKey: process.env.COMMAND_CODE_API_KEY ?? process.env.COMMANDCODE_API_KEY ?? null, skipVersionCheck: process.env.PI_SKIP_VERSION_CHECK, })) NODE @@ -38,7 +38,8 @@ NODE PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ""}`, FAKE_PI_LOG: logPath, PI_CODING_AGENT_DIR: "/existing/pi-agent", - COMMANDCODE_API_KEY: "existing-key", + COMMAND_CODE_API_KEY: "official-existing-key", + COMMANDCODE_API_KEY: "legacy-existing-key", }, encoding: "utf8", }) diff --git a/tests/test-pi-isolated.mjs b/tests/test-pi-isolated.mjs index 0b809a6..3ca85c6 100644 --- a/tests/test-pi-isolated.mjs +++ b/tests/test-pi-isolated.mjs @@ -26,7 +26,8 @@ appendFileSync(process.env.FAKE_PI_LOG, JSON.stringify({ skipVersionCheck: process.env.PI_SKIP_VERSION_CHECK, home: process.env.HOME, userProfile: process.env.USERPROFILE, - inheritedApiKey: process.env.COMMANDCODE_API_KEY ?? null, + inheritedApiKey: + process.env.COMMAND_CODE_API_KEY ?? process.env.COMMANDCODE_API_KEY ?? null, }) + "\\n") NODE if [ "$1" = "install" ]; then exit 0; fi @@ -42,7 +43,8 @@ exit ${exitStatus} ...process.env, PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ""}`, FAKE_PI_LOG: logPath, - COMMANDCODE_API_KEY: "must-not-leak", + COMMAND_CODE_API_KEY: "must-not-leak-official", + COMMANDCODE_API_KEY: "must-not-leak-legacy", }, encoding: "utf8", }) diff --git a/tests/test-pi-local.mjs b/tests/test-pi-local.mjs index 5099d08..3bce2c8 100644 --- a/tests/test-pi-local.mjs +++ b/tests/test-pi-local.mjs @@ -15,7 +15,8 @@ import { fileURLToPath } from "node:url" const __dirname = dirname(fileURLToPath(import.meta.url)) const PROJECT_DIR = resolve(__dirname, "..") const EXT_PATH = resolve(PROJECT_DIR, "index.ts") -const TEST_MODEL = "deepseek/deepseek-v4-flash" +const TEST_MODEL = "gpt-5.4" +const CLAUDE_TEST_MODEL = "claude-sonnet-4-6" function findPiBinary() { if (process.env.PI_BIN) return process.env.PI_BIN @@ -63,9 +64,17 @@ function modelCatalog() { object: "model", created: 1779824324, owned_by: "command-code", - name: "DeepSeek V4 Flash", + name: "GPT 5.4", context_length: 1_000_000, }, + { + id: CLAUDE_TEST_MODEL, + object: "model", + created: 1779824324, + owned_by: "command-code", + name: "Claude Sonnet 4.6", + context_length: 200_000, + }, { id: "cc-second-model", object: "model", @@ -101,7 +110,9 @@ const server = createServer((req, res) => { return } - if (req.method !== "POST" || req.url !== "/alpha/generate") { + const isOpenAIRequest = req.method === "POST" && req.url === "/provider/v1/chat/completions" + const isAnthropicRequest = req.method === "POST" && req.url === "/provider/v1/messages" + if (!isOpenAIRequest && !isAnthropicRequest) { res.writeHead(404) res.end("Not found") return @@ -129,12 +140,20 @@ const server = createServer((req, res) => { if (overflowMode && overflowRequestCount === 2) { res.writeHead(400, { "Content-Type": "application/json; charset=utf-8" }) - res.end(JSON.stringify({ error: { message: "Input exceeds context limit" } })) + res.end( + JSON.stringify({ + error: { + message: "Input exceeds context limit", + type: "invalid_request_error", + code: "context_length_exceeded", + }, + }), + ) return } res.writeHead(200, { - "Content-Type": "text/plain; charset=utf-8", + "Content-Type": "text/event-stream; charset=utf-8", "Transfer-Encoding": "chunked", }) const text = overflowMode @@ -144,11 +163,36 @@ const server = createServer((req, res) => { ? "compaction-summary" : "overflow-recovered" : "mock-pi-ok" - res.write(`${JSON.stringify({ type: "text-delta", text })}\n`) + if (isAnthropicRequest) { + res.write( + `event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "mock", type: "message", role: "assistant", content: [], model: CLAUDE_TEST_MODEL, stop_reason: null, stop_sequence: null, usage: { input_tokens: 1, output_tokens: 0 } } })}\n\n`, + ) + res.write( + `event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } })}\n\n`, + ) + res.write( + `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text } })}\n\n`, + ) + res.write( + `event: content_block_stop\ndata: ${JSON.stringify({ type: "content_block_stop", index: 0 })}\n\n`, + ) + res.write( + `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 1 } })}\n\n`, + ) + res.end(`event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n`) + return + } + res.write( - `${JSON.stringify({ type: "finish", finishReason: "stop", totalUsage: { inputTokens: 1, outputTokens: 1 } })}\n`, + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: text }, finish_reason: null }] })}\n\n`, ) - res.end() + res.write( + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })}\n\n`, + ) + res.write( + `data: ${JSON.stringify({ id: "mock", object: "chat.completion.chunk", choices: [], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } })}\n\n`, + ) + res.end("data: [DONE]\n\n") }) }) @@ -170,8 +214,9 @@ const env = { USERPROFILE: tempHome, PI_CODING_AGENT_DIR: agentDir, PI_CODING_AGENT_SESSION_DIR: join(tempHome, "sessions"), - COMMANDCODE_API_BASE: apiBase, - COMMANDCODE_API_KEY: "mock-key", + COMMANDCODE_API_BASE: `${apiBase}/provider/v1`, + COMMAND_CODE_API_KEY: "mock-key", + CMD_ZDR: "1", COMMANDCODE_MODELS_URL: `${apiBase}/provider/v1/models`, } @@ -403,7 +448,7 @@ async function runRpcExtensionCommands(timeoutMs = 30_000) { event.type === "extension_ui_request" && event.method === "notify" && typeof event.message === "string" && - event.message.includes("model count: 2"), + event.message.includes("model count: 3"), ) includeRefreshedModel = true @@ -414,7 +459,7 @@ async function runRpcExtensionCommands(timeoutMs = 30_000) { event.type === "extension_ui_request" && event.method === "notify" && typeof event.message === "string" && - event.message.includes("3 models from live"), + event.message.includes("4 models from live"), ) send({ id: "status-after", type: "prompt", message: "/commandcode-status" }) @@ -426,7 +471,7 @@ async function runRpcExtensionCommands(timeoutMs = 30_000) { event.type === "extension_ui_request" && event.method === "notify" && typeof event.message === "string" && - event.message.includes("model count: 3"), + event.message.includes("model count: 4"), ) return { @@ -565,7 +610,7 @@ try { ) assert.equal(recoveryList.code, 0, recoveryList.stderr) const recoveryOutput = recoveryList.stdout || recoveryList.stderr - assert.match(recoveryOutput, /deepseek\/deepseek-v4-flash/) + assert.match(recoveryOutput, /gpt-5\.4/) assert.match(recoveryOutput, /cc-second-model/) assert.doesNotMatch(recoveryList.stderr, /no valid cached catalog/) assert.doesNotMatch(recoveryList.stderr, /Failed to load extension/) @@ -578,7 +623,7 @@ try { assert.equal(list.code, 0, list.stderr) const listOutput = list.stdout || list.stderr assert.match(listOutput, /commandcode/) - assert.match(listOutput, /deepseek\/deepseek-v4-flash/) + assert.match(listOutput, /gpt-5\.4/) assert.match(listOutput, /cc-second-model/) assert.equal(modelListRequestCount, 1) assert.doesNotThrow(() => accessSync(modelsCachePath, constants.R_OK)) @@ -591,7 +636,7 @@ try { ) assert.equal(offlineList.code, 0, offlineList.stderr) const offlineListOutput = offlineList.stdout || offlineList.stderr - assert.match(offlineListOutput, /deepseek\/deepseek-v4-flash/) + assert.match(offlineListOutput, /gpt-5\.4/) assert.match(offlineListOutput, /cc-second-model/) assert.match(offlineList.stderr, /Using the cached catalog/) @@ -659,27 +704,55 @@ try { lastRequestHeaders.authorization.startsWith("Bearer "), "should send a bearer Authorization header", ) - assert.equal(lastRequestBody?.params?.model, TEST_MODEL) - assert.equal(lastRequestBody?.params?.reasoning_effort, "high") - const sentTools = lastRequestBody?.params?.tools + assert.equal(lastRequestHeaders["x-cmd-zdr"], "1") + assert.equal(lastRequestBody?.model, TEST_MODEL) + assert.equal(lastRequestBody?.reasoning_effort, "high") + const sentTools = lastRequestBody?.tools assert.ok(Array.isArray(sentTools) && sentTools.length > 0) - const editTool = sentTools.find((tool) => tool.name === "edit") - assert.equal(editTool?.input_schema?.type, "object") - assert.equal(editTool?.input_schema?.properties?.edits?.type, "array") - assert.equal(editTool?.input_schema?.properties?.edits?.items?.type, "object") + const editTool = sentTools.find((tool) => tool.function?.name === "edit") + assert.equal(editTool?.function?.parameters?.type, "object") + assert.equal(editTool?.function?.parameters?.properties?.edits?.type, "array") + assert.equal(editTool?.function?.parameters?.properties?.edits?.items?.type, "object") assert.equal( - editTool?.input_schema?.properties?.edits?.items?.properties?.oldText?.type, + editTool?.function?.parameters?.properties?.edits?.items?.properties?.oldText?.type, "string", ) + console.log("[pi-local] Claude request through Anthropic Messages endpoint") + requestCount = 0 + const claudePrint = await runPi( + [ + "--no-extensions", + "-e", + EXT_PATH, + "-p", + "say mock token", + "--provider", + "commandcode", + "--model", + CLAUDE_TEST_MODEL, + "--thinking", + "high", + ], + 30_000, + ) + assert.equal(claudePrint.code, 0, claudePrint.stderr) + assert.match(claudePrint.stdout, /mock-pi-ok/) + assert.equal(requestCount, 1) + assert.equal(lastRequestBody?.model, CLAUDE_TEST_MODEL) + assert.equal(lastRequestBody?.thinking?.type, "adaptive") + assert.deepEqual(lastRequestBody?.output_config, { effort: "high" }) + assert.equal(lastRequestHeaders["x-api-key"], "mock-key") + assert.equal(lastRequestHeaders["x-cmd-zdr"], "1") + console.log("[pi-local] runtime commands through real RPC extension lifecycle") includeRefreshedModel = false const runtimeCommands = await runRpcExtensionCommands() assert.ok(runtimeCommands.commandNames.includes("commandcode-refresh")) assert.ok(runtimeCommands.commandNames.includes("commandcode-status")) assert.match(runtimeCommands.statusBefore, /source: live/) - assert.match(runtimeCommands.refreshNotification, /3 models from live/) - assert.match(runtimeCommands.statusAfter, /model count: 3/) + assert.match(runtimeCommands.refreshNotification, /4 models from live/) + assert.match(runtimeCommands.statusAfter, /model count: 4/) assert.doesNotMatch( `${runtimeCommands.statusBefore}\n${runtimeCommands.statusAfter}\n${runtimeCommands.stderr}`, /mock-key/, @@ -702,7 +775,7 @@ try { assert.equal(rpc.sawTextDelta, true) assert.equal(requestCount, 1) - console.log("[pi-local] reject image input through real RPC preflight/provider path") + console.log("[pi-local] forward image input through the documented provider schema") requestCount = 0 const imageRpc = await runRpcQuery(10_000, "describe image", [], { images: [ @@ -713,14 +786,15 @@ try { }, ], }) - assert.equal(requestCount, 0) + assert.equal(imageRpc.ok, true, imageRpc.stderr) + assert.equal(requestCount, 1) + const imageContent = lastRequestBody?.messages?.find( + (message) => message.role === "user", + )?.content + assert.ok(Array.isArray(imageContent), JSON.stringify(lastRequestBody?.messages)) assert.ok( - imageRpc.events.some( - (event) => - event.type === "message_end" && - event.message?.role === "assistant" && - event.message?.stopReason === "error", - ) || imageRpc.stderr.includes("does not support image"), + imageContent.some((part) => part.type === "image_url"), + JSON.stringify(imageContent), ) console.log("[pi-local] verify overflow normalization and compaction recovery") @@ -729,8 +803,7 @@ try { const overflowRpc = await runRpcOverflowRecovery() assert.equal(overflowRpc.ok, true) assert.ok(overflowRpc.requests >= 4) - assert.equal(overflowRpc.sawNormalizedOverflow, true) - assert.equal(overflowRpc.sawCompactionRetry, true) + assert.equal(overflowRpc.sawCompactionRetry, true, JSON.stringify(overflowRpc)) assert.equal(overflowRpc.stderrHasSecrets, false) overflowMode = false diff --git a/tests/test-pricing.ts b/tests/test-pricing.ts index a782a73..d7141a5 100644 --- a/tests/test-pricing.ts +++ b/tests/test-pricing.ts @@ -27,7 +27,7 @@ const fixtureUrl = new URL("./fixtures/commandcode-model-ids.json", import.meta. const fixture = JSON.parse(await readFile(fixtureUrl, "utf-8")) as ModelCatalogSnapshot const pricingFixtureUrl = new URL("./fixtures/commandcode-pricing.json", import.meta.url) const pricingFixture = JSON.parse(await readFile(pricingFixtureUrl, "utf-8")) as PricingSnapshot -const freeModels = new Set(["poolside/laguna-s-2.1-free", "inclusionai/ling-3.0-flash-free"]) +const freeModels = new Set(["poolside/laguna-s-2.1-free", "stealth/ox-alpha"]) function assertCost( modelId: string, @@ -50,7 +50,7 @@ function assertCost( describe("MODEL_COSTS pricing overlay", () => { it("covers the current Command Code model catalog snapshot", () => { assert.equal(fixture.source, "https://api.commandcode.ai/provider/v1/models") - assert.match(fixture.fetchedAt, /^2026-08-04T/) + assert.match(fixture.fetchedAt, /^2026-08-25T/) const catalogIds = [...fixture.modelIds].sort() const pricedIds = Object.keys(MODEL_COSTS).sort() @@ -108,10 +108,16 @@ describe("MODEL_COSTS pricing overlay", () => { }) it("matches corrected official rates", () => { + assertCost("deepseek/deepseek-v4-pro", { + input: 0.66, + output: 1.98, + cacheRead: 0.022, + cacheWrite: 0, + }) assertCost("deepseek/deepseek-v4-flash", { - input: 0.14, - output: 0.28, - cacheRead: 0.0028, + input: 0.22, + output: 0.66, + cacheRead: 0.007, cacheWrite: 0, }) assertCost("Qwen/Qwen3.7-Max", { @@ -132,6 +138,24 @@ describe("MODEL_COSTS pricing overlay", () => { cacheRead: 0.03, cacheWrite: 0, }) + assertCost("Qwen/Qwen3.8-27B", { + input: 0.4, + output: 3, + cacheRead: 0.04, + cacheWrite: 0, + }) + assertCost("google/gemini-3.7-flash", { + input: 0.75, + output: 3.75, + cacheRead: 0.075, + cacheWrite: 0.04167, + }) + assertCost("meta/muse-spark-1.2-contributor", { + input: 0.1, + output: 0.2, + cacheRead: 0.002, + cacheWrite: 0, + }) }) it("uses the documented base rates for context-dependent models", () => { @@ -148,16 +172,31 @@ describe("MODEL_COSTS pricing overlay", () => { cacheWrite: 0.038, }) assertCost("gpt-5.6-terra", { - input: 1, - output: 6, - cacheRead: 0.1, - cacheWrite: 1.25, + input: 2, + output: 12, + cacheRead: 0.2, + cacheWrite: 2.5, }) + assertCost("gpt-5.6-luna", { + input: 0.2, + output: 1.2, + cacheRead: 0.02, + cacheWrite: 0.25, + }) + assert.deepEqual(MODEL_COSTS["xai/grok-4.6"]?.tiers, [ + { + inputTokensAbove: 200_000, + input: 4, + output: 12, + cacheRead: 1, + cacheWrite: 0, + }, + ]) }) it("tracks pricing provenance", () => { assert.equal(PRICING_SOURCE_URL, "https://commandcode.ai/docs/resources/pricing-limits") - assert.equal(PRICING_LAST_VERIFIED, "2026-08-04") + assert.equal(PRICING_LAST_VERIFIED, "2026-08-25") }) it("fails once temporary pricing needs review", () => { diff --git a/tests/test-pure-functions.ts b/tests/test-pure-functions.ts index c5d235a..bc320d4 100644 --- a/tests/test-pure-functions.ts +++ b/tests/test-pure-functions.ts @@ -16,6 +16,7 @@ import { mapFinishReason, messagesToCC, parseStreamEventLine, + pickCommandCodeApiKey, projectSlugFromPath, textContent, toJsonSchema, @@ -26,8 +27,18 @@ import { redactCommandCodeErrorText } from "../src/overflow.ts" import { objectAt } from "./helpers.ts" describe("getApiKey()", () => { - it("uses COMMANDCODE_API_KEY from provided env", () => { - assert.equal(getApiKey({ env: { COMMANDCODE_API_KEY: "env-key" }, authPaths: [] }), "env-key") + it("uses the official API key env var before the legacy alias", () => { + assert.equal( + getApiKey({ + env: { COMMAND_CODE_API_KEY: "official-key", COMMANDCODE_API_KEY: "legacy-key" }, + authPaths: [], + }), + "official-key", + ) + assert.equal( + getApiKey({ env: { COMMANDCODE_API_KEY: "legacy-key" }, authPaths: [] }), + "legacy-key", + ) }) it("reads apiKey, commandcode, pi OAuth, and official CLI credential fields", () => { @@ -106,6 +117,39 @@ describe("error redaction", () => { }) }) +describe("pickCommandCodeApiKey()", () => { + it("falls back to the host key for a placeholder registry value", () => { + assert.equal(pickCommandCodeApiKey("$COMMAND_CODE_API_KEY", "file-key"), "file-key") + assert.equal(pickCommandCodeApiKey("COMMAND_CODE_API_KEY", "file-key"), "file-key") + assert.equal(pickCommandCodeApiKey("$COMMANDCODE_API_KEY", "file-key"), "file-key") + assert.equal(pickCommandCodeApiKey("COMMANDCODE_API_KEY", "file-key"), "file-key") + }) + + it("returns undefined when only a placeholder is provided (no fallback)", () => { + assert.equal(pickCommandCodeApiKey("$COMMAND_CODE_API_KEY", undefined), undefined) + assert.equal(pickCommandCodeApiKey("$COMMANDCODE_API_KEY", undefined), undefined) + }) + + it("prefers a real registry key over the host fallback", () => { + assert.equal(pickCommandCodeApiKey("real-registry-key", "file-key"), "real-registry-key") + }) + + it("falls back to the host key when the registry has none", () => { + assert.equal(pickCommandCodeApiKey(undefined, "file-key"), "file-key") + assert.equal(pickCommandCodeApiKey(undefined, undefined), undefined) + }) + + it("falls back to the host key for empty or whitespace registry values", () => { + assert.equal(pickCommandCodeApiKey("", "file-key"), "file-key") + assert.equal(pickCommandCodeApiKey(" ", "file-key"), "file-key") + assert.equal(pickCommandCodeApiKey(" ", undefined), undefined) + }) + + it("trims a real registry key", () => { + assert.equal(pickCommandCodeApiKey(" real-registry-key ", "file-key"), "real-registry-key") + }) +}) + describe("projectSlugFromPath()", () => { it("matches the official CLI-style slug from an absolute working directory", () => { assert.equal( @@ -117,7 +161,7 @@ describe("projectSlugFromPath()", () => { }) describe("text-only image handling", () => { - it("rejects image content for models without image support", () => { + it("rejects direct image input for models without image support", () => { assert.throws( () => assertTextOnlyMessages([ @@ -128,16 +172,17 @@ describe("text-only image handling", () => { ]), /does not support image content/i, ) - assert.throws( - () => - assertTextOnlyMessages([ - { - role: "toolResult", - toolCallId: "c1", - content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], - }, - ]), - /does not support image content/i, + }) + + it("allows historical tool-result images to be omitted for text-only models", () => { + assert.doesNotThrow(() => + assertTextOnlyMessages([ + { + role: "toolResult", + toolCallId: "c1", + content: [{ type: "image", data: "base64-data", mimeType: "image/png" }], + }, + ]), ) }) }) @@ -168,6 +213,12 @@ describe("textContent()", () => { ) }) + it("normalizes malformed string and object content", () => { + assert.equal(textContent({ content: "raw result" }), "raw result") + assert.equal(textContent({ content: { ok: true } }), '{"ok":true}') + assert.equal(textContent({ content: null }), "") + }) + it("handles empty or missing content", () => { assert.equal(textContent({ content: [] }), "") assert.equal(textContent({}), "") @@ -359,7 +410,7 @@ describe("toJsonSchema()", () => { if (!outputProperties || typeof outputProperties !== "object") { throw new Error("expected object properties") } - assert.ok(Object.prototype.hasOwnProperty.call(outputProperties, "__proto__")) + assert.ok(Object.hasOwn(outputProperties, "__proto__")) assert.deepEqual(Object.getOwnPropertyDescriptor(outputProperties, "__proto__")?.value, { type: "string", }) @@ -500,6 +551,23 @@ describe("messagesToCC()", () => { assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "hello\nworld") }) + it("preserves malformed string tool results instead of sending empty output", () => { + const result = messagesToCC([ + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: "raw result", + }, + ]) + + assert.equal(objectAt(result, ["1", "content", "0", "output", "value"]), "raw result") + }) + it("serializes image inputs in the current Command Code wire format", () => { assert.deepEqual( messagesToCC( @@ -530,6 +598,48 @@ describe("messagesToCC()", () => { ) }) + it("omits tool-result images for text-only models while preserving their text", () => { + const result = messagesToCC([ + { role: "user", content: "read image" }, + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: [ + { type: "text", text: "image attached" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }, + ], + }, + ]) + + assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "image attached") + assert.equal(objectAt(result, ["3"]), undefined) + }) + + it("describes an omitted image-only tool result for text-only models", () => { + const result = messagesToCC([ + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }], + }, + ]) + + assert.equal( + objectAt(result, ["1", "content", "0", "output", "value"]), + "[Image omitted: model does not support images]", + ) + }) + it("preserves tool-result images as a following user image message", () => { const result = messagesToCC( [ @@ -601,7 +711,7 @@ describe("messagesToCC()", () => { ]) }) - it("drops orphaned tool calls that have no matching tool result", () => { + it("synthesizes missing results for orphaned tool calls", () => { const result = messagesToCC([ { role: "user", content: "edit a file" }, { @@ -620,7 +730,12 @@ describe("messagesToCC()", () => { assert.equal(objectAt(result, ["1", "role"]), "assistant") assert.equal(objectAt(result, ["1", "content", "0", "type"]), "text") - assert.equal(objectAt(result, ["1", "content", "1"]), undefined) + assert.equal(objectAt(result, ["1", "content", "1", "type"]), "tool-call") + assert.equal(objectAt(result, ["2", "role"]), "tool") + assert.match( + String(objectAt(result, ["2", "content", "0", "output", "value"])), + /did not complete/, + ) }) it("handles empty conversations", () => { diff --git a/tests/test-quota-command.ts b/tests/test-quota-command.ts new file mode 100644 index 0000000..3ae753c --- /dev/null +++ b/tests/test-quota-command.ts @@ -0,0 +1,117 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { registerCommandCodeQuota, type QuotaCommandContext } from "../src/quota-command.ts" +import type { CommandCodeQuotaResult } from "../src/quota-types.ts" + +class CommandApiDouble { + handler?: (args: string, ctx: QuotaCommandContext) => Promise + + registerCommand( + name: string, + options: { + description: string + handler: (args: string, ctx: QuotaCommandContext) => Promise + }, + ): void { + assert.equal(name, "commandcode-quota") + assert.match(options.description, /usage and quota/) + this.handler = options.handler + } +} + +function context(registryKey: string | undefined) { + const notifications: Array<{ message: string; type?: "info" | "warning" | "error" }> = [] + let waited = false + const value = { + async waitForIdle() { + waited = true + }, + modelRegistry: { + async getApiKeyForProvider(provider: string) { + assert.equal(provider, "commandcode") + return registryKey + }, + }, + ui: { + notify(message: string, type?: "info" | "warning" | "error") { + notifications.push({ message, type }) + }, + }, + } satisfies QuotaCommandContext + return { value, notifications, waited: () => waited } +} + +const quotaResult: CommandCodeQuotaResult = { + ok: true, + quota: { + account: { login: "alice", orgId: null }, + credits: null, + subscription: null, + summary: { totalCost: 1, totalCount: 2 }, + }, +} + +describe("commandcode-quota command", () => { + it("registers the command and resolves OMP placeholders through the fallback key", async () => { + const pi = new CommandApiDouble() + let requestKey = "" + let requestBase = "" + registerCommandCodeQuota(pi, { + apiBase: "https://api.commandcode.ai", + getConfiguredKey: () => "fallback-key", + fetchQuota: async (options) => { + requestKey = options.apiKey + requestBase = options.baseUrl ?? "" + return quotaResult + }, + }) + + assert.ok(pi.handler) + const ctx = context("$COMMAND_CODE_API_KEY") + await pi.handler("", ctx.value) + assert.equal(ctx.waited(), true) + assert.equal(requestKey, "fallback-key") + assert.equal(requestBase, "https://api.commandcode.ai") + assert.equal(ctx.notifications.at(-1)?.type, "info") + assert.match(ctx.notifications.at(-1)?.message ?? "", /Requests: 2/) + }) + + it("warns without calling the endpoint when no API key is available", async () => { + const pi = new CommandApiDouble() + let called = false + registerCommandCodeQuota(pi, { + apiBase: "https://api.commandcode.ai", + getConfiguredKey: () => undefined, + fetchQuota: async () => { + called = true + return quotaResult + }, + }) + + assert.ok(pi.handler) + const ctx = context(undefined) + await pi.handler("", ctx.value) + assert.equal(called, false) + assert.equal(ctx.notifications.at(-1)?.type, "warning") + assert.match(ctx.notifications.at(-1)?.message ?? "", /requires an API key/) + }) + + it("redacts endpoint failures before notifying the host", async () => { + const pi = new CommandApiDouble() + registerCommandCodeQuota(pi, { + apiBase: "https://api.commandcode.ai", + getConfiguredKey: () => "real-key", + fetchQuota: async () => ({ + ok: false, + error: { kind: "http", message: "api_key=supersecretvalue123456 failed" }, + }), + }) + + assert.ok(pi.handler) + const ctx = context("real-key") + await pi.handler("", ctx.value) + assert.equal(ctx.notifications.at(-1)?.type, "error") + assert.doesNotMatch(ctx.notifications.at(-1)?.message ?? "", /supersecret/) + }) +}) diff --git a/tests/test-quota.ts b/tests/test-quota.ts new file mode 100644 index 0000000..30f5a9c --- /dev/null +++ b/tests/test-quota.ts @@ -0,0 +1,417 @@ +/** + * Unit tests for the Command Code quota layer (src/quota.ts). + * + * These are hermetic: no pi runtime and no network. Fetching is exercised with + * a mocked `fetchImpl`, while parsing and formatting are pure function checks. + */ + +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { formatQuota, formatWindowLimits } from "../src/quota-format.ts" +import { + DEFAULT_API_BASE, + fetchCommandCodeQuota, + redactValue, + windowLimitsFromCredits, +} from "../src/quota.ts" +import type { + CommandCodeCredits, + CommandCodeQuota, + CommandCodeWindowLimit, +} from "../src/quota-types.ts" + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }) +} + +function okFetch(handlers: Record) { + const urls: string[] = [] + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + urls.push(url) + for (const [needle, body] of Object.entries(handlers)) { + if (url.includes(needle)) return jsonResponse(body) + } + throw new Error(`Unexpected URL: ${url}`) + } + return { fetchImpl, urls: () => urls } +} + +describe("Command Code quota", () => { + it("parses window limits from the credits windowLimits object", () => { + const limits = windowLimitsFromCredits({ + limited: true, + // resetAt as reported by the live API: milliseconds since epoch. + fiveHour: { used: 8, cap: 14, resetAt: 1_700_000_000_000 }, + weekly: { used: 30, cap: 35, resetAt: 1_700_000_000_000 }, + }) + assert.deepEqual(limits, [ + { window: "fiveHour", used: 8, cap: 14, resetAt: 1_700_000_000 }, + { window: "weekly", used: 30, cap: 35, resetAt: 1_700_000_000 }, + ]) + }) + + it("skips empty window limit entries", () => { + const limits = windowLimitsFromCredits({ + limited: false, + fiveHour: { used: 0, cap: 0, resetAt: null }, + weekly: { used: 0, cap: 0, resetAt: null }, + }) + assert.deepEqual(limits, []) + }) + + it("parses resetAt as numeric string or ISO timestamp string", () => { + const limits = windowLimitsFromCredits({ + fiveHour: { used: 1, cap: 2, resetAt: "1700000000000" }, + weekly: { used: 1, cap: 2, resetAt: "2023-11-14T22:13:20.000Z" }, + }) + // numeric ms string -> epoch seconds; ISO string -> epoch seconds + assert.equal(limits[0]?.resetAt, 1_700_000_000) + assert.equal(limits[1]?.resetAt, 1_700_000_000) + }) + + it("renders valid zero usage without claiming an unknown billing period", () => { + const quota: CommandCodeQuota = { + account: { login: "alice", orgId: null }, + credits: null, + subscription: null, + summary: { totalCost: 0, totalCount: 0 }, + } + const output = formatQuota(quota, () => 1_700_000_000_000) + assert.match(output, /Usage\n/) + assert.doesNotMatch(output, /billing period/) + assert.match(output, /Requests: 0/) + }) + + it("formats window limits with percentage and reset clock", () => { + const limits: CommandCodeWindowLimit[] = [ + { window: "fiveHour", used: 7, cap: 14, resetAt: 1_700_000_000 }, + { window: "weekly", used: 0, cap: 35, resetAt: null }, + ] + const lines = formatWindowLimits(limits) + assert.match(lines[0] ?? "", /^5-hour: 7\.00 \/ 14\.00 credits \(50% used\) \(resets/) + assert.match(lines[1] ?? "", /^Weekly: 0\.00 \/ 35\.00 credits \(0% used\)/) + }) + + it("uses the injected clock for the reset countdown", () => { + const limit: CommandCodeWindowLimit = { + window: "fiveHour", + used: 7, + cap: 14, + resetAt: 1_700_000_000, // seconds since epoch + } + // now() shortly before reset -> a short "in Nm" countdown + const soon = formatWindowLimits([limit], () => 1_699_999_000 * 1000)[0] + assert.match(soon ?? "", /\(resets in \d+m\)/) + // already past reset -> "soon" + const past = formatWindowLimits([limit], () => 1_700_100_000 * 1000)[0] + assert.match(past ?? "", /\(resets soon\)/) + }) + + it("fetches and normalizes the full quota snapshot", async () => { + const { fetchImpl, urls } = okFetch({ + whoami: { user: { userName: "alice" }, org: { id: "org_1", login: "alice-inc" } }, + credits: { + credits: { + monthlyCredits: 40, + purchasedCredits: 10, + freeCredits: 5, + planId: "pro", + }, + windowLimits: { + fiveHour: { used: 8, cap: 16, resetAt: 1_700_000_000_000 }, + weekly: { used: 20, cap: 40, resetAt: null }, + }, + }, + subscriptions: { + data: { + planId: "pro", + status: "active", + currentPeriodStart: "2026-01-01T00:00:00Z", + currentPeriodEnd: "2026-02-01T00:00:00Z", + }, + }, + summary: { totalCost: 12.34, totalCount: 1500 }, + }) + + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, true) + if (!result.ok) return + + assert.equal(result.quota.account.login, "alice-inc") + assert.equal(result.quota.account.orgId, "org_1") + assert.deepEqual(result.quota.credits?.remainingCredits, 55) + assert.equal(result.quota.credits?.windowLimits.length, 2) + assert.equal(result.quota.subscription?.planId, "pro") + assert.equal(result.quota.summary?.totalCost, 12.34) + + // Regression: requested URLs must carry the base exactly once (no + // double prefix), and all hit the alpha usage endpoints. + const fetched = urls() + assert.equal(fetched.length, 4) + for (const url of fetched) { + assert.ok( + /^https:\/\/api\.commandcode\.ai\/alpha\//.test(url), + `expected base-prefixed alpha URL, got: ${url}`, + ) + assert.equal((url.match(/https:\/\//g) ?? []).length, 1) + assert.equal(url.includes(`${DEFAULT_API_BASE}${DEFAULT_API_BASE}`), false) + } + }) + + it("rejects unrecognized successful endpoint schemas instead of displaying zero usage", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + return jsonResponse({ changed: "schema" }) + } + + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.error.kind, "http") + assert.match(result.error.message, /no recognized usage data/i) + }) + + it("degrades gracefully when individual billing endpoints fail", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + if (url.includes("summary")) return jsonResponse({ totalCost: 3.0, totalCount: 10 }) + if (url.includes("credits") || url.includes("subscriptions")) { + return jsonResponse({ error: "boom" }, 500) + } + throw new Error(`Unexpected URL: ${url}`) + } + + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, true) + if (!result.ok) return + assert.equal(result.quota.credits, null) + assert.equal(result.quota.summary?.totalCost, 3.0) + assert.deepEqual(result.quota.unavailable, ["credits", "subscription"]) + assert.match(formatQuota(result.quota), /Unavailable: credits, subscription/) + // Optional aggregate tokens are parsed when the summary reports them. + assert.equal(result.quota.summary?.totalTokens, undefined) + }) + + it("degrades on thrown network failures from optional endpoints, not just HTTP 5xx", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + if (url.includes("summary")) return jsonResponse({ totalCost: 3.0, totalCount: 10 }) + if (url.includes("credits")) throw new Error("network down") + if (url.includes("subscriptions")) return jsonResponse({ data: { planId: "pro" } }) + throw new Error(`Unexpected URL: ${url}`) + } + + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, true) + if (!result.ok) return + assert.equal(result.quota.credits, null) + assert.equal(result.quota.subscription?.planId, "pro") + assert.equal(result.quota.summary?.totalCost, 3.0) + assert.deepEqual(result.quota.unavailable, ["credits"]) + }) + + it("fails the command when the summary endpoint rejects auth/permission", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + if (url.includes("credits")) return jsonResponse({ credits: { monthlyCredits: 5 } }) + if (url.includes("subscriptions")) return jsonResponse({ data: { planId: "pro" } }) + if (url.includes("summary")) return jsonResponse({ error: "nope" }, 403) + throw new Error(`Unexpected URL: ${url}`) + } + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.error.kind, "http") + assert.match(result.error.message, /summary/) + }) + + it("does not treat 429 on billing endpoints as fatal", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + if (url.includes("summary")) return jsonResponse({ totalCost: 3.0, totalCount: 10 }) + if (url.includes("credits") || url.includes("subscriptions")) { + return jsonResponse({ error: "rate limited" }, 429) + } + throw new Error(`Unexpected URL: ${url}`) + } + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, true) + if (!result.ok) return + assert.equal(result.quota.credits, null) + assert.equal(result.quota.summary?.totalCost, 3.0) + }) + + it("sends extra headers (ZDR) on quota requests", async () => { + let sent: Headers | undefined + const fetchImpl = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + sent = (init?.headers as Headers) ?? undefined + const url = String(input) + if (url.includes("whoami")) return jsonResponse({ user: { userName: "alice" }, org: null }) + if (url.includes("credits")) return jsonResponse({ credits: { monthlyCredits: 5 } }) + if (url.includes("subscriptions")) return jsonResponse({ data: { planId: "pro" } }) + if (url.includes("summary")) return jsonResponse({ totalCost: 1, totalCount: 1 }) + throw new Error(`Unexpected URL: ${url}`) + } + const result = await fetchCommandCodeQuota({ + apiKey: "cc_test_key", + fetchImpl, + extraHeaders: { "x-cmd-zdr": "1" }, + }) + assert.equal(result.ok, true) + const headers = new Headers(sent) + assert.equal(headers.get("x-cmd-zdr"), "1") + }) + + it("parses optional token count and key name when present", async () => { + const { fetchImpl } = okFetch({ + whoami: { user: { userName: "alice", keyName: "Pi Agent" }, org: null }, + credits: { credits: { monthlyCredits: 5, purchasedCredits: 0, freeCredits: 0 } }, + subscriptions: { data: { planId: "pro", status: "active" } }, + summary: { totalCost: 1.06, totalCount: 654, totalTokens: 74_200_000 }, + }) + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, true) + if (!result.ok) return + assert.equal(result.quota.summary?.totalTokens, 74_200_000) + assert.equal(result.quota.account.keyName, "Pi Agent") + }) + + it("rejects missing API keys as a config error", async () => { + const result = await fetchCommandCodeQuota({ apiKey: "" }) + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.error.kind, "config") + }) + + it("fails with a config-style error when the API key is rejected", async () => { + const fetchImpl = async (input: RequestInfo | URL): Promise => { + if (String(input).includes("whoami")) return jsonResponse({ error: "unauthorized" }, 401) + throw new Error(`Unexpected URL: ${input}`) + } + const result = await fetchCommandCodeQuota({ apiKey: "cc_bad_key", fetchImpl }) + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.error.kind, "http") + assert.match(result.error.message, /401/) + }) + + it("formats a complete quota snapshot into readable output", () => { + const quota: CommandCodeQuota = { + account: { login: "alice-inc", orgId: "org_1" }, + credits: { + monthlyCredits: 40, + purchasedCredits: 10, + freeCredits: 5, + remainingCredits: 55, + windowLimits: [ + { window: "fiveHour", used: 8, cap: 16, resetAt: null }, + { window: "weekly", used: 20, cap: 40, resetAt: null }, + ], + } satisfies CommandCodeCredits, + subscription: { + planId: "pro", + status: "active", + currentPeriodStart: "2026-01-01T00:00:00Z", + currentPeriodEnd: "2026-02-01T00:00:00Z", + }, + summary: { totalCost: 12.34, totalCount: 1500 }, + } + + const output = formatQuota(quota, () => 1_700_000_000_000) + assert.doesNotMatch(output, /Command Code quota —/) + assert.match(output, /Credits/) + assert.match(output, /Remaining: \$55\.00 of \$67\.34/) + assert.match(output, /Used: \$12\.34/) + assert.match(output, /Sources: monthly \$40\.00 \/ purchased \$10\.00 \/ free \$5\.00/) + assert.match(output, /Plan: pro \(active\)/) + assert.match(output, /Usage \(billing period\)/) + assert.match(output, /Cost: \$12\.34/) + assert.match(output, /Requests: 1,500/) + assert.match(output, /Account/) + assert.match(output, /alice-inc/) + assert.match(output, /5-hour: 8\.00 \/ 16\.00 credits/) + assert.match(output, /Weekly: 20\.00 \/ 40\.00 credits/) + assert.match(output, /https:\/\/commandcode\.ai\/usage/) + }) + + it("redacts token-like values from error messages", () => { + // 16+ char run after a credential key is redacted by the shared redactor. + assert.equal(redactValue("api_key=abcdefghijklmnop123456"), "api_key=[redacted]") + assert.equal(redactValue("Bearer user_12345678901234 failed"), "Bearer [redacted] failed") + }) + + it("redacts named credential fields and short tokens from error bodies", () => { + // Credential key-value forms (with = or : separator) are redacted. + assert.equal(redactValue("api_key=abc123"), "api_key=[redacted]") + assert.equal( + redactValue("authorization=Basic abc:def failed"), + "authorization=[redacted] abc:def failed", + ) + assert.equal(redactValue("user_123456789 failed"), "[redacted] failed") + assert.equal(redactValue("cc_abcdefghijkl failed"), "[redacted] failed") + assert.equal( + redactValue("token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.secret"), + "token=[redacted]", + ) + }) + + it("redacts JSON-quoted credential fields in error bodies", () => { + assert.equal( + redactValue('{"apiKey":"sk-abcdefghijklmnop123456","ok":true}'), + '{"apiKey":"[redacted]","ok":true}', + ) + assert.equal( + redactValue('{"error":"bad","access_token":"opaque-internal-token-12345"}'), + '{"error":"bad","access_token":"[redacted]"}', + ) + assert.equal( + redactValue('{"authorization":"Bearer user_1234"}'), + '{"authorization":"[redacted]"}', + ) + }) + + it("redacts thrown network errors from the outer catch path", async () => { + const fetchImpl = async (_input: RequestInfo | URL): Promise => { + throw new Error("connection reset by proxy api_key=supersecretvalue123456") + } + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl }) + assert.equal(result.ok, false) + if (result.ok) return + assert.doesNotMatch(result.error.message, /supersecretvalue123456/) + assert.match(result.error.kind, /network/) + }) + + it("honors the overall deadline once it has already fired (no phase starts after abort)", async () => { + const start = Date.now() + const fetchImpl = async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = String(input) + if (url.includes("whoami")) { + // Never resolve; let the per-request controller abort it at timeoutMs. + return new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => + reject(Object.assign(new Error("aborted"), { name: "AbortError" })), + ) + }) + } + throw new Error(`Unexpected URL: ${url}`) + } + + const result = await fetchCommandCodeQuota({ apiKey: "cc_test_key", fetchImpl, timeoutMs: 30 }) + const elapsed = Date.now() - start + assert.equal(result.ok, false) + if (result.ok) return + assert.equal(result.error.kind, "timeout") + // The overall deadline governs the whole command; no phase may add ~30ms on top. + assert.ok(elapsed < 200, `elapsed ${elapsed}ms exceeded overall deadline`) + }) +}) diff --git a/tests/test-runtime.ts b/tests/test-runtime.ts index c8b0af0..2a89f12 100644 --- a/tests/test-runtime.ts +++ b/tests/test-runtime.ts @@ -49,6 +49,7 @@ class CommandContext implements CommandCodeCommandContext { const FIRST_MODEL: CommandCodeModel = { id: "first-model", name: "First Model", + api: "openai-completions", reasoning: true, contextWindow: 128_000, maxTokens: 16_384, @@ -57,6 +58,7 @@ const FIRST_MODEL: CommandCodeModel = { const SECOND_MODEL: CommandCodeModel = { id: "second-model", name: "Second Model", + api: "openai-completions", reasoning: true, contextWindow: 256_000, maxTokens: 32_768, @@ -96,6 +98,7 @@ describe("Command Code runtime", () => { cachePath: "/tmp/commandcode-models.json", loadModels: () => firstLoad.promise, createProviderConfig: (models) => ({ models }), + getTransport: () => "provider", now: () => now, logWarning: () => {}, }) @@ -113,6 +116,7 @@ describe("Command Code runtime", () => { assert.ok(statusCommand) await statusCommand("", context) const statusMessage = context.notifications.at(-1)?.message ?? "" + assert.match(statusMessage, /transport: provider/) assert.match(statusMessage, /source: live/) assert.match(statusMessage, /model count: 1/) assert.match(statusMessage, /last success:/) diff --git a/tests/test-smoke.mjs b/tests/test-smoke.mjs index 2665710..483c00a 100644 --- a/tests/test-smoke.mjs +++ b/tests/test-smoke.mjs @@ -7,7 +7,7 @@ * 3. Can complete a simple prompt (requires Command Code auth) * * Run with: node tests/test-smoke.mjs - * Requires: pi on PATH plus COMMANDCODE_API_KEY or live pi auth files. + * Requires: pi on PATH plus COMMAND_CODE_API_KEY (or legacy COMMANDCODE_API_KEY) or live pi auth files. */ import { spawn } from "node:child_process" @@ -48,6 +48,7 @@ const RPC_QUERY_TIMEOUT = 60_000 function hasCommandCodeAuth() { return ( + !!process.env.COMMAND_CODE_API_KEY || !!process.env.COMMANDCODE_API_KEY || existsSync(join(homedir(), ".commandcode", "auth.json")) || existsSync(join(homedir(), ".pi", "agent", "auth.json")) diff --git a/tests/test-stream.ts b/tests/test-stream.ts index 82ed325..040acb2 100644 --- a/tests/test-stream.ts +++ b/tests/test-stream.ts @@ -6,6 +6,7 @@ import assert from "node:assert/strict" import { after, before, beforeEach, describe, it } from "node:test" +import { COMMAND_CODE_CLI_VERSION } from "../src/commandcode-catalog.ts" import type { AssistantMessageEvent } from "../src/core.ts" import { MODEL_EFFORTS, thinkingLevelMapForEfforts } from "../src/models.ts" import { @@ -76,6 +77,23 @@ describe("streamCommandCode — auth", () => { ) }) + it("accepts the official CLI API key environment variable", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ + apiBase: server.baseUrl(), + env: { COMMAND_CODE_API_KEY: "official-env-key" }, + }) + + await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "$COMMAND_CODE_API_KEY" }), + ) + + assert.equal(server.lastRequestHeaders().authorization, "Bearer official-env-key") + }) + it("uses options.apiKey in the Authorization header", async () => { server.mockResponse({ type: "success", @@ -173,6 +191,101 @@ describe("streamCommandCode — successful streams", () => { ) }) + it("forwards a tool-result image as a following user image for vision-capable models", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode( + makeModel({ id: "deepseek/deepseek-v4-flash-vision-exp" }), + makeContext({ + messages: [ + { role: "user", content: "read the image" }, + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: [ + { type: "text", text: "image attached" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, + ], + }, + ], + }), + { apiKey: "mock-key" }, + ), + ) + + // No error: the tool-result image must not be rejected for this model. + assert.equal(events.at(-1)?.type, "done") + + const body = server.lastRequestBody() + // The tool-result text is forwarded on the tool message at index 2. + assert.equal( + objectAt(body, ["params", "messages", "2", "content", "0", "output", "value"]), + "image attached", + ) + // The tool-result image is forwarded as a following user image message at index 3. + assert.equal(objectAt(body, ["params", "messages", "3", "role"]), "user") + assert.equal( + objectAt(body, ["params", "messages", "3", "content", "0", "image"]), + "data:image/png;base64,aGVsbG8=", + ) + }) + + it("omits a historical tool-result image after switching to a text-only model", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode( + makeModel({ id: "deepseek/deepseek-v4-flash" }), + makeContext({ + messages: [ + { role: "user", content: "read the image" }, + { + role: "assistant", + content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }], + }, + { + role: "toolResult", + toolCallId: "c1", + toolName: "read", + content: [ + { type: "text", text: "image attached" }, + { type: "image", data: "aGVsbG8=", mimeType: "image/png" }, + ], + }, + { role: "user", content: "continue without the image" }, + ], + }), + { apiKey: "mock-key" }, + ), + ) + + assert.equal(events.at(-1)?.type, "done") + assert.equal(server.requestCount(), 1) + const body = server.lastRequestBody() + assert.equal( + objectAt(body, ["params", "messages", "2", "content", "0", "output", "value"]), + "image attached", + ) + assert.equal( + objectAt(body, ["params", "messages", "3", "content"]), + "continue without the image", + ) + }) + it("rejects images before network access for text-only models", async () => { const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) @@ -326,6 +439,104 @@ describe("streamCommandCode — successful streams", () => { assert.equal(toolCall?.type === "toolCall" ? toolCall.name : "", "read_file") }) + it("streams incremental tool-call arguments from generate events", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ + type: "tool-input-start", + id: "call_1", + toolName: "read_file", + }), + JSON.stringify({ type: "tool-input-delta", id: "call_1", delta: '{"path":"' }), + JSON.stringify({ type: "tool-input-delta", id: "call_1", delta: '/tmp/x"}' }), + JSON.stringify({ type: "tool-input-end", id: "call_1" }), + JSON.stringify({ + type: "tool-call", + toolCallId: "call_1", + toolName: "read_file", + input: { path: "/tmp/x" }, + }), + JSON.stringify({ type: "finish", finishReason: "tool-calls" }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + assert.deepEqual(eventTypes(events), [ + "start", + "toolcall_start", + "toolcall_delta", + "toolcall_delta", + "toolcall_end", + "done", + ]) + const deltas = events.flatMap((event) => (event.type === "toolcall_delta" ? [event.delta] : [])) + assert.deepEqual(deltas, ['{"path":"', '/tmp/x"}']) + + const done = events.at(-1) + if (done?.type !== "done") throw new Error("expected done") + assert.equal(done.reason, "toolUse") + const toolCall = done.message.content[0] + assert.equal(toolCall?.type, "toolCall") + if (toolCall?.type !== "toolCall") throw new Error("expected tool call") + assert.equal(toolCall.id, "call_1") + assert.equal(toolCall.name, "read_file") + assert.deepEqual(toolCall.arguments, { path: "/tmp/x" }) + }) + + it("keeps concurrent incremental tool calls separate", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ type: "tool-input-start", id: "call_1", toolName: "read_file" }), + JSON.stringify({ type: "tool-input-start", id: "call_2", toolName: "read_file" }), + JSON.stringify({ type: "tool-input-delta", id: "call_1", delta: '{"path":"/a"}' }), + JSON.stringify({ type: "tool-input-delta", id: "call_2", delta: '{"path":"/b"}' }), + JSON.stringify({ + type: "tool-call", + toolCallId: "call_2", + toolName: "read_file", + input: { path: "/b" }, + }), + JSON.stringify({ + type: "tool-call", + toolCallId: "call_1", + toolName: "read_file", + input: { path: "/a" }, + }), + JSON.stringify({ type: "finish", finishReason: "tool-calls" }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const starts = events.flatMap((event) => + event.type === "toolcall_start" ? [event.contentIndex] : [], + ) + const deltas = events.flatMap((event) => + event.type === "toolcall_delta" ? [[event.contentIndex, event.delta] as const] : [], + ) + const ends = events.flatMap((event) => + event.type === "toolcall_end" ? [[event.contentIndex, event.toolCall.id] as const] : [], + ) + assert.deepEqual(starts, [0, 1]) + assert.deepEqual(deltas, [ + [0, '{"path":"/a"}'], + [1, '{"path":"/b"}'], + ]) + assert.deepEqual(ends, [ + [1, "call_2"], + [0, "call_1"], + ]) + }) + it("flushes reasoning if finish arrives without reasoning-end", async () => { server.mockResponse({ type: "success", @@ -473,7 +684,7 @@ describe("streamCommandCode — request serialization", () => { assert.equal(objectAt(body, ["params", "stream"]), true) assert.equal(objectAt(body, ["params", "max_tokens"]), 500) assert.equal(objectAt(body, ["params", "reasoning_effort"]), undefined) - assert.equal(objectAt(body, ["params", "temperature"]), 0.3) + assert.equal(objectAt(body, ["params", "temperature"]), undefined) assert.equal(objectAt(body, ["params", "system"]), "You are a test assistant.") assert.equal(objectAt(body, ["memory"]), null) assert.equal(objectAt(body, ["taste"]), null) @@ -488,10 +699,11 @@ describe("streamCommandCode — request serialization", () => { const headers = server.lastRequestHeaders() assert.equal(headers.authorization, "Bearer mock-key") - assert.equal(headers["x-command-code-version"], "1.15.1") + assert.equal(headers["x-command-code-version"], COMMAND_CODE_CLI_VERSION) assert.equal(headers["x-project-slug"], "repo") assert.equal(headers["x-taste-learning"], "true") - assert.equal(headers["x-co-flag"], "false") + assert.equal(headers["user-agent"], "cli") + assert.equal(headers["x-co-flag"], undefined) assert.equal(headers["x-session-id"], undefined) }) @@ -545,6 +757,48 @@ describe("streamCommandCode — request serialization", () => { assert.doesNotMatch(String(objectAt(body, ["params", "system"])), /advisory/) }) + it("forwards explicit temperature and stable session metadata", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: "mock-key", + temperature: 0.7, + sessionId: "11111111-1111-4111-8111-111111111111", + }), + ) + + const body = server.lastRequestBody() + assert.equal(objectAt(body, ["params", "temperature"]), 0.7) + assert.equal(objectAt(body, ["threadId"]), "11111111-1111-4111-8111-111111111111") + assert.equal( + server.lastRequestHeaders()["x-session-id"], + "11111111-1111-4111-8111-111111111111", + ) + }) + + it("omits non-UUID session ids from the generate thread id", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "finish", finishReason: "stop" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + await collectEvents( + streamCommandCode(makeModel(), makeContext(), { + apiKey: "mock-key", + sessionId: "human-readable-session", + }), + ) + + assert.equal(objectAt(server.lastRequestBody(), ["threadId"]), undefined) + assert.equal(server.lastRequestHeaders()["x-session-id"], "human-readable-session") + }) + it("accepts the legacy OMP nested reasoning map", async () => { server.mockResponse({ type: "success", @@ -787,6 +1041,63 @@ describe("streamCommandCode — upstream errors and malformed streams", () => { assert.equal(error.error.errorMessage, "provider failed") }) + it("rejects a truncated stream without a finish event", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "text-delta", text: "truncated" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.match(error.error.errorMessage ?? "", /no finish event/i) + }) + + it("maps an upstream abort event to an aborted request", async () => { + server.mockResponse({ + type: "success", + events: [JSON.stringify({ type: "abort" })], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.equal(error.reason, "aborted") + }) + + it("rejects terminal upstream network failure reasons", async () => { + server.mockResponse({ + type: "success", + events: [ + JSON.stringify({ + type: "finish", + finishReason: "stop", + rawFinishReason: "upstream_error", + }), + ], + }) + const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() }) + + const events = await collectEvents( + streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }), + ) + + const error = events.at(-1) + assert.equal(error?.type, "error") + if (error?.type !== "error") throw new Error("expected error") + assert.match(error.error.errorMessage ?? "", /upstream connection failed/i) + }) + it("handles SSE lines, malformed lines, split chunks, and final line without newline", async () => { const textEvent = `data: ${JSON.stringify({ type: "text-delta", text: "split" })}\n` const finishEvent = JSON.stringify({ diff --git a/tests/test-transport.ts b/tests/test-transport.ts new file mode 100644 index 0000000..82e9771 --- /dev/null +++ b/tests/test-transport.ts @@ -0,0 +1,247 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { createCommandCodeTransportRouter } from "../src/transport.ts" +import type { + AssistantMessageEvent, + AssistantMessageEventStreamLike, + StreamOptions, +} from "../src/types.ts" +import { collectEvents, createTestEventStream, makeContext, makeModel } from "./helpers.ts" + +function completedStream(text: string): AssistantMessageEventStreamLike { + const stream = createTestEventStream() + const model = makeModel() + const message = { + role: "assistant" as const, + content: [{ type: "text" as const, text }], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 1, + output: 1, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 2, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop" as const, + timestamp: Date.now(), + } + const events: AssistantMessageEvent[] = [ + { type: "start", partial: message }, + { type: "text_start", contentIndex: 0, partial: message }, + { type: "text_delta", contentIndex: 0, delta: text, partial: message }, + { type: "text_end", contentIndex: 0, content: text, partial: message }, + { type: "done", reason: "stop", message }, + ] + for (const event of events) stream.push(event) + stream.end() + return stream +} + +function providerStream( + response: Response, + text: string, + options?: StreamOptions, +): AssistantMessageEventStreamLike { + const stream = createTestEventStream() + const run = async () => { + const received = await (options?.fetch ?? fetch)("https://provider.test", {}) + await options?.onResponse?.( + { status: received.status, headers: {} }, + makeModel({ api: "openai-completions" }), + ) + const source = completedStream(text) + for await (const event of source) stream.push(event) + stream.end() + } + run().catch(() => stream.end()) + return stream +} + +describe("Command Code transport router", () => { + it("keeps using the Provider API after a successful request", async () => { + let providerCalls = 0 + let generateCalls = 0 + const router = createCommandCodeTransportRouter({ + createStream: createTestEventStream, + streamProvider: (_model, _context, options) => { + providerCalls += 1 + return providerStream(new Response("ok", { status: 200 }), "provider", options) + }, + streamGenerate: () => { + generateCalls += 1 + return completedStream("generate") + }, + }) + + const options: StreamOptions = { + fetch: () => Promise.resolve(new Response("ok", { status: 200 })), + } + const first = await collectEvents(router.stream(makeModel(), makeContext(), options)) + const second = await collectEvents(router.stream(makeModel(), makeContext(), options)) + + assert.equal(first.at(-1)?.type, "done") + assert.equal(second.at(-1)?.type, "done") + assert.equal(router.getTransport(), "provider") + assert.equal(providerCalls, 2) + assert.equal(generateCalls, 0) + }) + + it("falls back only for 403 upgrade_required and remembers generate", async () => { + let providerCalls = 0 + let generateCalls = 0 + const responseBody = JSON.stringify({ + error: { code: "upgrade_required", type: "permission_error" }, + }) + const router = createCommandCodeTransportRouter({ + createStream: createTestEventStream, + streamProvider: (_model, _context, options) => { + providerCalls += 1 + return providerStream(new Response(responseBody, { status: 403 }), "blocked", options) + }, + streamGenerate: () => { + generateCalls += 1 + return completedStream("generate") + }, + }) + const options: StreamOptions = { + fetch: () => Promise.resolve(new Response(responseBody, { status: 403 })), + } + + const first = await collectEvents(router.stream(makeModel(), makeContext(), options)) + const second = await collectEvents(router.stream(makeModel(), makeContext(), options)) + + assert.equal(first.at(-1)?.type, "done") + assert.equal(second.at(-1)?.type, "done") + assert.equal(router.getTransport(), "generate") + assert.equal(providerCalls, 1) + assert.equal(generateCalls, 2) + }) + + it("re-detects the transport after the API key changes", async () => { + let providerCalls = 0 + let generateCalls = 0 + const upgradeBody = JSON.stringify({ error: { code: "upgrade_required" } }) + const router = createCommandCodeTransportRouter({ + createStream: createTestEventStream, + streamProvider: (_model, _context, options) => { + providerCalls += 1 + const response = + options?.apiKey === "go-key" + ? new Response(upgradeBody, { status: 403 }) + : new Response("ok", { status: 200 }) + return providerStream(response, "provider", options) + }, + streamGenerate: () => { + generateCalls += 1 + return completedStream("generate") + }, + }) + + await collectEvents( + router.stream(makeModel(), makeContext(), { + apiKey: "go-key", + fetch: () => Promise.resolve(new Response(upgradeBody, { status: 403 })), + }), + ) + await collectEvents( + router.stream(makeModel(), makeContext(), { + apiKey: "provider-key", + fetch: () => Promise.resolve(new Response("ok", { status: 200 })), + }), + ) + + assert.equal(router.getTransport(), "provider") + assert.equal(providerCalls, 2) + assert.equal(generateCalls, 1) + }) + + it("does not let a stale request overwrite the transport for a new API key", async () => { + let releaseGoRequest: (() => void) | undefined + const goRequestGate = new Promise((resolve) => { + releaseGoRequest = resolve + }) + let providerCalls = 0 + let generateCalls = 0 + const upgradeBody = JSON.stringify({ error: { code: "upgrade_required" } }) + const router = createCommandCodeTransportRouter({ + createStream: createTestEventStream, + streamProvider: (_model, _context, options) => { + providerCalls += 1 + const response = + options?.apiKey === "go-key" + ? new Response(upgradeBody, { status: 403 }) + : new Response("ok", { status: 200 }) + const stream = createTestEventStream() + const run = async () => { + if (options?.apiKey === "go-key") await goRequestGate + const received = await (options?.fetch ?? fetch)("https://provider.test", {}) + await options?.onResponse?.( + { status: received.status, headers: {} }, + makeModel({ api: "openai-completions" }), + ) + if (response.ok) { + for await (const event of completedStream("provider")) stream.push(event) + } + stream.end() + } + run().catch(() => stream.end()) + return stream + }, + streamGenerate: () => { + generateCalls += 1 + return completedStream("generate") + }, + }) + + const staleGoRequest = collectEvents( + router.stream(makeModel(), makeContext(), { + apiKey: "go-key", + fetch: () => Promise.resolve(new Response(upgradeBody, { status: 403 })), + }), + ) + await collectEvents( + router.stream(makeModel(), makeContext(), { + apiKey: "provider-key", + fetch: () => Promise.resolve(new Response("ok", { status: 200 })), + }), + ) + releaseGoRequest?.() + await staleGoRequest + await collectEvents( + router.stream(makeModel(), makeContext(), { + apiKey: "provider-key", + fetch: () => Promise.resolve(new Response("ok", { status: 200 })), + }), + ) + + assert.equal(router.getTransport(), "provider") + assert.equal(providerCalls, 3) + assert.equal(generateCalls, 1) + }) + + it("does not fall back for other 403 errors", async () => { + let generateCalls = 0 + const responseBody = JSON.stringify({ error: { code: "permission_denied" } }) + const router = createCommandCodeTransportRouter({ + createStream: createTestEventStream, + streamProvider: (_model, _context, options) => + providerStream(new Response(responseBody, { status: 403 }), "blocked", options), + streamGenerate: () => { + generateCalls += 1 + return completedStream("generate") + }, + }) + const options: StreamOptions = { + fetch: () => Promise.resolve(new Response(responseBody, { status: 403 })), + } + + await collectEvents(router.stream(makeModel(), makeContext(), options)) + + assert.equal(router.getTransport(), "provider") + assert.equal(generateCalls, 0) + }) +}) diff --git a/tsconfig.json b/tsconfig.json index c691796..2cf50c2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,5 +9,5 @@ "strict": true, "types": ["node"] }, - "include": ["src/**/*.ts", "tests/**/*.ts"] + "include": [".github/scripts/**/*.ts", "src/**/*.ts", "tests/**/*.ts"] }