From 349e50f829cba0280f5b552c7b0b9e4ae4a7bdfe Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 25 Aug 2026 15:54:09 +0200 Subject: [PATCH] fix(models): align Command Code catalog metadata --- .../check-commandcode-model-metadata.ts | 149 ++++++++++++++++-- .github/workflows/model-metadata.yml | 3 +- src/commandcode-catalog.ts | 57 +++++++ src/models.ts | 39 +++-- src/pricing.ts | 44 +++++- tests/fixtures/commandcode-model-ids.json | 12 +- tests/fixtures/commandcode-pricing.json | 31 ++-- tests/test-model-metadata-check.ts | 47 ++++-- tests/test-models.ts | 47 +++++- tests/test-pricing.ts | 33 +++- 10 files changed, 399 insertions(+), 63 deletions(-) diff --git a/.github/scripts/check-commandcode-model-metadata.ts b/.github/scripts/check-commandcode-model-metadata.ts index 292a4d3..b83c386 100644 --- a/.github/scripts/check-commandcode-model-metadata.ts +++ b/.github/scripts/check-commandcode-model-metadata.ts @@ -9,6 +9,8 @@ import { COMMAND_CODE_CLI_VERSION, MODEL_EFFORTS, MODEL_INPUT_MODALITIES, + MODEL_MAX_OUTPUT_TOKENS, + MODEL_REASONING, } from "../../src/commandcode-catalog.ts" const execFileAsync = promisify(execFile) @@ -21,7 +23,9 @@ const README_PATH = new URL("../../README.md", import.meta.url) export interface CommandCodeModelMetadata { imageModelIds: readonly string[] + reasoningModelIds: readonly string[] reasoningEfforts: Readonly> + maxOutputTokens: Readonly> } export interface ModelMetadataDiff { @@ -30,7 +34,12 @@ export interface ModelMetadataDiff { removedImageModelIds: readonly string[] addedReasoningModelIds: readonly string[] removedReasoningModelIds: readonly string[] - changedReasoningModelIds: readonly string[] + addedEffortModelIds: readonly string[] + removedEffortModelIds: readonly string[] + changedEffortModelIds: readonly string[] + addedMaxOutputModelIds: readonly string[] + removedMaxOutputModelIds: readonly string[] + changedMaxOutputModelIds: readonly string[] } interface PackedPackage { @@ -123,27 +132,93 @@ export function parseKnownTextOnlyModelIds(bundle: string): readonly string[] { return sorted(new Set(parsed)) } +function modelObject(bundle: string, modelId: string): string { + const start = bundle.indexOf(`{id:${JSON.stringify(modelId)},inputModalities:`) + if (start < 0) throw new Error(`Could not find model metadata for ${modelId}`) + + let depth = 0 + let quote = "" + let escaped = false + for (let index = start; index < bundle.length; index += 1) { + const character = bundle[index] ?? "" + if (quote) { + if (escaped) escaped = false + else if (character === "\\") escaped = true + else if (character === quote) quote = "" + continue + } + if (character === '"' || character === "'" || character === "`") { + quote = character + continue + } + if (character === "{") depth += 1 + else if (character === "}" && --depth === 0) return bundle.slice(start, index + 1) + } + + throw new Error(`Unterminated model metadata for ${modelId}`) +} + +export function parseBundleModelCapabilities( + bundle: string, + modelIds: readonly string[], +): { + reasoningModelIds: readonly string[] + maxOutputTokens: Readonly> +} { + const reasoningModelIds: string[] = [] + const maxOutputTokens: Record = {} + + for (const modelId of modelIds) { + const entry = modelObject(bundle, modelId) + if (entry.includes("reasoning:!0") || entry.includes("reasoningEfforts:[")) { + reasoningModelIds.push(modelId) + } + const maxOutput = /maxOutputTokens:([^,}]+)/.exec(entry)?.[1] + if (maxOutput) { + const value = Number(maxOutput) + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`Unexpected max output tokens for ${modelId}: ${maxOutput}`) + } + maxOutputTokens[modelId] = value + } + } + + return { + reasoningModelIds: sorted(reasoningModelIds), + maxOutputTokens: Object.fromEntries( + Object.entries(maxOutputTokens).sort(([left], [right]) => left.localeCompare(right)), + ), + } +} + export function commandCodeModelMetadataFromContents( modelsReference: string, cliBundle: string, ): CommandCodeModelMetadata { const reference = parseModelsReference(modelsReference) const textOnlyModelIds = new Set(parseKnownTextOnlyModelIds(cliBundle)) + const capabilities = parseBundleModelCapabilities(cliBundle, reference.modelIds) return { imageModelIds: reference.modelIds.filter((modelId) => !textOnlyModelIds.has(modelId)), + reasoningModelIds: capabilities.reasoningModelIds, reasoningEfforts: reference.reasoningEfforts, + maxOutputTokens: capabilities.maxOutputTokens, } } export function currentModelMetadata(): CommandCodeModelMetadata { return { imageModelIds: sorted(Object.keys(MODEL_INPUT_MODALITIES)), + reasoningModelIds: sorted(Object.keys(MODEL_REASONING)), reasoningEfforts: Object.fromEntries( Object.entries(MODEL_EFFORTS) .sort(([left], [right]) => left.localeCompare(right)) .map(([modelId, efforts]) => [modelId, [...efforts]]), ), + maxOutputTokens: Object.fromEntries( + Object.entries(MODEL_MAX_OUTPUT_TOKENS).sort(([left], [right]) => left.localeCompare(right)), + ), } } @@ -155,10 +230,16 @@ export function diffModelMetadata( ): ModelMetadataDiff { const currentImages = new Set(current.imageModelIds) const upstreamImages = new Set(upstream.imageModelIds) - const currentReasoningIds = Object.keys(current.reasoningEfforts) - const upstreamReasoningIds = Object.keys(upstream.reasoningEfforts) - const currentReasoningSet = new Set(currentReasoningIds) - const upstreamReasoningSet = new Set(upstreamReasoningIds) + const currentReasoning = new Set(current.reasoningModelIds) + const upstreamReasoning = new Set(upstream.reasoningModelIds) + const currentEffortIds = Object.keys(current.reasoningEfforts) + const upstreamEffortIds = Object.keys(upstream.reasoningEfforts) + const currentEffortSet = new Set(currentEffortIds) + const upstreamEffortSet = new Set(upstreamEffortIds) + const currentMaxOutputIds = Object.keys(current.maxOutputTokens) + const upstreamMaxOutputIds = Object.keys(upstream.maxOutputTokens) + const currentMaxOutputSet = new Set(currentMaxOutputIds) + const upstreamMaxOutputSet = new Set(upstreamMaxOutputIds) return { versionChanged: currentVersion !== upstreamVersion, @@ -169,19 +250,38 @@ export function diffModelMetadata( current.imageModelIds.filter((modelId) => !upstreamImages.has(modelId)), ), addedReasoningModelIds: sorted( - upstreamReasoningIds.filter((modelId) => !currentReasoningSet.has(modelId)), + upstream.reasoningModelIds.filter((modelId) => !currentReasoning.has(modelId)), ), removedReasoningModelIds: sorted( - currentReasoningIds.filter((modelId) => !upstreamReasoningSet.has(modelId)), + current.reasoningModelIds.filter((modelId) => !upstreamReasoning.has(modelId)), ), - changedReasoningModelIds: sorted( - upstreamReasoningIds.filter( + addedEffortModelIds: sorted( + upstreamEffortIds.filter((modelId) => !currentEffortSet.has(modelId)), + ), + removedEffortModelIds: sorted( + currentEffortIds.filter((modelId) => !upstreamEffortSet.has(modelId)), + ), + changedEffortModelIds: sorted( + upstreamEffortIds.filter( (modelId) => - currentReasoningSet.has(modelId) && + currentEffortSet.has(modelId) && JSON.stringify(current.reasoningEfforts[modelId]) !== JSON.stringify(upstream.reasoningEfforts[modelId]), ), ), + addedMaxOutputModelIds: sorted( + upstreamMaxOutputIds.filter((modelId) => !currentMaxOutputSet.has(modelId)), + ), + removedMaxOutputModelIds: sorted( + currentMaxOutputIds.filter((modelId) => !upstreamMaxOutputSet.has(modelId)), + ), + changedMaxOutputModelIds: sorted( + upstreamMaxOutputIds.filter( + (modelId) => + currentMaxOutputSet.has(modelId) && + current.maxOutputTokens[modelId] !== upstream.maxOutputTokens[modelId], + ), + ), } } @@ -229,14 +329,24 @@ export function renderCommandCodeCatalog( const imageEntries = sorted(metadata.imageModelIds) .map((modelId) => ` ${quoted(modelId)}: ["text", "image"],`) .join("\n") - const reasoningEntries = recordEntries(metadata.reasoningEfforts) + const reasoningEntries = sorted(metadata.reasoningModelIds) + .map((modelId) => ` ${quoted(modelId)}: true,`) + .join("\n") + const effortEntries = recordEntries(metadata.reasoningEfforts) .map( ([modelId, efforts]) => ` ${quoted(modelId)}: [${efforts.map((effort) => quoted(effort)).join(", ")}],`, ) .join("\n") + const maxOutputEntries = Object.entries(metadata.maxOutputTokens) + .sort(([left], [right]) => left.localeCompare(right)) + .map( + ([modelId, value]) => + ` ${quoted(modelId)}: ${value.toLocaleString("en-US").replaceAll(",", "_")},`, + ) + .join("\n") - return `export const COMMAND_CODE_CLI_VERSION = ${quoted(packageVersion)}\n\nexport type CommandCodeInputType = "text" | "image"\nexport type CommandCodeReasoningEffort = "minimal" | "low" | "medium" | "high" | "xhigh" | "max"\n\n/**\n * Generated from command-code@${packageVersion} by \`npm run sync:commandcode-catalog\`.\n * Do not edit manually.\n */\nexport const MODEL_INPUT_MODALITIES: Readonly> = {\n${imageEntries}\n}\n\nexport const MODEL_EFFORTS: Readonly> = {\n${reasoningEntries}\n}\n` + return `export const COMMAND_CODE_CLI_VERSION = ${quoted(packageVersion)}\n\nexport type CommandCodeInputType = "text" | "image"\nexport type CommandCodeReasoningEffort = "minimal" | "low" | "medium" | "high" | "xhigh" | "max"\n\n/**\n * Generated from command-code@${packageVersion} by \`npm run sync:commandcode-catalog\`.\n * Do not edit manually.\n */\nexport const MODEL_INPUT_MODALITIES: Readonly> = {\n${imageEntries}\n}\n\nexport const MODEL_REASONING: Readonly> = {\n${reasoningEntries}\n}\n\nexport const MODEL_EFFORTS: Readonly> = {\n${effortEntries}\n}\n\nexport const MODEL_MAX_OUTPUT_TOKENS: Readonly> = {\n${maxOutputEntries}\n}\n` } function updateDocumentedCatalogVersion( @@ -279,16 +389,23 @@ function metadataReport( `- Repository snapshot: \`command-code@${COMMAND_CODE_CLI_VERSION}\``, `- Inspected package: \`command-code@${packageVersion}\``, `- Image-capable models: ${current.imageModelIds.length} repository / ${upstream.imageModelIds.length} upstream`, - `- Reasoning models: ${Object.keys(current.reasoningEfforts).length} repository / ${Object.keys(upstream.reasoningEfforts).length} upstream`, + `- Reasoning models: ${current.reasoningModelIds.length} repository / ${upstream.reasoningModelIds.length} upstream`, + `- Models with selectable efforts: ${Object.keys(current.reasoningEfforts).length} repository / ${Object.keys(upstream.reasoningEfforts).length} upstream`, + `- Model-specific output limits: ${Object.keys(current.maxOutputTokens).length} repository / ${Object.keys(upstream.maxOutputTokens).length} upstream`, "", "| Change | Models |", "| --- | --- |", `| CLI version | ${diff.versionChanged ? `\`${COMMAND_CODE_CLI_VERSION}\` → \`${packageVersion}\`` : "Current"} |`, `| New image support | ${formatList(diff.addedImageModelIds)} |`, `| Removed image support | ${formatList(diff.removedImageModelIds)} |`, - `| New reasoning metadata | ${formatList(diff.addedReasoningModelIds)} |`, - `| Removed reasoning metadata | ${formatList(diff.removedReasoningModelIds)} |`, - `| Changed reasoning efforts | ${formatReasoningChanges(diff.changedReasoningModelIds, current, upstream)} |`, + `| New reasoning models | ${formatList(diff.addedReasoningModelIds)} |`, + `| Removed reasoning models | ${formatList(diff.removedReasoningModelIds)} |`, + `| New effort metadata | ${formatList(diff.addedEffortModelIds)} |`, + `| Removed effort metadata | ${formatList(diff.removedEffortModelIds)} |`, + `| Changed reasoning efforts | ${formatReasoningChanges(diff.changedEffortModelIds, current, upstream)} |`, + `| New output limits | ${formatList(diff.addedMaxOutputModelIds)} |`, + `| Removed output limits | ${formatList(diff.removedMaxOutputModelIds)} |`, + `| Changed output limits | ${formatList(diff.changedMaxOutputModelIds)} |`, "", ].join("\n") } diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml index de5a26f..88c9ca5 100644 --- a/.github/workflows/model-metadata.yml +++ b/.github/workflows/model-metadata.yml @@ -79,7 +79,8 @@ jobs: This updates only machine-readable compatibility metadata: - CLI version used in the `x-command-code-version` header - image-input capabilities - - supported reasoning efforts + - reasoning capability and selectable effort levels + - model-specific maximum output limits - documented catalog snapshot version Pricing remains review-only because CLI documentation does not represent every pricing tier and temporary promotion used by the provider. diff --git a/src/commandcode-catalog.ts b/src/commandcode-catalog.ts index 4249514..6840f5f 100644 --- a/src/commandcode-catalog.ts +++ b/src/commandcode-catalog.ts @@ -51,6 +51,57 @@ export const MODEL_INPUT_MODALITIES: Readonly> = { + "claude-fable-5": true, + "claude-opus-4-7": true, + "claude-opus-4-8": true, + "claude-opus-5": true, + "claude-sonnet-4-6": true, + "claude-sonnet-5": true, + "deepseek/deepseek-v4-flash": true, + "deepseek/deepseek-v4-flash-vision-exp": true, + "deepseek/deepseek-v4-pro": true, + "google/gemini-3.1-flash-lite": true, + "google/gemini-3.5-flash": true, + "google/gemini-3.5-flash-lite": true, + "google/gemini-3.6-flash": true, + "google/gemini-3.7-flash": true, + "gpt-5.3-codex": true, + "gpt-5.4": true, + "gpt-5.4-mini": true, + "gpt-5.5": true, + "gpt-5.6-luna": true, + "gpt-5.6-sol": true, + "gpt-5.6-terra": true, + "meta/muse-spark-1.1": true, + "meta/muse-spark-1.2": true, + "meta/muse-spark-1.2-contributor": true, + "MiniMaxAI/MiniMax-M3": true, + "moonshotai/Kimi-K2.7-Code": true, + "moonshotai/Kimi-K2.7-Code-Highspeed": true, + "moonshotai/Kimi-K3": true, + "nvidia/nemotron-3-ultra-550b-a55b": true, + "poolside/laguna-s-2.1-free": true, + "Qwen/Qwen3.6-Max-Preview": true, + "Qwen/Qwen3.6-Plus": true, + "Qwen/Qwen3.7-Flash": true, + "Qwen/Qwen3.7-Max": true, + "Qwen/Qwen3.7-Plus": true, + "Qwen/Qwen3.8-27B": true, + "Qwen/Qwen3.8-Max": true, + "sakana/fugu-ultra": true, + "stealth/ox-alpha": true, + "stepfun/Step-3.5-Flash": true, + "stepfun/Step-3.7-Flash": true, + "tencent/hy3-paid": true, + "thinkingmachines/inkling": true, + "thinkingmachines/inkling-small": true, + "xai/grok-4.5": true, + "xai/grok-4.6": true, + "zai-org/GLM-5.2": true, + "zai-org/GLM-5.3": true, +} + export const MODEL_EFFORTS: Readonly> = { "claude-fable-5": ["low", "medium", "high", "xhigh", "max"], "claude-opus-4-7": ["low", "medium", "high", "xhigh", "max"], @@ -82,3 +133,9 @@ export const MODEL_EFFORTS: Readonly> = { + "poolside/laguna-s-2.1-free": 32_768, + "Qwen/Qwen3.8-27B": 32_768, + "stealth/ox-alpha": 131_072, +} diff --git a/src/models.ts b/src/models.ts index 9b1ec93..be7c674 100644 --- a/src/models.ts +++ b/src/models.ts @@ -4,11 +4,13 @@ import { dirname } from "node:path" import { MODEL_EFFORTS, MODEL_INPUT_MODALITIES, + MODEL_MAX_OUTPUT_TOKENS, + MODEL_REASONING, type CommandCodeInputType, type CommandCodeReasoningEffort, } from "./commandcode-catalog.ts" -export { MODEL_EFFORTS, MODEL_INPUT_MODALITIES } +export { MODEL_EFFORTS, MODEL_INPUT_MODALITIES, MODEL_MAX_OUTPUT_TOKENS, MODEL_REASONING } export type { CommandCodeInputType } export const DEFAULT_PROVIDER_API_BASE = "https://api.commandcode.ai/provider/v1" @@ -55,7 +57,7 @@ export function thinkingLevelMapForEfforts( export interface ThinkingMetadata { thinkingLevelMap: Partial> - thinking: { + thinking?: { mode: "effort" effortMap: Partial> efforts: readonly CommandCodeReasoningEffort[] @@ -64,19 +66,26 @@ export interface ThinkingMetadata { export function thinkingMetadataForModel(modelId: string): ThinkingMetadata | undefined { const efforts = MODEL_EFFORTS[modelId] - if (!efforts) return undefined - return { - thinkingLevelMap: thinkingLevelMapForEfforts(efforts), - thinking: { - mode: "effort", - effortMap: Object.fromEntries(efforts.map((effort) => [effort, effort])), - efforts, - }, + if (efforts) { + return { + thinkingLevelMap: thinkingLevelMapForEfforts(efforts), + thinking: { + mode: "effort", + effortMap: Object.fromEntries(efforts.map((effort) => [effort, effort])), + efforts, + }, + } } + if (!isReasoningModel(modelId)) return undefined + return { thinkingLevelMap: thinkingLevelMapForEfforts([]) } } function isReasoningModel(modelId: string): boolean { - return MODEL_EFFORTS[modelId] !== undefined + return MODEL_REASONING[modelId] === true +} + +function maxOutputTokensForModel(modelId: string, contextLength: number): number { + return Math.min(contextLength, MODEL_MAX_OUTPUT_TOKENS[modelId] ?? DEFAULT_MAX_OUTPUT_TOKENS) } interface ApiModel { @@ -162,13 +171,15 @@ function parseCachedModel(value: unknown): CommandCodeModel { const id = stringField(value, "id") booleanField(value, "reasoning") + positiveNumberField(value, "maxTokens") + const contextWindow = positiveNumberField(value, "contextWindow") return { id, name: stringField(value, "name"), api: apiForModelId(id), reasoning: isReasoningModel(id), - contextWindow: positiveNumberField(value, "contextWindow"), - maxTokens: positiveNumberField(value, "maxTokens"), + contextWindow, + maxTokens: maxOutputTokensForModel(id, contextWindow), } } @@ -273,7 +284,7 @@ export function commandCodeModelsFromApiResponse(value: unknown): readonly Comma api: apiForModelId(model.id), reasoning: isReasoningModel(model.id), contextWindow: model.contextLength, - maxTokens: Math.min(model.contextLength, DEFAULT_MAX_OUTPUT_TOKENS), + maxTokens: maxOutputTokensForModel(model.id, model.contextLength), })) } diff --git a/src/pricing.ts b/src/pricing.ts index 5f3a951..fddd2a3 100644 --- a/src/pricing.ts +++ b/src/pricing.ts @@ -20,7 +20,7 @@ export interface TemporaryPricing { } export const PRICING_SOURCE_URL = "https://commandcode.ai/docs/resources/pricing-limits" -export const PRICING_LAST_VERIFIED = "2026-08-22" +export const PRICING_LAST_VERIFIED = "2026-08-25" export const ZERO_MODEL_COST: CommandCodeModelCost = { input: 0, @@ -40,7 +40,7 @@ export const ZERO_MODEL_COST: CommandCodeModelCost = { export const MODEL_COSTS: Readonly> = { // Free models "poolside/laguna-s-2.1-free": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - "inclusionai/ling-3.0-flash-free": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + "stealth/ox-alpha": { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, // Open and open-weight models "tencent/hy3-paid": { input: 0.14, output: 0.58, cacheRead: 0.035, cacheWrite: 0 }, @@ -76,7 +76,14 @@ export const MODEL_COSTS: Readonly> = { cacheRead: 0.007, cacheWrite: 0, }, + "deepseek/deepseek-v4-flash-vision-exp": { + input: 0.22, + output: 0.66, + cacheRead: 0.007, + cacheWrite: 0, + }, "Qwen/Qwen3.8-Max": { input: 2, output: 6, cacheRead: 0.25, cacheWrite: 2.5 }, + "Qwen/Qwen3.8-27B": { input: 0.4, output: 3, cacheRead: 0.04, cacheWrite: 0 }, "Qwen/Qwen3.7-Max": { input: 2.5, output: 7.5, cacheRead: 0.5, cacheWrite: 3.13 }, "Qwen/Qwen3.7-Plus": { input: 0.4, @@ -142,6 +149,13 @@ export const MODEL_COSTS: Readonly> = { cacheWrite: 0, }, "meta/muse-spark-1.1": { input: 1.25, output: 4.25, cacheRead: 0.15, cacheWrite: 0 }, + "meta/muse-spark-1.2": { input: 1.25, output: 4.25, cacheRead: 0.15, cacheWrite: 0 }, + "meta/muse-spark-1.2-contributor": { + input: 0.1, + output: 0.2, + cacheRead: 0.002, + cacheWrite: 0, + }, // Anthropic // Introductory pricing through 2026-08-31. @@ -168,6 +182,12 @@ export const MODEL_COSTS: Readonly> = { "gpt-5.4-mini": { input: 0.75, output: 4.5, cacheRead: 0.075, cacheWrite: 0 }, // Google and xAI + "google/gemini-3.7-flash": { + input: 0.75, + output: 3.75, + cacheRead: 0.075, + cacheWrite: 0.04167, + }, "google/gemini-3.6-flash": { input: 1.5, output: 7.5, cacheRead: 0.15, cacheWrite: 0 }, "google/gemini-3.5-flash": { input: 1.5, output: 9, cacheRead: 0.15, cacheWrite: 0 }, "google/gemini-3.5-flash-lite": { @@ -183,6 +203,21 @@ export const MODEL_COSTS: Readonly> = { cacheWrite: 0, }, "xai/grok-4.5": { input: 2, output: 6, cacheRead: 0.5, cacheWrite: 0 }, + "xai/grok-4.6": { + input: 2, + output: 6, + cacheRead: 0.5, + cacheWrite: 0, + tiers: [ + { + inputTokensAbove: 200_000, + input: 4, + output: 12, + cacheRead: 1, + cacheWrite: 0, + }, + ], + }, } export const TEMPORARY_PRICING: readonly TemporaryPricing[] = [ @@ -191,4 +226,9 @@ export const TEMPORARY_PRICING: readonly TemporaryPricing[] = [ expiresOn: "2026-08-31", description: "introductory pricing", }, + { + models: ["google/gemini-3.7-flash"], + expiresOn: "2026-12-31", + description: "50% promotional pricing", + }, ] diff --git a/tests/fixtures/commandcode-model-ids.json b/tests/fixtures/commandcode-model-ids.json index 1b9898a..2a69168 100644 --- a/tests/fixtures/commandcode-model-ids.json +++ b/tests/fixtures/commandcode-model-ids.json @@ -1,5 +1,5 @@ { - "fetchedAt": "2026-08-22T21:19:37.782Z", + "fetchedAt": "2026-08-25T13:32:11.631Z", "source": "https://api.commandcode.ai/provider/v1/models", "modelIds": [ "claude-sonnet-5", @@ -18,6 +18,7 @@ "gpt-5.4-mini", "deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-flash", + "deepseek/deepseek-v4-flash-vision-exp", "moonshotai/Kimi-K3", "moonshotai/Kimi-K2.7-Code", "moonshotai/Kimi-K2.7-Code-Highspeed", @@ -34,6 +35,7 @@ "xiaomi/mimo-v2.5-pro", "xiaomi/mimo-v2.5", "Qwen/Qwen3.8-Max", + "Qwen/Qwen3.8-27B", "Qwen/Qwen3.7-Max", "Qwen/Qwen3.7-Plus", "Qwen/Qwen3.7-Flash", @@ -42,6 +44,7 @@ "stepfun/Step-3.7-Flash", "stepfun/Step-3.5-Flash", "tencent/hy3-paid", + "google/gemini-3.7-flash", "google/gemini-3.6-flash", "google/gemini-3.5-flash", "google/gemini-3.5-flash-lite", @@ -50,9 +53,12 @@ "nvidia/nemotron-3-ultra-550b-a55b", "thinkingmachines/inkling", "thinkingmachines/inkling-small", + "stealth/ox-alpha", "poolside/laguna-s-2.1-free", - "inclusionai/ling-3.0-flash-free", "meta/muse-spark-1.1", - "xai/grok-4.5" + "meta/muse-spark-1.2", + "meta/muse-spark-1.2-contributor", + "xai/grok-4.5", + "xai/grok-4.6" ] } diff --git a/tests/fixtures/commandcode-pricing.json b/tests/fixtures/commandcode-pricing.json index 5b2070c..c65ae0c 100644 --- a/tests/fixtures/commandcode-pricing.json +++ b/tests/fixtures/commandcode-pricing.json @@ -1,5 +1,5 @@ { - "verifiedAt": "2026-08-22", + "verifiedAt": "2026-08-25", "source": "https://commandcode.ai/docs/resources/pricing-limits", "tierPolicy": "Use request-wide input tiers; the highest threshold exceeded by input plus cache tokens applies to the full request.", "tiers": { @@ -7,12 +7,13 @@ "Qwen/Qwen3.7-Flash": [ [32000, 0.1, 0.4, 0.02, 0.125], [256000, 0.2, 0.8, 0.04, 0.25] - ] + ], + "xai/grok-4.6": [[200000, 4, 12, 1, 0]] }, "costs": { - "poolside/laguna-s-2.1-free": [0, 0, 0, 0], - "inclusionai/ling-3.0-flash-free": [0, 0, 0, 0], - "tencent/hy3-paid": [0.14, 0.58, 0.035, 0], + "deepseek/deepseek-v4-pro": [0.66, 1.98, 0.022, 0], + "deepseek/deepseek-v4-flash": [0.22, 0.66, 0.007, 0], + "deepseek/deepseek-v4-flash-vision-exp": [0.22, 0.66, 0.007, 0], "moonshotai/Kimi-K3": [3, 15, 0.3, 0], "moonshotai/Kimi-K2.7-Code": [0.95, 4, 0.19, 0], "moonshotai/Kimi-K2.7-Code-Highspeed": [1.9, 8, 0.38, 0], @@ -26,9 +27,10 @@ "MiniMaxAI/MiniMax-M3": [0.3, 1.2, 0.06, 0], "MiniMaxAI/MiniMax-M2.7": [0.3, 1.2, 0.06, 0], "MiniMaxAI/MiniMax-M2.5": [0.3, 1.2, 0.03, 0], - "deepseek/deepseek-v4-pro": [0.66, 1.98, 0.022, 0], - "deepseek/deepseek-v4-flash": [0.22, 0.66, 0.007, 0], + "xiaomi/mimo-v2.5-pro": [0.435, 0.87, 0.0036, 0], + "xiaomi/mimo-v2.5": [0.14, 0.28, 0.0028, 0], "Qwen/Qwen3.8-Max": [2, 6, 0.25, 2.5], + "Qwen/Qwen3.8-27B": [0.4, 3, 0.04, 0], "Qwen/Qwen3.7-Max": [2.5, 7.5, 0.5, 3.13], "Qwen/Qwen3.7-Plus": [0.4, 1.6, 0.08, 0.5], "Qwen/Qwen3.7-Flash": [0.03, 0.13, 0.006, 0.038], @@ -36,13 +38,12 @@ "Qwen/Qwen3.6-Plus": [0.5, 3, 0.1, 0], "stepfun/Step-3.7-Flash": [0.2, 1.15, 0.04, 0], "stepfun/Step-3.5-Flash": [0.1, 0.3, 0.02, 0], - "xiaomi/mimo-v2.5-pro": [0.435, 0.87, 0.0036, 0], - "xiaomi/mimo-v2.5": [0.14, 0.28, 0.0028, 0], + "tencent/hy3-paid": [0.14, 0.58, 0.035, 0], "nvidia/nemotron-3-ultra-550b-a55b": [0.6, 2.4, 0.12, 0], - "sakana/fugu-ultra": [5, 30, 0.5, 0], "thinkingmachines/inkling": [1, 4.05, 0.17, 0], "thinkingmachines/inkling-small": [0.5, 1.2, 0.1, 0], - "meta/muse-spark-1.1": [1.25, 4.25, 0.15, 0], + "poolside/laguna-s-2.1-free": [0, 0, 0, 0], + "stealth/ox-alpha": [0, 0, 0, 0], "claude-sonnet-5": [2, 10, 0.2, 2.5], "claude-sonnet-4-6": [3, 15, 0.3, 3.75], "claude-fable-5": [10, 50, 1, 12.5], @@ -57,10 +58,16 @@ "gpt-5.4": [2.5, 15, 0.25, 0], "gpt-5.3-codex": [2, 8, 0.5, 0], "gpt-5.4-mini": [0.75, 4.5, 0.075, 0], + "google/gemini-3.7-flash": [0.75, 3.75, 0.075, 0.04167], "google/gemini-3.6-flash": [1.5, 7.5, 0.15, 0], "google/gemini-3.5-flash": [1.5, 9, 0.15, 0], "google/gemini-3.5-flash-lite": [0.3, 2.5, 0.03, 0], "google/gemini-3.1-flash-lite": [0.25, 1.5, 0.03, 0], - "xai/grok-4.5": [2, 6, 0.5, 0] + "sakana/fugu-ultra": [5, 30, 0.5, 0], + "meta/muse-spark-1.1": [1.25, 4.25, 0.15, 0], + "meta/muse-spark-1.2": [1.25, 4.25, 0.15, 0], + "meta/muse-spark-1.2-contributor": [0.1, 0.2, 0.002, 0], + "xai/grok-4.5": [2, 6, 0.5, 0], + "xai/grok-4.6": [2, 6, 0.5, 0] } } diff --git a/tests/test-model-metadata-check.ts b/tests/test-model-metadata-check.ts index ac1b61f..97fe30e 100644 --- a/tests/test-model-metadata-check.ts +++ b/tests/test-model-metadata-check.ts @@ -5,6 +5,7 @@ import { commandCodeModelMetadataFromContents, diffModelMetadata, hasModelMetadataDiff, + parseBundleModelCapabilities, parseKnownTextOnlyModelIds, parseModelsReference, parsePackageVersion, @@ -21,7 +22,7 @@ const MODELS_REFERENCE = ` ` const CLI_BUNDLE = - 'const catalog=new Set(["text-model"]),__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' + 'const V={id:"vision-model",inputModalities:["text","image"],reasoning:!0,reasoningEfforts:["low","high"],maxOutputTokens:32768},T={id:"text-model",inputModalities:["text"]},catalog=new Set(["text-model"]),__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' describe("Command Code model metadata checker", () => { it("parses model ids and reasoning efforts from the generated reference", () => { @@ -42,29 +43,39 @@ describe("Command Code model metadata checker", () => { assert.throws(() => parsePackageVersion("latest"), /one semantic version/) }) - it("derives image support by excluding known text-only models", () => { + it("derives image, reasoning, effort, and output-limit metadata", () => { + assert.deepEqual(parseBundleModelCapabilities(CLI_BUNDLE, ["text-model", "vision-model"]), { + reasoningModelIds: ["vision-model"], + maxOutputTokens: { "vision-model": 32_768 }, + }) assert.deepEqual(commandCodeModelMetadataFromContents(MODELS_REFERENCE, CLI_BUNDLE), { imageModelIds: ["vision-model"], + reasoningModelIds: ["vision-model"], reasoningEfforts: { "vision-model": ["low", "high"] }, + maxOutputTokens: { "vision-model": 32_768 }, }) }) it("reports additions, removals, and changed reasoning efforts", () => { const current: CommandCodeModelMetadata = { imageModelIds: ["removed-image", "stable-image"], + reasoningModelIds: ["removed-reasoning", "stable-reasoning"], reasoningEfforts: { - "changed-reasoning": ["low"], - "removed-reasoning": ["high"], - "stable-reasoning": ["low", "high"], + "changed-effort": ["low"], + "removed-effort": ["high"], + "stable-effort": ["low", "high"], }, + maxOutputTokens: { "changed-output": 1, "removed-output": 2, "stable-output": 3 }, } const upstream: CommandCodeModelMetadata = { imageModelIds: ["added-image", "stable-image"], + reasoningModelIds: ["added-reasoning", "stable-reasoning"], reasoningEfforts: { - "added-reasoning": ["max"], - "changed-reasoning": ["low", "high"], - "stable-reasoning": ["low", "high"], + "added-effort": ["max"], + "changed-effort": ["low", "high"], + "stable-effort": ["low", "high"], }, + maxOutputTokens: { "added-output": 4, "changed-output": 5, "stable-output": 3 }, } const diff = diffModelMetadata(current, upstream) @@ -75,7 +86,12 @@ describe("Command Code model metadata checker", () => { removedImageModelIds: ["removed-image"], addedReasoningModelIds: ["added-reasoning"], removedReasoningModelIds: ["removed-reasoning"], - changedReasoningModelIds: ["changed-reasoning"], + addedEffortModelIds: ["added-effort"], + removedEffortModelIds: ["removed-effort"], + changedEffortModelIds: ["changed-effort"], + addedMaxOutputModelIds: ["added-output"], + removedMaxOutputModelIds: ["removed-output"], + changedMaxOutputModelIds: ["changed-output"], }) assert.equal(hasModelMetadataDiff(diff), true) }) @@ -83,7 +99,9 @@ describe("Command Code model metadata checker", () => { it("reports CLI version drift even when model metadata is unchanged", () => { const metadata: CommandCodeModelMetadata = { imageModelIds: ["vision-model"], + reasoningModelIds: ["vision-model"], reasoningEfforts: { "vision-model": ["low"] }, + maxOutputTokens: { "vision-model": 32_768 }, } const diff = diffModelMetadata(metadata, metadata, "1.32.2", "1.33.0") @@ -96,10 +114,12 @@ describe("Command Code model metadata checker", () => { assert.equal( renderCommandCodeCatalog("1.33.0", { imageModelIds: ["b-model", "a-model"], + reasoningModelIds: ["c-model", "a-model"], reasoningEfforts: { "b-model": ["high", "max"], "a-model": ["low"], }, + maxOutputTokens: { "b-model": 32_768 }, }), `export const COMMAND_CODE_CLI_VERSION = "1.33.0" @@ -115,10 +135,19 @@ export const MODEL_INPUT_MODALITIES: Readonly> = { + "a-model": true, + "c-model": true, +} + export const MODEL_EFFORTS: Readonly> = { "a-model": ["low"], "b-model": ["high", "max"], } + +export const MODEL_MAX_OUTPUT_TOKENS: Readonly> = { + "b-model": 32_768, +} `, ) assert.equal( diff --git a/tests/test-models.ts b/tests/test-models.ts index 391e06b..c40d321 100644 --- a/tests/test-models.ts +++ b/tests/test-models.ts @@ -16,6 +16,8 @@ import { loadCommandCodeModels, MODEL_EFFORTS, MODEL_INPUT_MODALITIES, + MODEL_MAX_OUTPUT_TOKENS, + MODEL_REASONING, modelSupportsImageInput, thinkingLevelMapForEfforts, thinkingMetadataForModel, @@ -41,7 +43,7 @@ const EXPECTED_MODELS: readonly CommandCodeModel[] = [ id: "Qwen/Qwen3.7-Max", name: "Qwen 3.7 Max (CC)", api: "openai-completions", - reasoning: false, + reasoning: true, contextWindow: 1_000_000, maxTokens: 65_536, }, @@ -124,17 +126,55 @@ describe("commandCodeModelsFromApiResponse()", () => { } }) - it("marks only known reasoning models as reasoning-capable", () => { + it("tracks reasoning independently from selectable effort levels", () => { const models = commandCodeModelsFromApiResponse({ object: "list", data: [ { ...API_RESPONSE.data[0], id: "deepseek/deepseek-v4-flash" }, + { ...API_RESPONSE.data[0], id: "moonshotai/Kimi-K3" }, { ...API_RESPONSE.data[0], id: "new-model-without-metadata" }, ], }) assert.equal(models[0]?.reasoning, true) - assert.equal(models[1]?.reasoning, false) + assert.equal(models[1]?.reasoning, true) + assert.deepEqual(thinkingMetadataForModel("moonshotai/Kimi-K3"), { + thinkingLevelMap: { + minimal: null, + low: null, + medium: null, + high: null, + xhigh: null, + max: null, + }, + }) + assert.equal(models[2]?.reasoning, false) + assert.equal(Object.keys(MODEL_REASONING).length, 48) + }) + + it("uses model-specific output limits from the CLI catalog", () => { + const models = commandCodeModelsFromApiResponse({ + object: "list", + data: [ + { ...API_RESPONSE.data[0], id: "Qwen/Qwen3.8-27B", context_length: 262_144 }, + { ...API_RESPONSE.data[0], id: "stealth/ox-alpha", context_length: 1_048_576 }, + { + ...API_RESPONSE.data[0], + id: "poolside/laguna-s-2.1-free", + context_length: 256_000, + }, + ], + }) + + assert.deepEqual( + models.map(({ id, maxTokens }) => ({ id, maxTokens })), + [ + { id: "Qwen/Qwen3.8-27B", maxTokens: 32_768 }, + { id: "stealth/ox-alpha", maxTokens: 131_072 }, + { id: "poolside/laguna-s-2.1-free", maxTokens: 32_768 }, + ], + ) + assert.equal(Object.keys(MODEL_MAX_OUTPUT_TOKENS).length, 3) }) it(`uses the command-code@${COMMAND_CODE_CLI_VERSION} reasoning effort catalog`, () => { @@ -151,6 +191,7 @@ describe("commandCodeModelsFromApiResponse()", () => { for (const [modelId, efforts] of Object.entries(MODEL_EFFORTS)) { const metadata = thinkingMetadataForModel(modelId) assert.ok(metadata, `${modelId} should have reasoning metadata`) + assert.ok(metadata.thinking) assert.equal(metadata.thinking.mode, "effort") assert.deepEqual(metadata.thinking.efforts, efforts) assert.deepEqual( diff --git a/tests/test-pricing.ts b/tests/test-pricing.ts index 202d8aa..d7141a5 100644 --- a/tests/test-pricing.ts +++ b/tests/test-pricing.ts @@ -27,7 +27,7 @@ const fixtureUrl = new URL("./fixtures/commandcode-model-ids.json", import.meta. const fixture = JSON.parse(await readFile(fixtureUrl, "utf-8")) as ModelCatalogSnapshot const pricingFixtureUrl = new URL("./fixtures/commandcode-pricing.json", import.meta.url) const pricingFixture = JSON.parse(await readFile(pricingFixtureUrl, "utf-8")) as PricingSnapshot -const freeModels = new Set(["poolside/laguna-s-2.1-free", "inclusionai/ling-3.0-flash-free"]) +const freeModels = new Set(["poolside/laguna-s-2.1-free", "stealth/ox-alpha"]) function assertCost( modelId: string, @@ -50,7 +50,7 @@ function assertCost( describe("MODEL_COSTS pricing overlay", () => { it("covers the current Command Code model catalog snapshot", () => { assert.equal(fixture.source, "https://api.commandcode.ai/provider/v1/models") - assert.match(fixture.fetchedAt, /^2026-08-22T/) + assert.match(fixture.fetchedAt, /^2026-08-25T/) const catalogIds = [...fixture.modelIds].sort() const pricedIds = Object.keys(MODEL_COSTS).sort() @@ -138,6 +138,24 @@ describe("MODEL_COSTS pricing overlay", () => { cacheRead: 0.03, cacheWrite: 0, }) + assertCost("Qwen/Qwen3.8-27B", { + input: 0.4, + output: 3, + cacheRead: 0.04, + cacheWrite: 0, + }) + assertCost("google/gemini-3.7-flash", { + input: 0.75, + output: 3.75, + cacheRead: 0.075, + cacheWrite: 0.04167, + }) + assertCost("meta/muse-spark-1.2-contributor", { + input: 0.1, + output: 0.2, + cacheRead: 0.002, + cacheWrite: 0, + }) }) it("uses the documented base rates for context-dependent models", () => { @@ -165,11 +183,20 @@ describe("MODEL_COSTS pricing overlay", () => { cacheRead: 0.02, cacheWrite: 0.25, }) + assert.deepEqual(MODEL_COSTS["xai/grok-4.6"]?.tiers, [ + { + inputTokensAbove: 200_000, + input: 4, + output: 12, + cacheRead: 1, + cacheWrite: 0, + }, + ]) }) it("tracks pricing provenance", () => { assert.equal(PRICING_SOURCE_URL, "https://commandcode.ai/docs/resources/pricing-limits") - assert.equal(PRICING_LAST_VERIFIED, "2026-08-22") + assert.equal(PRICING_LAST_VERIFIED, "2026-08-25") }) it("fails once temporary pricing needs review", () => {