cat-shark/pi-commandcode-provider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge main into fix/tool-call-streaming

Browse Source
This commit is contained in:
Patrick Wozniak
2026-08-25 17:18:02 +02:00
parent a624643d90 ff922d0cb3
commit 98ed83fc9c
37 changed files with 1022 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fixtures/commandcode-model-ids.json
Vendored
+9 -3
View File
@@ -1,5 +1,5 @@
{
"fetchedAt": "2026-08-22T21:19:37.782Z",
"fetchedAt": "2026-08-25T13:32:11.631Z",
"source": "https://api.commandcode.ai/provider/v1/models",
"modelIds": [
"claude-sonnet-5",
@@ -18,6 +18,7 @@
"gpt-5.4-mini",
"deepseek/deepseek-v4-pro",
"deepseek/deepseek-v4-flash",
"deepseek/deepseek-v4-flash-vision-exp",
"moonshotai/Kimi-K3",
"moonshotai/Kimi-K2.7-Code",
"moonshotai/Kimi-K2.7-Code-Highspeed",
@@ -34,6 +35,7 @@
"xiaomi/mimo-v2.5-pro",
"xiaomi/mimo-v2.5",
"Qwen/Qwen3.8-Max",
"Qwen/Qwen3.8-27B",
"Qwen/Qwen3.7-Max",
"Qwen/Qwen3.7-Plus",
"Qwen/Qwen3.7-Flash",
@@ -42,6 +44,7 @@
"stepfun/Step-3.7-Flash",
"stepfun/Step-3.5-Flash",
"tencent/hy3-paid",
"google/gemini-3.7-flash",
"google/gemini-3.6-flash",
"google/gemini-3.5-flash",
"google/gemini-3.5-flash-lite",
@@ -50,9 +53,12 @@
"nvidia/nemotron-3-ultra-550b-a55b",
"thinkingmachines/inkling",
"thinkingmachines/inkling-small",
"stealth/ox-alpha",
"poolside/laguna-s-2.1-free",
"inclusionai/ling-3.0-flash-free",
"meta/muse-spark-1.1",
"xai/grok-4.5"
"meta/muse-spark-1.2",
"meta/muse-spark-1.2-contributor",
"xai/grok-4.5",
"xai/grok-4.6"
]
}
tests/fixtures/commandcode-pricing.json
Vendored
+19 -12
View File
@@ -1,5 +1,5 @@
{
"verifiedAt": "2026-08-22",
"verifiedAt": "2026-08-25",
"source": "https://commandcode.ai/docs/resources/pricing-limits",
"tierPolicy": "Use request-wide input tiers; the highest threshold exceeded by input plus cache tokens applies to the full request.",
"tiers": {
@@ -7,12 +7,13 @@
"Qwen/Qwen3.7-Flash": [
[32000, 0.1, 0.4, 0.02, 0.125],
[256000, 0.2, 0.8, 0.04, 0.25]
]
],
"xai/grok-4.6": [[200000, 4, 12, 1, 0]]
},
"costs": {
"poolside/laguna-s-2.1-free": [0, 0, 0, 0],
"inclusionai/ling-3.0-flash-free": [0, 0, 0, 0],
"tencent/hy3-paid": [0.14, 0.58, 0.035, 0],
"deepseek/deepseek-v4-pro": [0.66, 1.98, 0.022, 0],
"deepseek/deepseek-v4-flash": [0.22, 0.66, 0.007, 0],
"deepseek/deepseek-v4-flash-vision-exp": [0.22, 0.66, 0.007, 0],
"moonshotai/Kimi-K3": [3, 15, 0.3, 0],
"moonshotai/Kimi-K2.7-Code": [0.95, 4, 0.19, 0],
"moonshotai/Kimi-K2.7-Code-Highspeed": [1.9, 8, 0.38, 0],
@@ -26,9 +27,10 @@
"MiniMaxAI/MiniMax-M3": [0.3, 1.2, 0.06, 0],
"MiniMaxAI/MiniMax-M2.7": [0.3, 1.2, 0.06, 0],
"MiniMaxAI/MiniMax-M2.5": [0.3, 1.2, 0.03, 0],
"deepseek/deepseek-v4-pro": [0.66, 1.98, 0.022, 0],
"deepseek/deepseek-v4-flash": [0.22, 0.66, 0.007, 0],
"xiaomi/mimo-v2.5-pro": [0.435, 0.87, 0.0036, 0],
"xiaomi/mimo-v2.5": [0.14, 0.28, 0.0028, 0],
"Qwen/Qwen3.8-Max": [2, 6, 0.25, 2.5],
"Qwen/Qwen3.8-27B": [0.4, 3, 0.04, 0],
"Qwen/Qwen3.7-Max": [2.5, 7.5, 0.5, 3.13],
"Qwen/Qwen3.7-Plus": [0.4, 1.6, 0.08, 0.5],
"Qwen/Qwen3.7-Flash": [0.03, 0.13, 0.006, 0.038],
@@ -36,13 +38,12 @@
"Qwen/Qwen3.6-Plus": [0.5, 3, 0.1, 0],
"stepfun/Step-3.7-Flash": [0.2, 1.15, 0.04, 0],
"stepfun/Step-3.5-Flash": [0.1, 0.3, 0.02, 0],
"xiaomi/mimo-v2.5-pro": [0.435, 0.87, 0.0036, 0],
"xiaomi/mimo-v2.5": [0.14, 0.28, 0.0028, 0],
"tencent/hy3-paid": [0.14, 0.58, 0.035, 0],
"nvidia/nemotron-3-ultra-550b-a55b": [0.6, 2.4, 0.12, 0],
"sakana/fugu-ultra": [5, 30, 0.5, 0],
"thinkingmachines/inkling": [1, 4.05, 0.17, 0],
"thinkingmachines/inkling-small": [0.5, 1.2, 0.1, 0],
"meta/muse-spark-1.1": [1.25, 4.25, 0.15, 0],
"poolside/laguna-s-2.1-free": [0, 0, 0, 0],
"stealth/ox-alpha": [0, 0, 0, 0],
"claude-sonnet-5": [2, 10, 0.2, 2.5],
"claude-sonnet-4-6": [3, 15, 0.3, 3.75],
"claude-fable-5": [10, 50, 1, 12.5],
@@ -57,10 +58,16 @@
"gpt-5.4": [2.5, 15, 0.25, 0],
"gpt-5.3-codex": [2, 8, 0.5, 0],
"gpt-5.4-mini": [0.75, 4.5, 0.075, 0],
"google/gemini-3.7-flash": [0.75, 3.75, 0.075, 0.04167],
"google/gemini-3.6-flash": [1.5, 7.5, 0.15, 0],
"google/gemini-3.5-flash": [1.5, 9, 0.15, 0],
"google/gemini-3.5-flash-lite": [0.3, 2.5, 0.03, 0],
"google/gemini-3.1-flash-lite": [0.25, 1.5, 0.03, 0],
"xai/grok-4.5": [2, 6, 0.5, 0]
"sakana/fugu-ultra": [5, 30, 0.5, 0],
"meta/muse-spark-1.1": [1.25, 4.25, 0.15, 0],
"meta/muse-spark-1.2": [1.25, 4.25, 0.15, 0],
"meta/muse-spark-1.2-contributor": [0.1, 0.2, 0.002, 0],
"xai/grok-4.5": [2, 6, 0.5, 0],
"xai/grok-4.6": [2, 6, 0.5, 0]
}
}
tests/test-api-key.ts
+10 -3
View File
@@ -21,10 +21,17 @@ async function withAuthFile(
}
describe("getConfiguredApiKey()", () => {
it("prefers the environment variable", () => {
it("prefers the official environment variable and keeps the legacy alias", () => {
assert.equal(
getConfiguredApiKey({ env: { COMMANDCODE_API_KEY: "env-key" }, authPaths: [] }),
"env-key",
getConfiguredApiKey({
env: { COMMAND_CODE_API_KEY: "official-key", COMMANDCODE_API_KEY: "legacy-key" },
authPaths: [],
}),
"official-key",
)
assert.equal(
getConfiguredApiKey({ env: { COMMANDCODE_API_KEY: "legacy-key" }, authPaths: [] }),
"legacy-key",
)
})
tests/test-live-e2e.mjs
+130 -14
View File
@@ -27,7 +27,20 @@ const extensionPath = join(projectDir, "index.ts")
const testModel = process.env.COMMANDCODE_E2E_MODEL ?? "deepseek/deepseek-v4-flash"
const testProfile = process.env.COMMANDCODE_E2E_PROFILE
const expectedTransport =
testProfile === "go" ? "generate" : testProfile === "provider" ? "provider" : undefined
testProfile === "go"
? "generate"
: testProfile === "goat" || testProfile === "provider"
? "provider"
: undefined
const expectedPlan =
testProfile === "go"
? "go"
: testProfile === "goat"
? "goat"
: testProfile === "provider"
? "provider"
: undefined
const goatVisionModel = process.env.COMMANDCODE_E2E_GOAT_VISION_MODEL ?? "google/gemini-3.7-flash"
const marker = "commandcode-live-e2e-ok"
function findPiBinary() {
@@ -48,6 +61,7 @@ function findPiBinary() {
function hasAuthMetadata() {
return (
Boolean(process.env.COMMAND_CODE_API_KEY) ||
Boolean(process.env.COMMANDCODE_API_KEY) ||
existsSync(join(homedir(), ".commandcode", "auth.json")) ||
existsSync(join(homedir(), ".pi", "agent", "auth.json"))
@@ -70,6 +84,7 @@ function safeEnv(overrides = {}) {
env.PI_CODING_AGENT_DIR = profileAgentDir
env.COMMANDCODE_MODELS_CACHE = join(profileAgentDir, "commandcode-models.json")
} else {
delete env.COMMAND_CODE_API_KEY
delete env.COMMANDCODE_API_KEY
}
return env
@@ -102,7 +117,7 @@ function run(command, args, options = {}) {
})
}
async function runRpc(extension, action, timeoutMs = 120_000) {
async function runRpc(extension, action, timeoutMs = 120_000, model = testModel) {
const child = spawn(
piBin,
[
@@ -114,7 +129,9 @@ async function runRpc(extension, action, timeoutMs = 120_000) {
"--provider",
"commandcode",
"--model",
testModel,
model,
"--thinking",
"high",
],
{ cwd: projectDir, env: safeEnv(), stdio: ["pipe", "pipe", "pipe"] },
)
@@ -222,8 +239,19 @@ try {
await waitFor(
(event) => event.type === "response" && event.id === "reasoning-turn-1" && event.success,
)
await waitFor((event) => event.type === "agent_settled")
const firstThinkingDeltas = countThinkingDeltas(firstStart)
const firstSettled = await waitFor(
(event) => event.type === "agent_settled" && events.indexOf(event) >= firstStart,
)
const firstSettledIndex = events.indexOf(firstSettled)
const firstThinkingDeltas = events
.slice(firstStart, firstSettledIndex + 1)
.filter(
(event) =>
event.type === "message_update" &&
event.assistantMessageEvent?.type === "thinking_delta" &&
typeof event.assistantMessageEvent.delta === "string" &&
event.assistantMessageEvent.delta.length > 0,
).length
const secondStart = events.length
send({
@@ -235,15 +263,16 @@ try {
await waitFor(
(event) => event.type === "response" && event.id === "reasoning-turn-2" && event.success,
)
await waitFor((event) => event.type === "agent_settled" && events.indexOf(event) >= secondStart)
const secondSettled = await waitFor(
(event) => event.type === "agent_settled" && events.indexOf(event) >= secondStart,
)
const secondThinkingDeltas = countThinkingDeltas(secondStart)
assert.ok(events.indexOf(secondSettled) >= secondStart)
return { firstThinkingDeltas, secondThinkingDeltas, stderr: getStderr() }
})
if (testProfile !== "provider") {
assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning")
assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning")
}
assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning")
assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning")
assert.doesNotMatch(multiTurn.stderr, /Bearer\s+\S+/i)
console.log("[live-e2e] live runtime refresh/status commands")
@@ -281,15 +310,66 @@ try {
typeof event.message === "string" &&
event.message.includes("source:"),
)
return { names, refresh: refresh.message, status: status.message, stderr: getStderr() }
send({ id: "quota", type: "prompt", message: "/commandcode-quota" })
await waitFor((event) => event.type === "response" && event.id === "quota" && event.success)
const quota = await waitFor(
(event) =>
event.type === "extension_ui_request" &&
event.method === "notify" &&
typeof event.message === "string" &&
event.message.includes("Plan:"),
)
return {
names,
refresh: refresh.message,
status: status.message,
quota: quota.message,
stderr: getStderr(),
}
})
assert.ok(runtime.names.includes("commandcode-refresh"))
assert.ok(runtime.names.includes("commandcode-status"))
assert.ok(runtime.names.includes("commandcode-quota"))
assert.match(runtime.refresh, /model catalog (?:refreshed|unchanged)/)
if (expectedTransport) assert.match(runtime.status, new RegExp(`transport: ${expectedTransport}`))
assert.match(runtime.status, /source: (?:live|cache)/)
assert.match(runtime.status, /model count: [1-9][0-9]*/)
assert.doesNotMatch(`${runtime.refresh}\n${runtime.status}\n${runtime.stderr}`, /Bearer\s+\S+/i)
if (expectedPlan) assert.match(runtime.quota, new RegExp(`Plan:.*\\b${expectedPlan}\\b`, "i"))
assert.doesNotMatch(
`${runtime.refresh}\n${runtime.status}\n${runtime.quota}\n${runtime.stderr}`,
/Bearer\s+\S+/i,
)
console.log("[live-e2e] live abort through real RPC host")
const abortResult = await runRpc(extensionPath, async ({ send, waitFor, events, getStderr }) => {
const startIndex = events.length
send({
id: "abort-turn",
type: "prompt",
message: "Write a very long detailed explanation of every integer from 1 to 10000.",
})
await waitFor(
(event) => event.type === "response" && event.id === "abort-turn" && event.success,
)
await waitFor((event) => event.type === "message_update" && events.indexOf(event) >= startIndex)
send({ id: "abort", type: "abort" })
await waitFor((event) => event.type === "response" && event.id === "abort" && event.success)
await waitFor((event) => event.type === "agent_settled" && events.indexOf(event) >= startIndex)
return {
aborted: events
.slice(startIndex)
.some(
(event) =>
event.type === "message_end" &&
event.message?.role === "assistant" &&
event.message?.stopReason === "aborted",
),
stderr: getStderr(),
}
})
assert.equal(abortResult.aborted, true)
assert.doesNotMatch(abortResult.stderr, /Bearer\s+\S+/i)
console.log("[live-e2e] live tool-call round trip")
const toolRoot = join(tempRoot, "tool-roundtrip")
@@ -317,9 +397,45 @@ try {
)
assert.equal(toolResult.code, 0, toolResult.stderr)
assert.match(toolResult.stdout, new RegExp(marker))
assert.equal(readFileSync(targetPath, "utf-8"), marker)
assert.equal(readFileSync(targetPath, "utf-8").trimEnd(), marker)
if (testProfile !== "provider") {
if (testProfile === "goat") {
console.log("[live-e2e] live vision request through Provider API")
const vision = await runRpc(
extensionPath,
async ({ send, waitFor, events, getStderr }) => {
const startIndex = events.length
send({
id: "vision",
type: "prompt",
message: "Describe the attached image briefly.",
images: [
{
type: "image",
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
mimeType: "image/png",
},
],
})
await waitFor(
(event) => event.type === "response" && event.id === "vision" && event.success,
)
await waitFor(
(event) => event.type === "agent_settled" && events.indexOf(event) >= startIndex,
)
const messageEnd = events
.slice(startIndex)
.find((event) => event.type === "message_end" && event.message?.role === "assistant")
return { messageEnd, stderr: getStderr() }
},
180_000,
goatVisionModel,
)
assert.notEqual(vision.messageEnd?.message?.stopReason, "error")
assert.doesNotMatch(vision.stderr, /Bearer\s+\S+/i)
}
if (testProfile === "go") {
console.log("[live-e2e] image rejection through real RPC host")
const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => {
send({
tests/test-model-metadata-check.ts
+38 -9
View File
@@ -5,6 +5,7 @@ import {
commandCodeModelMetadataFromContents,
diffModelMetadata,
hasModelMetadataDiff,
parseBundleModelCapabilities,
parseKnownTextOnlyModelIds,
parseModelsReference,
parsePackageVersion,
@@ -21,7 +22,7 @@ const MODELS_REFERENCE = `
`
const CLI_BUNDLE =
'const catalog=new Set(["text-model"]),__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")'
'const V={id:"vision-model",inputModalities:["text","image"],reasoning:!0,reasoningEfforts:["low","high"],maxOutputTokens:32768},T={id:"text-model",inputModalities:["text"]},catalog=new Set(["text-model"]),__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")'
describe("Command Code model metadata checker", () => {
it("parses model ids and reasoning efforts from the generated reference", () => {
@@ -42,29 +43,39 @@ describe("Command Code model metadata checker", () => {
assert.throws(() => parsePackageVersion("latest"), /one semantic version/)
})
it("derives image support by excluding known text-only models", () => {
it("derives image, reasoning, effort, and output-limit metadata", () => {
assert.deepEqual(parseBundleModelCapabilities(CLI_BUNDLE, ["text-model", "vision-model"]), {
reasoningModelIds: ["vision-model"],
maxOutputTokens: { "vision-model": 32_768 },
})
assert.deepEqual(commandCodeModelMetadataFromContents(MODELS_REFERENCE, CLI_BUNDLE), {
imageModelIds: ["vision-model"],
reasoningModelIds: ["vision-model"],
reasoningEfforts: { "vision-model": ["low", "high"] },
maxOutputTokens: { "vision-model": 32_768 },
})
})
it("reports additions, removals, and changed reasoning efforts", () => {
const current: CommandCodeModelMetadata = {
imageModelIds: ["removed-image", "stable-image"],
reasoningModelIds: ["removed-reasoning", "stable-reasoning"],
reasoningEfforts: {
"changed-reasoning": ["low"],
"removed-reasoning": ["high"],
"stable-reasoning": ["low", "high"],
"changed-effort": ["low"],
"removed-effort": ["high"],
"stable-effort": ["low", "high"],
},
maxOutputTokens: { "changed-output": 1, "removed-output": 2, "stable-output": 3 },
}
const upstream: CommandCodeModelMetadata = {
imageModelIds: ["added-image", "stable-image"],
reasoningModelIds: ["added-reasoning", "stable-reasoning"],
reasoningEfforts: {
"added-reasoning": ["max"],
"changed-reasoning": ["low", "high"],
"stable-reasoning": ["low", "high"],
"added-effort": ["max"],
"changed-effort": ["low", "high"],
"stable-effort": ["low", "high"],
},
maxOutputTokens: { "added-output": 4, "changed-output": 5, "stable-output": 3 },
}
const diff = diffModelMetadata(current, upstream)
@@ -75,7 +86,12 @@ describe("Command Code model metadata checker", () => {
removedImageModelIds: ["removed-image"],
addedReasoningModelIds: ["added-reasoning"],
removedReasoningModelIds: ["removed-reasoning"],
changedReasoningModelIds: ["changed-reasoning"],
addedEffortModelIds: ["added-effort"],
removedEffortModelIds: ["removed-effort"],
changedEffortModelIds: ["changed-effort"],
addedMaxOutputModelIds: ["added-output"],
removedMaxOutputModelIds: ["removed-output"],
changedMaxOutputModelIds: ["changed-output"],
})
assert.equal(hasModelMetadataDiff(diff), true)
})
@@ -83,7 +99,9 @@ describe("Command Code model metadata checker", () => {
it("reports CLI version drift even when model metadata is unchanged", () => {
const metadata: CommandCodeModelMetadata = {
imageModelIds: ["vision-model"],
reasoningModelIds: ["vision-model"],
reasoningEfforts: { "vision-model": ["low"] },
maxOutputTokens: { "vision-model": 32_768 },
}
const diff = diffModelMetadata(metadata, metadata, "1.32.2", "1.33.0")
@@ -96,10 +114,12 @@ describe("Command Code model metadata checker", () => {
assert.equal(
renderCommandCodeCatalog("1.33.0", {
imageModelIds: ["b-model", "a-model"],
reasoningModelIds: ["c-model", "a-model"],
reasoningEfforts: {
"b-model": ["high", "max"],
"a-model": ["low"],
},
maxOutputTokens: { "b-model": 32_768 },
}),
`export const COMMAND_CODE_CLI_VERSION = "1.33.0"
@@ -115,10 +135,19 @@ export const MODEL_INPUT_MODALITIES: Readonly<Record<string, readonly CommandCod
"b-model": ["text", "image"],
}
export const MODEL_REASONING: Readonly<Record<string, true>> = {
"a-model": true,
"c-model": true,
}
export const MODEL_EFFORTS: Readonly<Record<string, readonly CommandCodeReasoningEffort[]>> = {
"a-model": ["low"],
"b-model": ["high", "max"],
}
export const MODEL_MAX_OUTPUT_TOKENS: Readonly<Record<string, number>> = {
"b-model": 32_768,
}
`,
)
assert.equal(
tests/test-models.ts
+44 -3
View File
@@ -16,6 +16,8 @@ import {
loadCommandCodeModels,
MODEL_EFFORTS,
MODEL_INPUT_MODALITIES,
MODEL_MAX_OUTPUT_TOKENS,
MODEL_REASONING,
modelSupportsImageInput,
thinkingLevelMapForEfforts,
thinkingMetadataForModel,
@@ -41,7 +43,7 @@ const EXPECTED_MODELS: readonly CommandCodeModel[] = [
id: "Qwen/Qwen3.7-Max",
name: "Qwen 3.7 Max (CC)",
api: "openai-completions",
reasoning: false,
reasoning: true,
contextWindow: 1_000_000,
maxTokens: 65_536,
},
@@ -124,17 +126,55 @@ describe("commandCodeModelsFromApiResponse()", () => {
}
})
it("marks only known reasoning models as reasoning-capable", () => {
it("tracks reasoning independently from selectable effort levels", () => {
const models = commandCodeModelsFromApiResponse({
object: "list",
data: [
{ ...API_RESPONSE.data[0], id: "deepseek/deepseek-v4-flash" },
{ ...API_RESPONSE.data[0], id: "moonshotai/Kimi-K3" },
{ ...API_RESPONSE.data[0], id: "new-model-without-metadata" },
],
})
assert.equal(models[0]?.reasoning, true)
assert.equal(models[1]?.reasoning, false)
assert.equal(models[1]?.reasoning, true)
assert.deepEqual(thinkingMetadataForModel("moonshotai/Kimi-K3"), {
thinkingLevelMap: {
minimal: null,
low: null,
medium: null,
high: null,
xhigh: null,
max: null,
},
})
assert.equal(models[2]?.reasoning, false)
assert.equal(Object.keys(MODEL_REASONING).length, 48)
})
it("uses model-specific output limits from the CLI catalog", () => {
const models = commandCodeModelsFromApiResponse({
object: "list",
data: [
{ ...API_RESPONSE.data[0], id: "Qwen/Qwen3.8-27B", context_length: 262_144 },
{ ...API_RESPONSE.data[0], id: "stealth/ox-alpha", context_length: 1_048_576 },
{
...API_RESPONSE.data[0],
id: "poolside/laguna-s-2.1-free",
context_length: 256_000,
},
],
})
assert.deepEqual(
models.map(({ id, maxTokens }) => ({ id, maxTokens })),
[
{ id: "Qwen/Qwen3.8-27B", maxTokens: 32_768 },
{ id: "stealth/ox-alpha", maxTokens: 131_072 },
{ id: "poolside/laguna-s-2.1-free", maxTokens: 32_768 },
],
)
assert.equal(Object.keys(MODEL_MAX_OUTPUT_TOKENS).length, 3)
})
it(`uses the command-code@${COMMAND_CODE_CLI_VERSION} reasoning effort catalog`, () => {
@@ -151,6 +191,7 @@ describe("commandCodeModelsFromApiResponse()", () => {
for (const [modelId, efforts] of Object.entries(MODEL_EFFORTS)) {
const metadata = thinkingMetadataForModel(modelId)
assert.ok(metadata, `${modelId} should have reasoning metadata`)
assert.ok(metadata.thinking)
assert.equal(metadata.thinking.mode, "effort")
assert.deepEqual(metadata.thinking.efforts, efforts)
assert.deepEqual(
tests/test-oauth.ts
+119 -39
View File
@@ -9,7 +9,7 @@ import assert from "node:assert/strict"
import { describe, it } from "node:test"
import { startAuthServer, type AuthCallback } from "../src/auth-server.ts"
import { getApiKey, login, refreshToken, sanitizeApiKey } from "../src/oauth.ts"
import { getApiKey, login, refreshToken, sanitizeApiKey, validateApiKey } from "../src/oauth.ts"
/**
* Helper: wait for an HTTP server to close, or resolve immediately if already closed.
@@ -24,9 +24,27 @@ function waitForClose(server: {
})
}
async function withValidApiKeyFetch<T>(run: () => Promise<T>): Promise<T> {
const originalFetch = globalThis.fetch
globalThis.fetch = (input, init) => {
if (String(input).endsWith("/alpha/whoami")) {
return Promise.resolve(new Response(JSON.stringify({ user: {} }), { status: 200 }))
}
return originalFetch(input, init)
}
try {
return await run()
} finally {
globalThis.fetch = originalFetch
}
}
describe("startAuthServer()", () => {
it("starts on a localhost port and accepts a valid callback POST", async () => {
const { server, port, waitForCallback } = await startAuthServer({ startPort: 0 })
const { server, port, waitForCallback } = await startAuthServer({
startPort: 0,
expectedState: "test-state-token",
})
const callbackData: AuthCallback = {
apiKey: "user_testKey123",
@@ -57,6 +75,42 @@ describe("startAuthServer()", () => {
await waitForClose(server)
})
it("rejects a mismatched state without closing the callback server", async () => {
const { server, port, waitForCallback } = await startAuthServer({
startPort: 0,
expectedState: "correct-state",
})
const invalidResponse = await fetch(`http://127.0.0.1:${port}/callback`, {
method: "POST",
headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" },
body: JSON.stringify({
apiKey: "user_badState",
state: "wrong-state",
userId: "user_789",
userName: "Attacker",
keyName: "evil-key",
}),
})
assert.equal(invalidResponse.status, 403)
assert.equal(server.listening, true)
const validResponse = await fetch(`http://127.0.0.1:${port}/callback`, {
method: "POST",
headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" },
body: JSON.stringify({
apiKey: "user_valid",
state: "correct-state",
userId: "user_123",
userName: "Valid User",
keyName: "valid-key",
}),
})
assert.equal(validResponse.status, 200)
assert.equal((await waitForCallback).apiKey, "user_valid")
await waitForClose(server)
})
it("rejects when the callback indicates access_denied", async () => {
const { server, port, waitForCallback } = await startAuthServer({ startPort: 0 })
@@ -176,6 +230,18 @@ describe("OAuth functions", () => {
it("sanitizeApiKey removes paste markers, control chars, and whitespace", () => {
assert.equal(sanitizeApiKey("\u001b[200~ user_manualKey\n\u001b[201~"), "user_manualKey")
})
it("validates manual API keys through whoami", async () => {
await validateApiKey("valid-key", {
fetchImpl: () => Promise.resolve(new Response(JSON.stringify({ user: {} }), { status: 200 })),
})
await assert.rejects(
validateApiKey("invalid-key", {
fetchImpl: () => Promise.resolve(new Response("unauthorized", { status: 401 })),
}),
/Invalid Command Code API key/,
)
})
})
describe("login()", () => {
@@ -242,15 +308,17 @@ describe("login()", () => {
const promptMessages: string[] = []
try {
const result = await login({
onAuth(params: { url: string }) {
authUrl = params.url
},
async onPrompt(params: { message: string }): Promise<string> {
promptMessages.push(params.message)
return promptMessages.length === 1 ? "" : "\u001b[200~ user_manualApiKey\n\u001b[201~"
},
})
const result = await withValidApiKeyFetch(() =>
login({
onAuth(params: { url: string }) {
authUrl = params.url
},
async onPrompt(params: { message: string }): Promise<string> {
promptMessages.push(params.message)
return promptMessages.length === 1 ? "" : "\u001b[200~ user_manualApiKey\n\u001b[201~"
},
}),
)
assert.match(authUrl, /^https:\/\/commandcode\.ai\/studio\/auth\/cli\?/)
assert.match(promptMessages[1] ?? "", /Paste your Command Code API key/)
@@ -265,14 +333,16 @@ describe("login()", () => {
it("accepts a directly pasted API key", async () => {
let authOpened = false
const result = await login({
onAuth() {
authOpened = true
},
onPrompt(): Promise<string> {
return Promise.resolve("user_directApiKey")
},
})
const result = await withValidApiKeyFetch(() =>
login({
onAuth() {
authOpened = true
},
onPrompt(): Promise<string> {
return Promise.resolve("user_directApiKey")
},
}),
)
assert.equal(authOpened, false)
assert.equal(result.access, "user_directApiKey")
@@ -280,21 +350,23 @@ describe("login()", () => {
it("offers an explicit API key prompt", async () => {
let promptCount = 0
const result = await login({
onAuth() {
throw new Error("browser should not open")
},
onPrompt(): Promise<string> {
promptCount += 1
return Promise.resolve(promptCount === 1 ? "key" : "user_promptedApiKey")
},
})
const result = await withValidApiKeyFetch(() =>
login({
onAuth() {
throw new Error("browser should not open")
},
onPrompt(): Promise<string> {
promptCount += 1
return Promise.resolve(promptCount === 1 ? "key" : "user_promptedApiKey")
},
}),
)
assert.equal(result.access, "user_promptedApiKey")
assert.equal(promptCount, 2)
})
it("rejects on state token mismatch", async () => {
it("keeps waiting after a state mismatch and accepts the legitimate callback", async () => {
let authUrl = ""
const callbacks = {
onAuth(params: { url: string }) {
@@ -305,12 +377,7 @@ describe("login()", () => {
},
}
const loginPromise: Promise<string> = login(callbacks).then(
() => {
throw new Error("Expected login to reject")
},
(e: Error) => e.message,
)
const loginPromise = login(callbacks)
// Wait for onAuth to be called asynchronously
while (!authUrl) await new Promise((resolve) => setTimeout(resolve, 10))
@@ -318,8 +385,8 @@ describe("login()", () => {
const url = new URL(authUrl)
const port = parseInt(url.searchParams.get("callback")?.match(/localhost:(\d+)/)?.[1] ?? "0")
// Post back with a wrong state token
await fetch(`http://127.0.0.1:${port}/callback`, {
// Post back with a wrong state token.
const invalidResponse = await fetch(`http://127.0.0.1:${port}/callback`, {
method: "POST",
headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" },
body: JSON.stringify({
@@ -331,7 +398,20 @@ describe("login()", () => {
}),
})
const errorMsg = await loginPromise
assert.match(errorMsg, /State token mismatch/)
assert.equal(invalidResponse.status, 403)
const validResponse = await fetch(`http://127.0.0.1:${port}/callback`, {
method: "POST",
headers: { "Content-Type": "application/json", Origin: "https://commandcode.ai" },
body: JSON.stringify({
apiKey: "user_goodState",
state: url.searchParams.get("state"),
userId: "user_123",
userName: "Real User",
keyName: "real-key",
}),
})
assert.equal(validResponse.status, 200)
assert.equal((await loginPromise).access, "user_goodState")
})
})
tests/test-omp-compat.mjs
+1 -1
View File
@@ -133,7 +133,7 @@ function runOmp(args, timeoutMs = 30_000) {
HOME: tempHome,
USERPROFILE: tempHome,
PI_CODING_AGENT_DIR: join(tempHome, ".omp", "agent"),
COMMANDCODE_API_KEY: "mock-key",
COMMAND_CODE_API_KEY: "mock-key",
COMMANDCODE_API_BASE: `${apiBase}/provider/v1`,
COMMANDCODE_MODELS_URL: `${apiBase}/provider/v1/models`,
},
tests/test-pi-authenticated.mjs
+3 -2
View File
@@ -22,7 +22,7 @@ const { writeFileSync } = require("node:fs")
writeFileSync(process.env.FAKE_PI_LOG, JSON.stringify({
args: process.argv.slice(2),
agentDir: process.env.PI_CODING_AGENT_DIR ?? null,
apiKey: process.env.COMMANDCODE_API_KEY ?? null,
apiKey: process.env.COMMAND_CODE_API_KEY ?? process.env.COMMANDCODE_API_KEY ?? null,
skipVersionCheck: process.env.PI_SKIP_VERSION_CHECK,
}))
NODE
@@ -38,7 +38,8 @@ NODE
PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ""}`,
FAKE_PI_LOG: logPath,
PI_CODING_AGENT_DIR: "/existing/pi-agent",
COMMANDCODE_API_KEY: "existing-key",
COMMAND_CODE_API_KEY: "official-existing-key",
COMMANDCODE_API_KEY: "legacy-existing-key",
},
encoding: "utf8",
})
tests/test-pi-isolated.mjs
+4 -2
View File
@@ -26,7 +26,8 @@ appendFileSync(process.env.FAKE_PI_LOG, JSON.stringify({
skipVersionCheck: process.env.PI_SKIP_VERSION_CHECK,
home: process.env.HOME,
userProfile: process.env.USERPROFILE,
inheritedApiKey: process.env.COMMANDCODE_API_KEY ?? null,
inheritedApiKey:
process.env.COMMAND_CODE_API_KEY ?? process.env.COMMANDCODE_API_KEY ?? null,
}) + "\\n")
NODE
if [ "$1" = "install" ]; then exit 0; fi
@@ -42,7 +43,8 @@ exit ${exitStatus}
...process.env,
PATH: `${fakeBin}${delimiter}${process.env.PATH ?? ""}`,
FAKE_PI_LOG: logPath,
COMMANDCODE_API_KEY: "must-not-leak",
COMMAND_CODE_API_KEY: "must-not-leak-official",
COMMANDCODE_API_KEY: "must-not-leak-legacy",
},
encoding: "utf8",
})
tests/test-pi-local.mjs
+2 -2
View File
@@ -215,8 +215,8 @@ const env = {
PI_CODING_AGENT_DIR: agentDir,
PI_CODING_AGENT_SESSION_DIR: join(tempHome, "sessions"),
COMMANDCODE_API_BASE: `${apiBase}/provider/v1`,
COMMANDCODE_API_KEY: "mock-key",
COMMANDCODE_ZDR: "1",
COMMAND_CODE_API_KEY: "mock-key",
CMD_ZDR: "1",
COMMANDCODE_MODELS_URL: `${apiBase}/provider/v1/models`,
}
tests/test-pricing.ts
+30 -3
View File
@@ -27,7 +27,7 @@ const fixtureUrl = new URL("./fixtures/commandcode-model-ids.json", import.meta.
const fixture = JSON.parse(await readFile(fixtureUrl, "utf-8")) as ModelCatalogSnapshot
const pricingFixtureUrl = new URL("./fixtures/commandcode-pricing.json", import.meta.url)
const pricingFixture = JSON.parse(await readFile(pricingFixtureUrl, "utf-8")) as PricingSnapshot
const freeModels = new Set(["poolside/laguna-s-2.1-free", "inclusionai/ling-3.0-flash-free"])
const freeModels = new Set(["poolside/laguna-s-2.1-free", "stealth/ox-alpha"])
function assertCost(
modelId: string,
@@ -50,7 +50,7 @@ function assertCost(
describe("MODEL_COSTS pricing overlay", () => {
it("covers the current Command Code model catalog snapshot", () => {
assert.equal(fixture.source, "https://api.commandcode.ai/provider/v1/models")
assert.match(fixture.fetchedAt, /^2026-08-22T/)
assert.match(fixture.fetchedAt, /^2026-08-25T/)
const catalogIds = [...fixture.modelIds].sort()
const pricedIds = Object.keys(MODEL_COSTS).sort()
@@ -138,6 +138,24 @@ describe("MODEL_COSTS pricing overlay", () => {
cacheRead: 0.03,
cacheWrite: 0,
})
assertCost("Qwen/Qwen3.8-27B", {
input: 0.4,
output: 3,
cacheRead: 0.04,
cacheWrite: 0,
})
assertCost("google/gemini-3.7-flash", {
input: 0.75,
output: 3.75,
cacheRead: 0.075,
cacheWrite: 0.04167,
})
assertCost("meta/muse-spark-1.2-contributor", {
input: 0.1,
output: 0.2,
cacheRead: 0.002,
cacheWrite: 0,
})
})
it("uses the documented base rates for context-dependent models", () => {
@@ -165,11 +183,20 @@ describe("MODEL_COSTS pricing overlay", () => {
cacheRead: 0.02,
cacheWrite: 0.25,
})
assert.deepEqual(MODEL_COSTS["xai/grok-4.6"]?.tiers, [
{
inputTokensAbove: 200_000,
input: 4,
output: 12,
cacheRead: 1,
cacheWrite: 0,
},
])
})
it("tracks pricing provenance", () => {
assert.equal(PRICING_SOURCE_URL, "https://commandcode.ai/docs/resources/pricing-limits")
assert.equal(PRICING_LAST_VERIFIED, "2026-08-22")
assert.equal(PRICING_LAST_VERIFIED, "2026-08-25")
})
it("fails once temporary pricing needs review", () => {
tests/test-pure-functions.ts
+45 -4
View File
@@ -27,8 +27,18 @@ import { redactCommandCodeErrorText } from "../src/overflow.ts"
import { objectAt } from "./helpers.ts"
describe("getApiKey()", () => {
it("uses COMMANDCODE_API_KEY from provided env", () => {
assert.equal(getApiKey({ env: { COMMANDCODE_API_KEY: "env-key" }, authPaths: [] }), "env-key")
it("uses the official API key env var before the legacy alias", () => {
assert.equal(
getApiKey({
env: { COMMAND_CODE_API_KEY: "official-key", COMMANDCODE_API_KEY: "legacy-key" },
authPaths: [],
}),
"official-key",
)
assert.equal(
getApiKey({ env: { COMMANDCODE_API_KEY: "legacy-key" }, authPaths: [] }),
"legacy-key",
)
})
it("reads apiKey, commandcode, pi OAuth, and official CLI credential fields", () => {
@@ -109,11 +119,14 @@ describe("error redaction", () => {
describe("pickCommandCodeApiKey()", () => {
it("falls back to the host key for a placeholder registry value", () => {
assert.equal(pickCommandCodeApiKey("$COMMAND_CODE_API_KEY", "file-key"), "file-key")
assert.equal(pickCommandCodeApiKey("COMMAND_CODE_API_KEY", "file-key"), "file-key")
assert.equal(pickCommandCodeApiKey("$COMMANDCODE_API_KEY", "file-key"), "file-key")
assert.equal(pickCommandCodeApiKey("COMMANDCODE_API_KEY", "file-key"), "file-key")
})
it("returns undefined when only a placeholder is provided (no fallback)", () => {
assert.equal(pickCommandCodeApiKey("$COMMAND_CODE_API_KEY", undefined), undefined)
assert.equal(pickCommandCodeApiKey("$COMMANDCODE_API_KEY", undefined), undefined)
})
@@ -199,6 +212,12 @@ describe("textContent()", () => {
)
})
it("normalizes malformed string and object content", () => {
assert.equal(textContent({ content: "raw result" }), "raw result")
assert.equal(textContent({ content: { ok: true } }), '{"ok":true}')
assert.equal(textContent({ content: null }), "")
})
it("handles empty or missing content", () => {
assert.equal(textContent({ content: [] }), "")
assert.equal(textContent({}), "")
@@ -531,6 +550,23 @@ describe("messagesToCC()", () => {
assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "hello\nworld")
})
it("preserves malformed string tool results instead of sending empty output", () => {
const result = messagesToCC([
{
role: "assistant",
content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }],
},
{
role: "toolResult",
toolCallId: "c1",
toolName: "read",
content: "raw result",
},
])
assert.equal(objectAt(result, ["1", "content", "0", "output", "value"]), "raw result")
})
it("serializes image inputs in the current Command Code wire format", () => {
assert.deepEqual(
messagesToCC(
@@ -632,7 +668,7 @@ describe("messagesToCC()", () => {
])
})
it("drops orphaned tool calls that have no matching tool result", () => {
it("synthesizes missing results for orphaned tool calls", () => {
const result = messagesToCC([
{ role: "user", content: "edit a file" },
{
@@ -651,7 +687,12 @@ describe("messagesToCC()", () => {
assert.equal(objectAt(result, ["1", "role"]), "assistant")
assert.equal(objectAt(result, ["1", "content", "0", "type"]), "text")
assert.equal(objectAt(result, ["1", "content", "1"]), undefined)
assert.equal(objectAt(result, ["1", "content", "1", "type"]), "tool-call")
assert.equal(objectAt(result, ["2", "role"]), "tool")
assert.match(
String(objectAt(result, ["2", "content", "0", "output", "value"])),
/did not complete/,
)
})
it("handles empty conversations", () => {
tests/test-quota-command.ts
+1 -1
View File
@@ -68,7 +68,7 @@ describe("commandcode-quota command", () => {
})
assert.ok(pi.handler)
const ctx = context("$COMMANDCODE_API_KEY")
const ctx = context("$COMMAND_CODE_API_KEY")
await pi.handler("", ctx.value)
assert.equal(ctx.waited(), true)
assert.equal(requestKey, "fallback-key")
tests/test-smoke.mjs
+2 -1
View File
@@ -7,7 +7,7 @@
* 3. Can complete a simple prompt (requires Command Code auth)
*
* Run with: node tests/test-smoke.mjs
* Requires: pi on PATH plus COMMANDCODE_API_KEY or live pi auth files.
* Requires: pi on PATH plus COMMAND_CODE_API_KEY (or legacy COMMANDCODE_API_KEY) or live pi auth files.
*/
import { spawn } from "node:child_process"
@@ -48,6 +48,7 @@ const RPC_QUERY_TIMEOUT = 60_000
function hasCommandCodeAuth() {
return (
!!process.env.COMMAND_CODE_API_KEY ||
!!process.env.COMMANDCODE_API_KEY ||
existsSync(join(homedir(), ".commandcode", "auth.json")) ||
existsSync(join(homedir(), ".pi", "agent", "auth.json"))
tests/test-stream.ts
+119 -2
View File
@@ -77,6 +77,23 @@ describe("streamCommandCode — auth", () => {
)
})
it("accepts the official CLI API key environment variable", async () => {
server.mockResponse({
type: "success",
events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
})
const { streamCommandCode } = createTestDeps({
apiBase: server.baseUrl(),
env: { COMMAND_CODE_API_KEY: "official-env-key" },
})
await collectEvents(
streamCommandCode(makeModel(), makeContext(), { apiKey: "$COMMAND_CODE_API_KEY" }),
)
assert.equal(server.lastRequestHeaders().authorization, "Bearer official-env-key")
})
it("uses options.apiKey in the Authorization header", async () => {
server.mockResponse({
type: "success",
@@ -653,7 +670,7 @@ describe("streamCommandCode — request serialization", () => {
assert.equal(objectAt(body, ["params", "stream"]), true)
assert.equal(objectAt(body, ["params", "max_tokens"]), 500)
assert.equal(objectAt(body, ["params", "reasoning_effort"]), undefined)
assert.equal(objectAt(body, ["params", "temperature"]), 0.3)
assert.equal(objectAt(body, ["params", "temperature"]), undefined)
assert.equal(objectAt(body, ["params", "system"]), "You are a test assistant.")
assert.equal(objectAt(body, ["memory"]), null)
assert.equal(objectAt(body, ["taste"]), null)
@@ -671,10 +688,53 @@ describe("streamCommandCode — request serialization", () => {
assert.equal(headers["x-command-code-version"], COMMAND_CODE_CLI_VERSION)
assert.equal(headers["x-project-slug"], "repo")
assert.equal(headers["x-taste-learning"], "true")
assert.equal(headers["x-co-flag"], "false")
assert.equal(headers["user-agent"], "cli")
assert.equal(headers["x-co-flag"], undefined)
assert.equal(headers["x-session-id"], undefined)
})
it("forwards explicit temperature and stable session metadata", async () => {
server.mockResponse({
type: "success",
events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
})
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
await collectEvents(
streamCommandCode(makeModel(), makeContext(), {
apiKey: "mock-key",
temperature: 0.7,
sessionId: "11111111-1111-4111-8111-111111111111",
}),
)
const body = server.lastRequestBody()
assert.equal(objectAt(body, ["params", "temperature"]), 0.7)
assert.equal(objectAt(body, ["threadId"]), "11111111-1111-4111-8111-111111111111")
assert.equal(
server.lastRequestHeaders()["x-session-id"],
"11111111-1111-4111-8111-111111111111",
)
})
it("omits non-UUID session ids from the generate thread id", async () => {
server.mockResponse({
type: "success",
events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
})
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
await collectEvents(
streamCommandCode(makeModel(), makeContext(), {
apiKey: "mock-key",
sessionId: "human-readable-session",
}),
)
assert.equal(objectAt(server.lastRequestBody(), ["threadId"]), undefined)
assert.equal(server.lastRequestHeaders()["x-session-id"], "human-readable-session")
})
it("accepts the legacy OMP nested reasoning map", async () => {
server.mockResponse({
type: "success",
@@ -917,6 +977,63 @@ describe("streamCommandCode — upstream errors and malformed streams", () => {
assert.equal(error.error.errorMessage, "provider failed")
})
it("rejects a truncated stream without a finish event", async () => {
server.mockResponse({
type: "success",
events: [JSON.stringify({ type: "text-delta", text: "truncated" })],
})
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }),
)
const error = events.at(-1)
assert.equal(error?.type, "error")
if (error?.type !== "error") throw new Error("expected error")
assert.match(error.error.errorMessage ?? "", /no finish event/i)
})
it("maps an upstream abort event to an aborted request", async () => {
server.mockResponse({
type: "success",
events: [JSON.stringify({ type: "abort" })],
})
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }),
)
const error = events.at(-1)
assert.equal(error?.type, "error")
if (error?.type !== "error") throw new Error("expected error")
assert.equal(error.reason, "aborted")
})
it("rejects terminal upstream network failure reasons", async () => {
server.mockResponse({
type: "success",
events: [
JSON.stringify({
type: "finish",
finishReason: "stop",
rawFinishReason: "upstream_error",
}),
],
})
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }),
)
const error = events.at(-1)
assert.equal(error?.type, "error")
if (error?.type !== "error") throw new Error("expected error")
assert.match(error.error.errorMessage ?? "", /upstream connection failed/i)
})
it("handles SSE lines, malformed lines, split chunks, and final line without newline", async () => {
const textEvent = `data: ${JSON.stringify({ type: "text-delta", text: "split" })}\n`
const finishEvent = JSON.stringify({