cat-shark/pi-commandcode-provider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(models): add vision input capabilities (#43)

Browse Source
This commit is contained in:
Patrick Wozniak
2026-08-11 00:28:03 +02:00
committed by GitHub
parent 4b6e1cd85a
commit a367aed63c
9 changed files with 265 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+3
View File
@@ -2,6 +2,9 @@
## Unreleased
- Add model-specific image input capabilities from the `command-code@1.15.1` catalog and forward user and tool-result images using the current Command Code wire format.
- Update the Command Code client version header to `1.15.1`.
## 0.5.0 - 2026-08-07
- Stop replaying completed assistant reasoning traces to Command Code while preserving visible text and completed tool calls in follow-up request history.
README.md
+2 -2
View File
@@ -132,9 +132,9 @@ The following environment variables are intended for tests, local mocks, and com
## Image input
This provider currently advertises and accepts **text input only**. The extension uses Command Code's legacy `/alpha/generate` protocol, while the public Provider API documentation describes image parts for its documented `/provider/v1` endpoints. The legacy request path has no documented image-part contract, and the model catalog fixture exposes model IDs and context lengths but no image capability or limit fields.
The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.15.1`; unknown models default to text-only until their upstream metadata is reviewed.
To avoid silently dropping or changing image data, the provider rejects image content in user messages and tool results before making a network request. It does not claim image capability or define image-size/count limits. This limitation can be revisited when Command Code documents image parts and limits for the protocol used here.
For vision-capable models, image blocks from user messages and tool results are forwarded in Command Code's current data-URL wire format. Text-only models reject image content before making a network request instead of silently dropping it.
## Pricing display
index.ts
+3 -7
View File
@@ -15,16 +15,12 @@ import {
} from "@earendil-works/pi-coding-agent"
import { join } from "node:path"
import {
COMMAND_CODE_CLI_VERSION,
COMMAND_CODE_INPUT_TYPES,
createStreamCommandCode,
DEFAULT_API_BASE,
} from "./src/core.ts"
import { COMMAND_CODE_CLI_VERSION, createStreamCommandCode, DEFAULT_API_BASE } from "./src/core.ts"
import { calculateCommandCodeCost } from "./src/cost.ts"
import {
DEFAULT_MODELS_URL,
getModelsTimeoutMs,
inputModalitiesForModel,
loadCommandCodeModels,
thinkingMetadataForModel,
type CommandCodeModel,
@@ -75,7 +71,7 @@ function createProviderModel(model: {
name: model.name,
reasoning: model.reasoning,
...(thinkingMetadataForModel(model.id) ?? {}),
input: COMMAND_CODE_INPUT_TYPES,
input: inputModalitiesForModel(model.id),
cost: MODEL_COSTS[model.id] ?? ZERO_MODEL_COST,
contextWindow: model.contextWindow,
maxTokens: model.maxTokens,
src/converters.ts
+47 -12
View File
@@ -55,26 +55,50 @@ function apiKeyFromCredentialRecord(value: unknown): string | undefined {
return stringValue(value.key) ?? stringValue(value.access)
}
function hasImageContent(value: unknown): boolean {
if (isRecord(value)) return value.type === "image"
return recordArray(value).some((part) => part.type === "image")
function imageParts(value: unknown): readonly Record<string, unknown>[] {
if (isRecord(value)) return value.type === "image" ? [value] : []
return recordArray(value).filter((part) => part.type === "image")
}
function imageContentError(role: string): Error {
return new Error(
`Command Code does not support image content in ${role}; refusing to send it to avoid lossy handling`,
)
return new Error(`Selected Command Code model does not support image content in ${role}`)
}
export function assertTextOnlyMessages(messages?: readonly MessageLike[]): void {
for (const message of messages ?? []) {
if (hasImageContent(message.content)) {
if (imageParts(message.content).length > 0) {
const role = message.role === "toolResult" ? "tool results" : `${message.role} messages`
throw imageContentError(role)
}
}
}
function imageToCommandCode(part: Record<string, unknown>): Record<string, string> {
const data = stringValue(part.data)
const mimeType = stringValue(part.mimeType)
if (!data || !mimeType)
throw new Error("Invalid image content: expected base64 data and mimeType")
return {
type: "image",
image: `data:${mimeType};base64,${data}`,
mimeType,
}
}
function userContentToCommandCode(content: unknown, allowImages: boolean): unknown {
if (typeof content === "string") return content
return recordArray(content).flatMap((part) => {
if (part.type === "text") return [{ type: "text", text: stringValue(part.text) ?? "" }]
if (part.type === "image") {
if (!allowImages) throw imageContentError("user messages")
return [imageToCommandCode(part)]
}
return []
})
}
export function getApiKey(
options: {
env?: NodeJS.ProcessEnv
@@ -115,8 +139,6 @@ export function getApiKey(
}
export function textContent(message: { content?: unknown }): string {
if (hasImageContent(message.content)) throw imageContentError("tool results")
return recordArray(message.content)
.filter((part) => part.type === "text")
.map((part) => stringValue(part.text) ?? "")
@@ -157,8 +179,12 @@ function completeToolCallIds(messages?: readonly MessageLike[]): Set<string> {
return new Set([...callIds].filter((id) => resultIds.has(id)))
}
export function messagesToCC(messages?: readonly MessageLike[]): unknown[] {
assertTextOnlyMessages(messages)
export function messagesToCC(
messages?: readonly MessageLike[],
options: { allowImages?: boolean } = {},
): unknown[] {
const allowImages = options.allowImages ?? false
if (!allowImages) assertTextOnlyMessages(messages)
const out: unknown[] = []
const pairedToolCallIds = completeToolCallIds(messages)
@@ -167,7 +193,7 @@ export function messagesToCC(messages?: readonly MessageLike[]): unknown[] {
if (message.role === "user") {
out.push({
role: "user",
content: typeof message.content === "string" ? message.content : message.content,
content: userContentToCommandCode(message.content, allowImages),
})
} else if (message.role === "assistant") {
const parts: unknown[] = []
@@ -201,6 +227,15 @@ export function messagesToCC(messages?: readonly MessageLike[]): unknown[] {
},
],
})
const images = imageParts(message.content)
if (images.length > 0) {
if (!allowImages) throw imageContentError("tool results")
out.push({
role: "user",
content: images.map(imageToCommandCode),
})
}
}
}
return out
src/core.ts
+5 -9
View File
@@ -8,6 +8,7 @@
import { randomUUID } from "node:crypto"
import { commandCodeErrorMessage, redactCommandCodeErrorText } from "./overflow.ts"
import { modelSupportsImageInput } from "./models.ts"
import {
getApiKey,
getEnvironmentInfo,
@@ -42,13 +43,7 @@ export * from "./overflow.ts"
export * from "./types.ts"
export const DEFAULT_API_BASE = "https://api.commandcode.ai"
export const COMMAND_CODE_CLI_VERSION = "0.29.0"
/**
* The legacy /alpha/generate request path used by this provider has no
* documented image-part contract. Keep the advertised capability text-only
* until Command Code documents and tests image handling for this endpoint.
*/
export const COMMAND_CODE_INPUT_TYPES = ["text"] as const
export const COMMAND_CODE_CLI_VERSION = "1.15.1"
const DEFAULT_GENERATE_MAX_TOKENS = 64_000
const DEFAULT_MAX_RETRIES = 0
@@ -472,7 +467,8 @@ export function createStreamCommandCode(deps: CoreDependencies) {
const reasoningEffort = mappedReasoningEffort(model, options)
const timeoutMs = options?.timeoutMs
assertTextOnlyMessages(context.messages)
const allowImages = modelSupportsImageInput(model.id)
if (!allowImages) assertTextOnlyMessages(context.messages)
let body: unknown = {
config: {
@@ -491,7 +487,7 @@ export function createStreamCommandCode(deps: CoreDependencies) {
skills: null,
params: {
model: model.id,
messages: messagesToCC(context.messages),
messages: messagesToCC(context.messages, { allowImages }),
tools: toolsToJson(context.tools),
system: systemPromptToText(context.systemPrompt),
max_tokens: generateMaxTokens(model, options),
src/models.ts
+58 -1
View File
@@ -7,6 +7,63 @@ export const DEFAULT_MODELS_TIMEOUT_MS = 10_000
const DEFAULT_MAX_OUTPUT_TOKENS = 65_536
const MODEL_CACHE_VERSION = 1
export type CommandCodeInputType = "text" | "image"
/**
* Model input modalities from the command-code@1.15.1 bundled catalog.
* Models omitted here remain text-only so newly discovered IDs never claim
* image support without upstream evidence.
*/
export const MODEL_INPUT_MODALITIES: Readonly<Record<string, readonly CommandCodeInputType[]>> = {
"MiniMaxAI/MiniMax-M3": ["text", "image"],
"Qwen/Qwen3.6-Plus": ["text", "image"],
"Qwen/Qwen3.7-Flash": ["text", "image"],
"Qwen/Qwen3.7-Plus": ["text", "image"],
"Qwen/Qwen3.8-Max": ["text", "image"],
"claude-fable-5": ["text", "image"],
"claude-haiku-4-5-20251001": ["text", "image"],
"claude-opus-4-7": ["text", "image"],
"claude-opus-4-8": ["text", "image"],
"claude-opus-5": ["text", "image"],
"claude-sonnet-4-6": ["text", "image"],
"claude-sonnet-5": ["text", "image"],
"google/gemini-3.1-flash-lite": ["text", "image"],
"google/gemini-3.5-flash": ["text", "image"],
"google/gemini-3.5-flash-lite": ["text", "image"],
"google/gemini-3.6-flash": ["text", "image"],
"gpt-5.3-codex": ["text", "image"],
"gpt-5.4": ["text", "image"],
"gpt-5.4-mini": ["text", "image"],
"gpt-5.5": ["text", "image"],
"gpt-5.6-luna": ["text", "image"],
"gpt-5.6-sol": ["text", "image"],
"gpt-5.6-terra": ["text", "image"],
"meta/muse-spark-1.1": ["text", "image"],
"meta/muse-spark-1.2": ["text", "image"],
"meta/muse-spark-1.2-contributor": ["text", "image"],
"moonshotai/Kimi-K2.5": ["text", "image"],
"moonshotai/Kimi-K2.6": ["text", "image"],
"moonshotai/Kimi-K2.7-Code": ["text", "image"],
"moonshotai/Kimi-K2.7-Code-Highspeed": ["text", "image"],
"moonshotai/Kimi-K3": ["text", "image"],
"sakana/fugu-ultra": ["text", "image"],
"stepfun/Step-3.7-Flash": ["text", "image"],
"thinkingmachines/inkling": ["text", "image"],
"thinkingmachines/inkling-small": ["text", "image"],
"xai/grok-4.5": ["text", "image"],
"xiaomi/mimo-v2.5": ["text", "image"],
}
const TEXT_INPUT_ONLY = ["text"] as const
export function inputModalitiesForModel(modelId: string): readonly CommandCodeInputType[] {
return MODEL_INPUT_MODALITIES[modelId] ?? TEXT_INPUT_ONLY
}
export function modelSupportsImageInput(modelId: string): boolean {
return inputModalitiesForModel(modelId).includes("image")
}
export type PiThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"
type CommandCodeReasoningEffort = Exclude<PiThinkingLevel, "off">
@@ -15,7 +72,7 @@ type CommandCodeReasoningEffort = Exclude<PiThinkingLevel, "off">
* Per-model reasoning efforts supported by Command Code's generate endpoint.
*
* The Provider API does not expose reasoning metadata. This is an exact
* snapshot of `reasoningEfforts` from the command-code@1.14.1 model catalog
* snapshot of `reasoningEfforts` from the command-code@1.15.1 model catalog
* (`packages/shared/src/model-catalog.ts`, also published in the generated
* `dist/bundled/command-code-knowledge/reference/models.md`). Models omitted
* here let Command Code choose their reasoning depth, matching the CLI.
tests/test-models.ts
+14 -1
View File
@@ -9,8 +9,11 @@ import {
commandCodeModelsFromCache,
DEFAULT_MODELS_TIMEOUT_MS,
getModelsTimeoutMs,
inputModalitiesForModel,
loadCommandCodeModels,
MODEL_EFFORTS,
MODEL_INPUT_MODALITIES,
modelSupportsImageInput,
thinkingLevelMapForEfforts,
thinkingMetadataForModel,
type CommandCodeModel,
@@ -81,6 +84,16 @@ describe("commandCodeModelsFromApiResponse()", () => {
assert.deepEqual(commandCodeModelsFromApiResponse(API_RESPONSE), EXPECTED_MODELS)
})
it("matches command-code@1.15.1 image input capabilities", () => {
assert.deepEqual(inputModalitiesForModel("gpt-5.6-luna"), ["text", "image"])
assert.deepEqual(inputModalitiesForModel("meta/muse-spark-1.2"), ["text", "image"])
assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-pro"), ["text"])
assert.deepEqual(inputModalitiesForModel("unknown-new-model"), ["text"])
assert.equal(modelSupportsImageInput("gpt-5.6-luna"), true)
assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-pro"), false)
assert.equal(Object.keys(MODEL_INPUT_MODALITIES).length, 37)
})
it("marks only known reasoning models as reasoning-capable", () => {
const models = commandCodeModelsFromApiResponse({
object: "list",
@@ -94,7 +107,7 @@ describe("commandCodeModelsFromApiResponse()", () => {
assert.equal(models[1]?.reasoning, false)
})
it("matches the exact command-code@1.14.1 reasoning effort catalog", () => {
it("matches the exact command-code@1.15.1 reasoning effort catalog", () => {
assert.deepEqual(MODEL_EFFORTS, {
"Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"],
"claude-fable-5": ["low", "medium", "high", "xhigh", "max"],
tests/test-pure-functions.ts
+78 -19
View File
@@ -11,7 +11,6 @@ import { describe, it } from "node:test"
import {
assertTextOnlyMessages,
COMMAND_CODE_INPUT_TYPES,
getApiKey,
getEnvironmentInfo,
mapFinishReason,
@@ -118,11 +117,7 @@ describe("projectSlugFromPath()", () => {
})
describe("text-only image handling", () => {
it("does not advertise image input capability", () => {
assert.deepEqual(COMMAND_CODE_INPUT_TYPES, ["text"])
})
it("rejects image content instead of dropping it", () => {
it("rejects image content for models without image support", () => {
assert.throws(
() =>
assertTextOnlyMessages([
@@ -131,7 +126,7 @@ describe("text-only image handling", () => {
content: [{ type: "image", data: "base64-data", mimeType: "image/png" }],
},
]),
/does not support image content.*refusing to send it/i,
/does not support image content/i,
)
assert.throws(
() =>
@@ -142,7 +137,7 @@ describe("text-only image handling", () => {
content: [{ type: "image", data: "base64-data", mimeType: "image/png" }],
},
]),
/does not support image content.*refusing to send it/i,
/does not support image content/i,
)
})
})
@@ -160,17 +155,16 @@ describe("textContent()", () => {
)
})
it("rejects mixed text and image content instead of dropping the image", () => {
assert.throws(
() =>
textContent({
content: [
{ type: "text", text: "hello" },
{ type: "image", data: "x", mimeType: "image/png" },
{ type: "text", text: "world" },
],
}),
/does not support image content.*refusing to send it/i,
it("extracts text while images are handled separately", () => {
assert.equal(
textContent({
content: [
{ type: "text", text: "hello" },
{ type: "image", data: "x", mimeType: "image/png" },
{ type: "text", text: "world" },
],
}),
"hello\nworld",
)
})
@@ -506,6 +500,71 @@ describe("messagesToCC()", () => {
assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "hello\nworld")
})
it("serializes image inputs in the current Command Code wire format", () => {
assert.deepEqual(
messagesToCC(
[
{
role: "user",
content: [
{ type: "text", text: "inspect this" },
{ type: "image", data: "aGVsbG8=", mimeType: "image/png" },
],
},
],
{ allowImages: true },
),
[
{
role: "user",
content: [
{ type: "text", text: "inspect this" },
{
type: "image",
image: "data:image/png;base64,aGVsbG8=",
mimeType: "image/png",
},
],
},
],
)
})
it("preserves tool-result images as a following user image message", () => {
const result = messagesToCC(
[
{ role: "user", content: "read image" },
{
role: "assistant",
content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }],
},
{
role: "toolResult",
toolCallId: "c1",
toolName: "read",
content: [
{ type: "text", text: "image attached" },
{ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" },
],
},
],
{ allowImages: true },
)
assert.equal(objectAt(result, ["2", "role"]), "tool")
assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "image attached")
assert.deepEqual(objectAt(result, ["3"]), {
role: "user",
content: [
{
type: "image",
image: "data:image/jpeg;base64,aGVsbG8=",
mimeType: "image/jpeg",
},
],
})
})
it("drops previous assistant reasoning while preserving text and tool calls", () => {
const result = messagesToCC([
{ role: "user", content: "first question" },
tests/test-stream.ts
+55 -5
View File
@@ -142,6 +142,59 @@ describe("streamCommandCode — successful streams", () => {
assert.equal(calculatedUsages.length, 1)
})
it("sends images for vision-capable models", async () => {
server.mockResponse({
type: "success",
events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
})
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
await collectEvents(
streamCommandCode(
makeModel({ id: "gpt-5.6-luna" }),
makeContext({
messages: [
{
role: "user",
content: [
{ type: "text", text: "inspect" },
{ type: "image", data: "aGVsbG8=", mimeType: "image/png" },
],
},
],
}),
{ apiKey: "mock-key" },
),
)
assert.equal(
objectAt(server.lastRequestBody(), ["params", "messages", "0", "content", "1", "image"]),
"data:image/png;base64,aGVsbG8=",
)
})
it("rejects images before network access for text-only models", async () => {
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(
makeModel({ id: "deepseek/deepseek-v4-pro" }),
makeContext({
messages: [
{
role: "user",
content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }],
},
],
}),
{ apiKey: "mock-key" },
),
)
assert.equal(events.at(-1)?.type, "error")
assert.equal(server.requestCount(), 0)
})
it("derives uncached input when noCacheTokens is missing", async () => {
server.mockResponse({
type: "success",
@@ -375,10 +428,7 @@ describe("streamCommandCode — request serialization", () => {
const lastEvent = events.at(-1)
assert.equal(lastEvent?.type, "error")
if (lastEvent?.type === "error") {
assert.match(
lastEvent.error.errorMessage ?? "",
/does not support image content.*refusing to send it/i,
)
assert.match(lastEvent.error.errorMessage ?? "", /does not support image content/i)
}
assert.equal(server.requestCount(), 0)
})
@@ -438,7 +488,7 @@ describe("streamCommandCode — request serialization", () => {
const headers = server.lastRequestHeaders()
assert.equal(headers.authorization, "Bearer mock-key")
assert.equal(headers["x-command-code-version"], "0.29.0")
assert.equal(headers["x-command-code-version"], "1.15.1")
assert.equal(headers["x-project-slug"], "repo")
assert.equal(headers["x-taste-learning"], "true")
assert.equal(headers["x-co-flag"], "false")