feat(models): add vision input capabilities (#43)

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+- Add model-specific image input capabilities from the `command-code@1.15.1` catalog and forward user and tool-result images using the current Command Code wire format.
+- Update the Command Code client version header to `1.15.1`.
+
 ## 0.5.0 - 2026-08-07
 
 - Stop replaying completed assistant reasoning traces to Command Code while preserving visible text and completed tool calls in follow-up request history.

README.md

@@ -132,9 +132,9 @@ The following environment variables are intended for tests, local mocks, and com
 
 ## Image input
 
-This provider currently advertises and accepts **text input only**. The extension uses Command Code's legacy `/alpha/generate` protocol, while the public Provider API documentation describes image parts for its documented `/provider/v1` endpoints. The legacy request path has no documented image-part contract, and the model catalog fixture exposes model IDs and context lengths but no image capability or limit fields.
+The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.15.1`; unknown models default to text-only until their upstream metadata is reviewed.
 
-To avoid silently dropping or changing image data, the provider rejects image content in user messages and tool results before making a network request. It does not claim image capability or define image-size/count limits. This limitation can be revisited when Command Code documents image parts and limits for the protocol used here.
+For vision-capable models, image blocks from user messages and tool results are forwarded in Command Code's current data-URL wire format. Text-only models reject image content before making a network request instead of silently dropping it.
 
 ## Pricing display
 

index.ts

@@ -15,16 +15,12 @@ import {
 } from "@earendil-works/pi-coding-agent"
 import { join } from "node:path"
 
-import {
+import { COMMAND_CODE_CLI_VERSION, createStreamCommandCode, DEFAULT_API_BASE } from "./src/core.ts"
-  COMMAND_CODE_CLI_VERSION,
-  COMMAND_CODE_INPUT_TYPES,
-  createStreamCommandCode,
-  DEFAULT_API_BASE,
-} from "./src/core.ts"
 import { calculateCommandCodeCost } from "./src/cost.ts"
 import {
   DEFAULT_MODELS_URL,
   getModelsTimeoutMs,
+  inputModalitiesForModel,
   loadCommandCodeModels,
   thinkingMetadataForModel,
   type CommandCodeModel,
@@ -75,7 +71,7 @@ function createProviderModel(model: {
     name: model.name,
     reasoning: model.reasoning,
     ...(thinkingMetadataForModel(model.id) ?? {}),
-    input: COMMAND_CODE_INPUT_TYPES,
+    input: inputModalitiesForModel(model.id),
     cost: MODEL_COSTS[model.id] ?? ZERO_MODEL_COST,
     contextWindow: model.contextWindow,
     maxTokens: model.maxTokens,

src/converters.ts

@@ -55,26 +55,50 @@ function apiKeyFromCredentialRecord(value: unknown): string | undefined {
   return stringValue(value.key) ?? stringValue(value.access)
 }
 
-function hasImageContent(value: unknown): boolean {
+function imageParts(value: unknown): readonly Record<string, unknown>[] {
-  if (isRecord(value)) return value.type === "image"
+  if (isRecord(value)) return value.type === "image" ? [value] : []
-  return recordArray(value).some((part) => part.type === "image")
+  return recordArray(value).filter((part) => part.type === "image")
 }
 
 function imageContentError(role: string): Error {
-  return new Error(
+  return new Error(`Selected Command Code model does not support image content in ${role}`)
-    `Command Code does not support image content in ${role}; refusing to send it to avoid lossy handling`,
-  )
 }
 
 export function assertTextOnlyMessages(messages?: readonly MessageLike[]): void {
   for (const message of messages ?? []) {
-    if (hasImageContent(message.content)) {
+    if (imageParts(message.content).length > 0) {
       const role = message.role === "toolResult" ? "tool results" : `${message.role} messages`
       throw imageContentError(role)
     }
   }
 }
 
+function imageToCommandCode(part: Record<string, unknown>): Record<string, string> {
+  const data = stringValue(part.data)
+  const mimeType = stringValue(part.mimeType)
+  if (!data || !mimeType)
+    throw new Error("Invalid image content: expected base64 data and mimeType")
+
+  return {
+    type: "image",
+    image: `data:${mimeType};base64,${data}`,
+    mimeType,
+  }
+}
+
+function userContentToCommandCode(content: unknown, allowImages: boolean): unknown {
+  if (typeof content === "string") return content
+
+  return recordArray(content).flatMap((part) => {
+    if (part.type === "text") return [{ type: "text", text: stringValue(part.text) ?? "" }]
+    if (part.type === "image") {
+      if (!allowImages) throw imageContentError("user messages")
+      return [imageToCommandCode(part)]
+    }
+    return []
+  })
+}
+
 export function getApiKey(
   options: {
     env?: NodeJS.ProcessEnv
@@ -115,8 +139,6 @@ export function getApiKey(
 }
 
 export function textContent(message: { content?: unknown }): string {
-  if (hasImageContent(message.content)) throw imageContentError("tool results")
-
   return recordArray(message.content)
     .filter((part) => part.type === "text")
     .map((part) => stringValue(part.text) ?? "")
@@ -157,8 +179,12 @@ function completeToolCallIds(messages?: readonly MessageLike[]): Set<string> {
   return new Set([...callIds].filter((id) => resultIds.has(id)))
 }
 
-export function messagesToCC(messages?: readonly MessageLike[]): unknown[] {
+export function messagesToCC(
-  assertTextOnlyMessages(messages)
+  messages?: readonly MessageLike[],
+  options: { allowImages?: boolean } = {},
+): unknown[] {
+  const allowImages = options.allowImages ?? false
+  if (!allowImages) assertTextOnlyMessages(messages)
 
   const out: unknown[] = []
   const pairedToolCallIds = completeToolCallIds(messages)
@@ -167,7 +193,7 @@ export function messagesToCC(messages?: readonly MessageLike[]): unknown[] {
     if (message.role === "user") {
       out.push({
         role: "user",
-        content: typeof message.content === "string" ? message.content : message.content,
+        content: userContentToCommandCode(message.content, allowImages),
       })
     } else if (message.role === "assistant") {
       const parts: unknown[] = []
@@ -201,6 +227,15 @@ export function messagesToCC(messages?: readonly MessageLike[]): unknown[] {
           },
         ],
       })
+
+      const images = imageParts(message.content)
+      if (images.length > 0) {
+        if (!allowImages) throw imageContentError("tool results")
+        out.push({
+          role: "user",
+          content: images.map(imageToCommandCode),
+        })
+      }
     }
   }
   return out

src/core.ts

@@ -8,6 +8,7 @@
 import { randomUUID } from "node:crypto"
 
 import { commandCodeErrorMessage, redactCommandCodeErrorText } from "./overflow.ts"
+import { modelSupportsImageInput } from "./models.ts"
 import {
   getApiKey,
   getEnvironmentInfo,
@@ -42,13 +43,7 @@ export * from "./overflow.ts"
 export * from "./types.ts"
 
 export const DEFAULT_API_BASE = "https://api.commandcode.ai"
-export const COMMAND_CODE_CLI_VERSION = "0.29.0"
+export const COMMAND_CODE_CLI_VERSION = "1.15.1"
-/**
- * The legacy /alpha/generate request path used by this provider has no
- * documented image-part contract. Keep the advertised capability text-only
- * until Command Code documents and tests image handling for this endpoint.
- */
-export const COMMAND_CODE_INPUT_TYPES = ["text"] as const
 
 const DEFAULT_GENERATE_MAX_TOKENS = 64_000
 const DEFAULT_MAX_RETRIES = 0
@@ -472,7 +467,8 @@ export function createStreamCommandCode(deps: CoreDependencies) {
         const reasoningEffort = mappedReasoningEffort(model, options)
         const timeoutMs = options?.timeoutMs
 
-        assertTextOnlyMessages(context.messages)
+        const allowImages = modelSupportsImageInput(model.id)
+        if (!allowImages) assertTextOnlyMessages(context.messages)
 
         let body: unknown = {
           config: {
@@ -491,7 +487,7 @@ export function createStreamCommandCode(deps: CoreDependencies) {
           skills: null,
           params: {
             model: model.id,
-            messages: messagesToCC(context.messages),
+            messages: messagesToCC(context.messages, { allowImages }),
             tools: toolsToJson(context.tools),
             system: systemPromptToText(context.systemPrompt),
             max_tokens: generateMaxTokens(model, options),

src/models.ts

@@ -7,6 +7,63 @@ export const DEFAULT_MODELS_TIMEOUT_MS = 10_000
 const DEFAULT_MAX_OUTPUT_TOKENS = 65_536
 const MODEL_CACHE_VERSION = 1
 
+export type CommandCodeInputType = "text" | "image"
+
+/**
+ * Model input modalities from the command-code@1.15.1 bundled catalog.
+ * Models omitted here remain text-only so newly discovered IDs never claim
+ * image support without upstream evidence.
+ */
+export const MODEL_INPUT_MODALITIES: Readonly<Record<string, readonly CommandCodeInputType[]>> = {
+  "MiniMaxAI/MiniMax-M3": ["text", "image"],
+  "Qwen/Qwen3.6-Plus": ["text", "image"],
+  "Qwen/Qwen3.7-Flash": ["text", "image"],
+  "Qwen/Qwen3.7-Plus": ["text", "image"],
+  "Qwen/Qwen3.8-Max": ["text", "image"],
+  "claude-fable-5": ["text", "image"],
+  "claude-haiku-4-5-20251001": ["text", "image"],
+  "claude-opus-4-7": ["text", "image"],
+  "claude-opus-4-8": ["text", "image"],
+  "claude-opus-5": ["text", "image"],
+  "claude-sonnet-4-6": ["text", "image"],
+  "claude-sonnet-5": ["text", "image"],
+  "google/gemini-3.1-flash-lite": ["text", "image"],
+  "google/gemini-3.5-flash": ["text", "image"],
+  "google/gemini-3.5-flash-lite": ["text", "image"],
+  "google/gemini-3.6-flash": ["text", "image"],
+  "gpt-5.3-codex": ["text", "image"],
+  "gpt-5.4": ["text", "image"],
+  "gpt-5.4-mini": ["text", "image"],
+  "gpt-5.5": ["text", "image"],
+  "gpt-5.6-luna": ["text", "image"],
+  "gpt-5.6-sol": ["text", "image"],
+  "gpt-5.6-terra": ["text", "image"],
+  "meta/muse-spark-1.1": ["text", "image"],
+  "meta/muse-spark-1.2": ["text", "image"],
+  "meta/muse-spark-1.2-contributor": ["text", "image"],
+  "moonshotai/Kimi-K2.5": ["text", "image"],
+  "moonshotai/Kimi-K2.6": ["text", "image"],
+  "moonshotai/Kimi-K2.7-Code": ["text", "image"],
+  "moonshotai/Kimi-K2.7-Code-Highspeed": ["text", "image"],
+  "moonshotai/Kimi-K3": ["text", "image"],
+  "sakana/fugu-ultra": ["text", "image"],
+  "stepfun/Step-3.7-Flash": ["text", "image"],
+  "thinkingmachines/inkling": ["text", "image"],
+  "thinkingmachines/inkling-small": ["text", "image"],
+  "xai/grok-4.5": ["text", "image"],
+  "xiaomi/mimo-v2.5": ["text", "image"],
+}
+
+const TEXT_INPUT_ONLY = ["text"] as const
+
+export function inputModalitiesForModel(modelId: string): readonly CommandCodeInputType[] {
+  return MODEL_INPUT_MODALITIES[modelId] ?? TEXT_INPUT_ONLY
+}
+
+export function modelSupportsImageInput(modelId: string): boolean {
+  return inputModalitiesForModel(modelId).includes("image")
+}
+
 export type PiThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"
 
 type CommandCodeReasoningEffort = Exclude<PiThinkingLevel, "off">
@@ -15,7 +72,7 @@ type CommandCodeReasoningEffort = Exclude<PiThinkingLevel, "off">
  * Per-model reasoning efforts supported by Command Code's generate endpoint.
  *
  * The Provider API does not expose reasoning metadata. This is an exact
- * snapshot of `reasoningEfforts` from the command-code@1.14.1 model catalog
+ * snapshot of `reasoningEfforts` from the command-code@1.15.1 model catalog
  * (`packages/shared/src/model-catalog.ts`, also published in the generated
  * `dist/bundled/command-code-knowledge/reference/models.md`). Models omitted
  * here let Command Code choose their reasoning depth, matching the CLI.

tests/test-models.ts

@@ -9,8 +9,11 @@ import {
   commandCodeModelsFromCache,
   DEFAULT_MODELS_TIMEOUT_MS,
   getModelsTimeoutMs,
+  inputModalitiesForModel,
   loadCommandCodeModels,
   MODEL_EFFORTS,
+  MODEL_INPUT_MODALITIES,
+  modelSupportsImageInput,
   thinkingLevelMapForEfforts,
   thinkingMetadataForModel,
   type CommandCodeModel,
@@ -81,6 +84,16 @@ describe("commandCodeModelsFromApiResponse()", () => {
     assert.deepEqual(commandCodeModelsFromApiResponse(API_RESPONSE), EXPECTED_MODELS)
   })
 
+  it("matches command-code@1.15.1 image input capabilities", () => {
+    assert.deepEqual(inputModalitiesForModel("gpt-5.6-luna"), ["text", "image"])
+    assert.deepEqual(inputModalitiesForModel("meta/muse-spark-1.2"), ["text", "image"])
+    assert.deepEqual(inputModalitiesForModel("deepseek/deepseek-v4-pro"), ["text"])
+    assert.deepEqual(inputModalitiesForModel("unknown-new-model"), ["text"])
+    assert.equal(modelSupportsImageInput("gpt-5.6-luna"), true)
+    assert.equal(modelSupportsImageInput("deepseek/deepseek-v4-pro"), false)
+    assert.equal(Object.keys(MODEL_INPUT_MODALITIES).length, 37)
+  })
+
   it("marks only known reasoning models as reasoning-capable", () => {
     const models = commandCodeModelsFromApiResponse({
       object: "list",
@@ -94,7 +107,7 @@ describe("commandCodeModelsFromApiResponse()", () => {
     assert.equal(models[1]?.reasoning, false)
   })
 
-  it("matches the exact command-code@1.14.1 reasoning effort catalog", () => {
+  it("matches the exact command-code@1.15.1 reasoning effort catalog", () => {
     assert.deepEqual(MODEL_EFFORTS, {
       "Qwen/Qwen3.8-Max": ["low", "medium", "xhigh"],
       "claude-fable-5": ["low", "medium", "high", "xhigh", "max"],

tests/test-pure-functions.ts

@@ -11,7 +11,6 @@ import { describe, it } from "node:test"
 
 import {
   assertTextOnlyMessages,
-  COMMAND_CODE_INPUT_TYPES,
   getApiKey,
   getEnvironmentInfo,
   mapFinishReason,
@@ -118,11 +117,7 @@ describe("projectSlugFromPath()", () => {
 })
 
 describe("text-only image handling", () => {
-  it("does not advertise image input capability", () => {
+  it("rejects image content for models without image support", () => {
-    assert.deepEqual(COMMAND_CODE_INPUT_TYPES, ["text"])
-  })
-
-  it("rejects image content instead of dropping it", () => {
     assert.throws(
       () =>
         assertTextOnlyMessages([
@@ -131,7 +126,7 @@ describe("text-only image handling", () => {
             content: [{ type: "image", data: "base64-data", mimeType: "image/png" }],
           },
         ]),
-      /does not support image content.*refusing to send it/i,
+      /does not support image content/i,
     )
     assert.throws(
       () =>
@@ -142,7 +137,7 @@ describe("text-only image handling", () => {
             content: [{ type: "image", data: "base64-data", mimeType: "image/png" }],
           },
         ]),
-      /does not support image content.*refusing to send it/i,
+      /does not support image content/i,
     )
   })
 })
@@ -160,17 +155,16 @@ describe("textContent()", () => {
     )
   })
 
-  it("rejects mixed text and image content instead of dropping the image", () => {
+  it("extracts text while images are handled separately", () => {
-    assert.throws(
+    assert.equal(
-      () =>
+      textContent({
-        textContent({
+        content: [
-          content: [
+          { type: "text", text: "hello" },
-            { type: "text", text: "hello" },
+          { type: "image", data: "x", mimeType: "image/png" },
-            { type: "image", data: "x", mimeType: "image/png" },
+          { type: "text", text: "world" },
-            { type: "text", text: "world" },
+        ],
-          ],
+      }),
-        }),
+      "hello\nworld",
-      /does not support image content.*refusing to send it/i,
     )
   })
 
@@ -506,6 +500,71 @@ describe("messagesToCC()", () => {
     assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "hello\nworld")
   })
 
+  it("serializes image inputs in the current Command Code wire format", () => {
+    assert.deepEqual(
+      messagesToCC(
+        [
+          {
+            role: "user",
+            content: [
+              { type: "text", text: "inspect this" },
+              { type: "image", data: "aGVsbG8=", mimeType: "image/png" },
+            ],
+          },
+        ],
+        { allowImages: true },
+      ),
+      [
+        {
+          role: "user",
+          content: [
+            { type: "text", text: "inspect this" },
+            {
+              type: "image",
+              image: "data:image/png;base64,aGVsbG8=",
+              mimeType: "image/png",
+            },
+          ],
+        },
+      ],
+    )
+  })
+
+  it("preserves tool-result images as a following user image message", () => {
+    const result = messagesToCC(
+      [
+        { role: "user", content: "read image" },
+        {
+          role: "assistant",
+          content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }],
+        },
+        {
+          role: "toolResult",
+          toolCallId: "c1",
+          toolName: "read",
+          content: [
+            { type: "text", text: "image attached" },
+            { type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" },
+          ],
+        },
+      ],
+      { allowImages: true },
+    )
+
+    assert.equal(objectAt(result, ["2", "role"]), "tool")
+    assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "image attached")
+    assert.deepEqual(objectAt(result, ["3"]), {
+      role: "user",
+      content: [
+        {
+          type: "image",
+          image: "data:image/jpeg;base64,aGVsbG8=",
+          mimeType: "image/jpeg",
+        },
+      ],
+    })
+  })
+
   it("drops previous assistant reasoning while preserving text and tool calls", () => {
     const result = messagesToCC([
       { role: "user", content: "first question" },

tests/test-stream.ts

@@ -142,6 +142,59 @@ describe("streamCommandCode — successful streams", () => {
     assert.equal(calculatedUsages.length, 1)
   })
 
+  it("sends images for vision-capable models", async () => {
+    server.mockResponse({
+      type: "success",
+      events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
+    })
+    const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
+
+    await collectEvents(
+      streamCommandCode(
+        makeModel({ id: "gpt-5.6-luna" }),
+        makeContext({
+          messages: [
+            {
+              role: "user",
+              content: [
+                { type: "text", text: "inspect" },
+                { type: "image", data: "aGVsbG8=", mimeType: "image/png" },
+              ],
+            },
+          ],
+        }),
+        { apiKey: "mock-key" },
+      ),
+    )
+
+    assert.equal(
+      objectAt(server.lastRequestBody(), ["params", "messages", "0", "content", "1", "image"]),
+      "data:image/png;base64,aGVsbG8=",
+    )
+  })
+
+  it("rejects images before network access for text-only models", async () => {
+    const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
+
+    const events = await collectEvents(
+      streamCommandCode(
+        makeModel({ id: "deepseek/deepseek-v4-pro" }),
+        makeContext({
+          messages: [
+            {
+              role: "user",
+              content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }],
+            },
+          ],
+        }),
+        { apiKey: "mock-key" },
+      ),
+    )
+
+    assert.equal(events.at(-1)?.type, "error")
+    assert.equal(server.requestCount(), 0)
+  })
+
   it("derives uncached input when noCacheTokens is missing", async () => {
     server.mockResponse({
       type: "success",
@@ -375,10 +428,7 @@ describe("streamCommandCode — request serialization", () => {
     const lastEvent = events.at(-1)
     assert.equal(lastEvent?.type, "error")
     if (lastEvent?.type === "error") {
-      assert.match(
+      assert.match(lastEvent.error.errorMessage ?? "", /does not support image content/i)
-        lastEvent.error.errorMessage ?? "",
-        /does not support image content.*refusing to send it/i,
-      )
     }
     assert.equal(server.requestCount(), 0)
   })
@@ -438,7 +488,7 @@ describe("streamCommandCode — request serialization", () => {
 
     const headers = server.lastRequestHeaders()
     assert.equal(headers.authorization, "Bearer mock-key")
-    assert.equal(headers["x-command-code-version"], "0.29.0")
+    assert.equal(headers["x-command-code-version"], "1.15.1")
     assert.equal(headers["x-project-slug"], "repo")
     assert.equal(headers["x-taste-learning"], "true")
     assert.equal(headers["x-co-flag"], "false")