feat(models): add vision input capabilities (#43)

2026-08-11 00:28:03 +02:00
parent 4b6e1cd85a
commit a367aed63c
9 changed files with 265 additions and 56 deletions
@@ -55,26 +55,50 @@ function apiKeyFromCredentialRecord(value: unknown): string | undefined {
  return stringValue(value.key) ?? stringValue(value.access)
 }

-function hasImageContent(value: unknown): boolean {
-  if (isRecord(value)) return value.type === "image"
-  return recordArray(value).some((part) => part.type === "image")
+function imageParts(value: unknown): readonly Record<string, unknown>[] {
+  if (isRecord(value)) return value.type === "image" ? [value] : []
+  return recordArray(value).filter((part) => part.type === "image")
 }

 function imageContentError(role: string): Error {
-  return new Error(
-    `Command Code does not support image content in ${role}; refusing to send it to avoid lossy handling`,
-  )
+  return new Error(`Selected Command Code model does not support image content in ${role}`)
 }

 export function assertTextOnlyMessages(messages?: readonly MessageLike[]): void {
  for (const message of messages ?? []) {
-    if (hasImageContent(message.content)) {
+    if (imageParts(message.content).length > 0) {
      const role = message.role === "toolResult" ? "tool results" : `${message.role} messages`
      throw imageContentError(role)
    }
  }
 }

+function imageToCommandCode(part: Record<string, unknown>): Record<string, string> {
+  const data = stringValue(part.data)
+  const mimeType = stringValue(part.mimeType)
+  if (!data || !mimeType)
+    throw new Error("Invalid image content: expected base64 data and mimeType")
+
+  return {
+    type: "image",
+    image: `data:${mimeType};base64,${data}`,
+    mimeType,
+  }
+}
+
+function userContentToCommandCode(content: unknown, allowImages: boolean): unknown {
+  if (typeof content === "string") return content
+
+  return recordArray(content).flatMap((part) => {
+    if (part.type === "text") return [{ type: "text", text: stringValue(part.text) ?? "" }]
+    if (part.type === "image") {
+      if (!allowImages) throw imageContentError("user messages")
+      return [imageToCommandCode(part)]
+    }
+    return []
+  })
+}
+
 export function getApiKey(
  options: {
    env?: NodeJS.ProcessEnv
@@ -115,8 +139,6 @@ export function getApiKey(
 }

 export function textContent(message: { content?: unknown }): string {
-  if (hasImageContent(message.content)) throw imageContentError("tool results")
-
  return recordArray(message.content)
    .filter((part) => part.type === "text")
    .map((part) => stringValue(part.text) ?? "")
@@ -157,8 +179,12 @@ function completeToolCallIds(messages?: readonly MessageLike[]): Set<string> {
  return new Set([...callIds].filter((id) => resultIds.has(id)))
 }

-export function messagesToCC(messages?: readonly MessageLike[]): unknown[] {
-  assertTextOnlyMessages(messages)
+export function messagesToCC(
+  messages?: readonly MessageLike[],
+  options: { allowImages?: boolean } = {},
+): unknown[] {
+  const allowImages = options.allowImages ?? false
+  if (!allowImages) assertTextOnlyMessages(messages)

  const out: unknown[] = []
  const pairedToolCallIds = completeToolCallIds(messages)
@@ -167,7 +193,7 @@ export function messagesToCC(messages?: readonly MessageLike[]): unknown[] {
    if (message.role === "user") {
      out.push({
        role: "user",
-        content: typeof message.content === "string" ? message.content : message.content,
+        content: userContentToCommandCode(message.content, allowImages),
      })
    } else if (message.role === "assistant") {
      const parts: unknown[] = []
@@ -201,6 +227,15 @@ export function messagesToCC(messages?: readonly MessageLike[]): unknown[] {
          },
        ],
      })
+
+      const images = imageParts(message.content)
+      if (images.length > 0) {
+        if (!allowImages) throw imageContentError("tool results")
+        out.push({
+          role: "user",
+          content: images.map(imageToCommandCode),
+        })
+      }
    }
  }
  return out
@@ -8,6 +8,7 @@
 import { randomUUID } from "node:crypto"

 import { commandCodeErrorMessage, redactCommandCodeErrorText } from "./overflow.ts"
+import { modelSupportsImageInput } from "./models.ts"
 import {
  getApiKey,
  getEnvironmentInfo,
@@ -42,13 +43,7 @@ export * from "./overflow.ts"
 export * from "./types.ts"

 export const DEFAULT_API_BASE = "https://api.commandcode.ai"
-export const COMMAND_CODE_CLI_VERSION = "0.29.0"
-/**
- * The legacy /alpha/generate request path used by this provider has no
- * documented image-part contract. Keep the advertised capability text-only
- * until Command Code documents and tests image handling for this endpoint.
- */
-export const COMMAND_CODE_INPUT_TYPES = ["text"] as const
+export const COMMAND_CODE_CLI_VERSION = "1.15.1"

 const DEFAULT_GENERATE_MAX_TOKENS = 64_000
 const DEFAULT_MAX_RETRIES = 0
@@ -472,7 +467,8 @@ export function createStreamCommandCode(deps: CoreDependencies) {
        const reasoningEffort = mappedReasoningEffort(model, options)
        const timeoutMs = options?.timeoutMs

-        assertTextOnlyMessages(context.messages)
+        const allowImages = modelSupportsImageInput(model.id)
+        if (!allowImages) assertTextOnlyMessages(context.messages)

        let body: unknown = {
          config: {
@@ -491,7 +487,7 @@ export function createStreamCommandCode(deps: CoreDependencies) {
          skills: null,
          params: {
            model: model.id,
-            messages: messagesToCC(context.messages),
+            messages: messagesToCC(context.messages, { allowImages }),
            tools: toolsToJson(context.tools),
            system: systemPromptToText(context.systemPrompt),
            max_tokens: generateMaxTokens(model, options),
@@ -7,6 +7,63 @@ export const DEFAULT_MODELS_TIMEOUT_MS = 10_000
 const DEFAULT_MAX_OUTPUT_TOKENS = 65_536
 const MODEL_CACHE_VERSION = 1

+export type CommandCodeInputType = "text" | "image"
+
+/**
+ * Model input modalities from the command-code@1.15.1 bundled catalog.
+ * Models omitted here remain text-only so newly discovered IDs never claim
+ * image support without upstream evidence.
+ */
+export const MODEL_INPUT_MODALITIES: Readonly<Record<string, readonly CommandCodeInputType[]>> = {
+  "MiniMaxAI/MiniMax-M3": ["text", "image"],
+  "Qwen/Qwen3.6-Plus": ["text", "image"],
+  "Qwen/Qwen3.7-Flash": ["text", "image"],
+  "Qwen/Qwen3.7-Plus": ["text", "image"],
+  "Qwen/Qwen3.8-Max": ["text", "image"],
+  "claude-fable-5": ["text", "image"],
+  "claude-haiku-4-5-20251001": ["text", "image"],
+  "claude-opus-4-7": ["text", "image"],
+  "claude-opus-4-8": ["text", "image"],
+  "claude-opus-5": ["text", "image"],
+  "claude-sonnet-4-6": ["text", "image"],
+  "claude-sonnet-5": ["text", "image"],
+  "google/gemini-3.1-flash-lite": ["text", "image"],
+  "google/gemini-3.5-flash": ["text", "image"],
+  "google/gemini-3.5-flash-lite": ["text", "image"],
+  "google/gemini-3.6-flash": ["text", "image"],
+  "gpt-5.3-codex": ["text", "image"],
+  "gpt-5.4": ["text", "image"],
+  "gpt-5.4-mini": ["text", "image"],
+  "gpt-5.5": ["text", "image"],
+  "gpt-5.6-luna": ["text", "image"],
+  "gpt-5.6-sol": ["text", "image"],
+  "gpt-5.6-terra": ["text", "image"],
+  "meta/muse-spark-1.1": ["text", "image"],
+  "meta/muse-spark-1.2": ["text", "image"],
+  "meta/muse-spark-1.2-contributor": ["text", "image"],
+  "moonshotai/Kimi-K2.5": ["text", "image"],
+  "moonshotai/Kimi-K2.6": ["text", "image"],
+  "moonshotai/Kimi-K2.7-Code": ["text", "image"],
+  "moonshotai/Kimi-K2.7-Code-Highspeed": ["text", "image"],
+  "moonshotai/Kimi-K3": ["text", "image"],
+  "sakana/fugu-ultra": ["text", "image"],
+  "stepfun/Step-3.7-Flash": ["text", "image"],
+  "thinkingmachines/inkling": ["text", "image"],
+  "thinkingmachines/inkling-small": ["text", "image"],
+  "xai/grok-4.5": ["text", "image"],
+  "xiaomi/mimo-v2.5": ["text", "image"],
+}
+
+const TEXT_INPUT_ONLY = ["text"] as const
+
+export function inputModalitiesForModel(modelId: string): readonly CommandCodeInputType[] {
+  return MODEL_INPUT_MODALITIES[modelId] ?? TEXT_INPUT_ONLY
+}
+
+export function modelSupportsImageInput(modelId: string): boolean {
+  return inputModalitiesForModel(modelId).includes("image")
+}
+
 export type PiThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max"

 type CommandCodeReasoningEffort = Exclude<PiThinkingLevel, "off">
@@ -15,7 +72,7 @@ type CommandCodeReasoningEffort = Exclude<PiThinkingLevel, "off">
 * Per-model reasoning efforts supported by Command Code's generate endpoint.
 *
 * The Provider API does not expose reasoning metadata. This is an exact
- * snapshot of `reasoningEfforts` from the command-code@1.14.1 model catalog
+ * snapshot of `reasoningEfforts` from the command-code@1.15.1 model catalog
 * (`packages/shared/src/model-catalog.ts`, also published in the generated
 * `dist/bundled/command-code-knowledge/reference/models.md`). Models omitted
 * here let Command Code choose their reasoning depth, matching the CLI.