fix(core): omit historical images for text models
This commit is contained in:
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
## Unreleased
|
## Unreleased
|
||||||
|
|
||||||
|
- Allow switching from a vision-capable model to a text-only model by omitting historical image tool results while preserving their text output; direct image prompts still fail clearly.
|
||||||
- Stream incremental tool-call arguments from the `/alpha/generate` transport instead of waiting for the final complete tool-call event.
|
- Stream incremental tool-call arguments from the `/alpha/generate` transport instead of waiting for the final complete tool-call event.
|
||||||
- Add a daily GitHub Actions synchronization job that opens or updates a pull request for CLI version, image capability, reasoning, effort, and output-limit changes in the latest published Command Code catalog.
|
- Add a daily GitHub Actions synchronization job that opens or updates a pull request for CLI version, image capability, reasoning, effort, and output-limit changes in the latest published Command Code catalog.
|
||||||
- Refresh static model capabilities from `command-code@1.32.2`, separating reasoning support from selectable effort levels and honoring model-specific output limits.
|
- Refresh static model capabilities from `command-code@1.32.2`, separating reasoning support from selectable effort levels and honoring model-specific output limits.
|
||||||
|
|||||||
+10
-8
@@ -66,9 +66,8 @@ function imageContentError(role: string): Error {
|
|||||||
|
|
||||||
export function assertTextOnlyMessages(messages?: readonly MessageLike[]): void {
|
export function assertTextOnlyMessages(messages?: readonly MessageLike[]): void {
|
||||||
for (const message of messages ?? []) {
|
for (const message of messages ?? []) {
|
||||||
if (imageParts(message.content).length > 0) {
|
if (message.role !== "toolResult" && imageParts(message.content).length > 0) {
|
||||||
const role = message.role === "toolResult" ? "tool results" : `${message.role} messages`
|
throw imageContentError(`${message.role} messages`)
|
||||||
throw imageContentError(role)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -268,6 +267,11 @@ export function messagesToCC(
|
|||||||
if (missingResults.length > 0) out.push({ role: "tool", content: missingResults })
|
if (missingResults.length > 0) out.push({ role: "tool", content: missingResults })
|
||||||
} else if (message.role === "toolResult") {
|
} else if (message.role === "toolResult") {
|
||||||
if (!message.toolCallId || !callIds.has(message.toolCallId)) continue
|
if (!message.toolCallId || !callIds.has(message.toolCallId)) continue
|
||||||
|
const images = imageParts(message.content)
|
||||||
|
const text = textContent(message)
|
||||||
|
const outputText =
|
||||||
|
text ||
|
||||||
|
(images.length > 0 && !allowImages ? "[Image omitted: model does not support images]" : "")
|
||||||
out.push({
|
out.push({
|
||||||
role: "tool",
|
role: "tool",
|
||||||
content: [
|
content: [
|
||||||
@@ -276,15 +280,13 @@ export function messagesToCC(
|
|||||||
toolCallId: message.toolCallId,
|
toolCallId: message.toolCallId,
|
||||||
toolName: message.toolName,
|
toolName: message.toolName,
|
||||||
output: message.isError
|
output: message.isError
|
||||||
? { type: "error-text", value: textContent(message) }
|
? { type: "error-text", value: outputText }
|
||||||
: { type: "text", value: textContent(message) },
|
: { type: "text", value: outputText },
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
})
|
})
|
||||||
|
|
||||||
const images = imageParts(message.content)
|
if (images.length > 0 && allowImages) {
|
||||||
if (images.length > 0) {
|
|
||||||
if (!allowImages) throw imageContentError("tool results")
|
|
||||||
out.push({
|
out.push({
|
||||||
role: "user",
|
role: "user",
|
||||||
content: images.map(imageToCommandCode),
|
content: images.map(imageToCommandCode),
|
||||||
|
|||||||
@@ -161,7 +161,7 @@ describe("projectSlugFromPath()", () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
describe("text-only image handling", () => {
|
describe("text-only image handling", () => {
|
||||||
it("rejects image content for models without image support", () => {
|
it("rejects direct image input for models without image support", () => {
|
||||||
assert.throws(
|
assert.throws(
|
||||||
() =>
|
() =>
|
||||||
assertTextOnlyMessages([
|
assertTextOnlyMessages([
|
||||||
@@ -172,8 +172,10 @@ describe("text-only image handling", () => {
|
|||||||
]),
|
]),
|
||||||
/does not support image content/i,
|
/does not support image content/i,
|
||||||
)
|
)
|
||||||
assert.throws(
|
})
|
||||||
() =>
|
|
||||||
|
it("allows historical tool-result images to be omitted for text-only models", () => {
|
||||||
|
assert.doesNotThrow(() =>
|
||||||
assertTextOnlyMessages([
|
assertTextOnlyMessages([
|
||||||
{
|
{
|
||||||
role: "toolResult",
|
role: "toolResult",
|
||||||
@@ -181,7 +183,6 @@ describe("text-only image handling", () => {
|
|||||||
content: [{ type: "image", data: "base64-data", mimeType: "image/png" }],
|
content: [{ type: "image", data: "base64-data", mimeType: "image/png" }],
|
||||||
},
|
},
|
||||||
]),
|
]),
|
||||||
/does not support image content/i,
|
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
@@ -597,6 +598,48 @@ describe("messagesToCC()", () => {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it("omits tool-result images for text-only models while preserving their text", () => {
|
||||||
|
const result = messagesToCC([
|
||||||
|
{ role: "user", content: "read image" },
|
||||||
|
{
|
||||||
|
role: "assistant",
|
||||||
|
content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: "toolResult",
|
||||||
|
toolCallId: "c1",
|
||||||
|
toolName: "read",
|
||||||
|
content: [
|
||||||
|
{ type: "text", text: "image attached" },
|
||||||
|
{ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
])
|
||||||
|
|
||||||
|
assert.equal(objectAt(result, ["2", "content", "0", "output", "value"]), "image attached")
|
||||||
|
assert.equal(objectAt(result, ["3"]), undefined)
|
||||||
|
})
|
||||||
|
|
||||||
|
it("describes an omitted image-only tool result for text-only models", () => {
|
||||||
|
const result = messagesToCC([
|
||||||
|
{
|
||||||
|
role: "assistant",
|
||||||
|
content: [{ type: "toolCall", id: "c1", name: "read", arguments: {} }],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: "toolResult",
|
||||||
|
toolCallId: "c1",
|
||||||
|
toolName: "read",
|
||||||
|
content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/jpeg" }],
|
||||||
|
},
|
||||||
|
])
|
||||||
|
|
||||||
|
assert.equal(
|
||||||
|
objectAt(result, ["1", "content", "0", "output", "value"]),
|
||||||
|
"[Image omitted: model does not support images]",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
it("preserves tool-result images as a following user image message", () => {
|
it("preserves tool-result images as a following user image message", () => {
|
||||||
const result = messagesToCC(
|
const result = messagesToCC(
|
||||||
[
|
[
|
||||||
|
|||||||
+22
-8
@@ -240,12 +240,16 @@ describe("streamCommandCode — successful streams", () => {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it("rejects a tool-result image before network access for text-only models", async () => {
|
it("omits a historical tool-result image after switching to a text-only model", async () => {
|
||||||
|
server.mockResponse({
|
||||||
|
type: "success",
|
||||||
|
events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
|
||||||
|
})
|
||||||
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
|
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
|
||||||
|
|
||||||
const events = await collectEvents(
|
const events = await collectEvents(
|
||||||
streamCommandCode(
|
streamCommandCode(
|
||||||
makeModel({ id: "deepseek/deepseek-v4-pro" }),
|
makeModel({ id: "deepseek/deepseek-v4-flash" }),
|
||||||
makeContext({
|
makeContext({
|
||||||
messages: [
|
messages: [
|
||||||
{ role: "user", content: "read the image" },
|
{ role: "user", content: "read the image" },
|
||||||
@@ -257,19 +261,29 @@ describe("streamCommandCode — successful streams", () => {
|
|||||||
role: "toolResult",
|
role: "toolResult",
|
||||||
toolCallId: "c1",
|
toolCallId: "c1",
|
||||||
toolName: "read",
|
toolName: "read",
|
||||||
content: [{ type: "image", data: "aGVsbG8=", mimeType: "image/png" }],
|
content: [
|
||||||
|
{ type: "text", text: "image attached" },
|
||||||
|
{ type: "image", data: "aGVsbG8=", mimeType: "image/png" },
|
||||||
|
],
|
||||||
},
|
},
|
||||||
|
{ role: "user", content: "continue without the image" },
|
||||||
],
|
],
|
||||||
}),
|
}),
|
||||||
{ apiKey: "mock-key" },
|
{ apiKey: "mock-key" },
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
const lastEvent = events.at(-1)
|
assert.equal(events.at(-1)?.type, "done")
|
||||||
assert.equal(lastEvent?.type, "error")
|
assert.equal(server.requestCount(), 1)
|
||||||
if (lastEvent?.type !== "error") throw new Error("expected error event")
|
const body = server.lastRequestBody()
|
||||||
assert.match(lastEvent.error.errorMessage ?? "", /does not support image content/i)
|
assert.equal(
|
||||||
assert.equal(server.requestCount(), 0)
|
objectAt(body, ["params", "messages", "2", "content", "0", "output", "value"]),
|
||||||
|
"image attached",
|
||||||
|
)
|
||||||
|
assert.equal(
|
||||||
|
objectAt(body, ["params", "messages", "3", "content"]),
|
||||||
|
"continue without the image",
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it("rejects images before network access for text-only models", async () => {
|
it("rejects images before network access for text-only models", async () => {
|
||||||
|
|||||||
Reference in New Issue
Block a user