From d0d69617da28b079453334631666a57a75422712 Mon Sep 17 00:00:00 2001 From: cat_shark <1716967236@qq.com> Date: Thu, 24 Sep 2026 17:17:25 +0800 Subject: [PATCH] fix: keep a turn's tool results together when one returns an image The OpenAI-shaped `tool` role content is a plain string, so a tool result carrying images (pi's read/screenshot tools) follows as an extra user message. It was emitted immediately after its own tool message, which broke multi-call turns: assistant(tool_calls=[a, b]) -> tool(a) -> user(image) -> tool(b) Qoder rejects that with "An assistant message with 'tool_calls' must be followed by tool messages responding to each 'tool_call_id' (insufficient tool messages following tool_calls message)", leaving the session unable to send any further request. Queue the image follow-up and flush it once the run of consecutive tool results ends, so every tool_call_id is answered before the user message. --- src/__tests__/transform.test.ts | 36 +++++++++++++++++++++++++++++++++ src/protocol/transform.ts | 28 ++++++++++++++++++++----- 2 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/__tests__/transform.test.ts b/src/__tests__/transform.test.ts index 2f56672..226e8f2 100644 --- a/src/__tests__/transform.test.ts +++ b/src/__tests__/transform.test.ts @@ -276,6 +276,42 @@ describe("transformMessagesForQoder", () => { expect(result[0].role).toBe("tool"); }); + it("keeps every tool result of one assistant turn together when a sibling returns an image", () => { + // Regression: the image follow-up must not land between the tool results of + // a multi-call assistant turn. `assistant(tool_calls) -> tool(a) -> user -> + // tool(b)` makes Qoder reject the request with "An assistant message with + // 'tool_calls' must be followed by tool messages responding to each + // 'tool_call_id' (insufficient tool messages following tool_calls message)". + const msgs = [ + { + role: "assistant", + content: [ + { type: "toolCall", id: "call_a", name: "read", arguments: { path: "/tmp/a.png" } }, + { type: "toolCall", id: "call_b", name: "read", arguments: { path: "/tmp/b.md" } }, + ], + }, + { + role: "toolResult", + toolCallId: "call_a", + content: [ + { type: "text", text: "Read image file [image/png]" }, + { type: "image", data: "abc123", mimeType: "image/png" }, + ], + }, + { + role: "toolResult", + toolCallId: "call_b", + content: [{ type: "text", text: "# notes" }], + }, + ] as unknown as Message[]; + const result = transformMessagesForQoder(msgs); + + expect(result.map((m) => m.role)).toEqual(["assistant", "tool", "tool", "user"]); + expect(result[1].tool_call_id).toBe("call_a"); + expect(result[2].tool_call_id).toBe("call_b"); + expect(result[3].content).toEqual(expect.arrayContaining([expect.objectContaining({ type: "image_url" })])); + }); + it("handles assistant message with string content", () => { const msgs = [ { diff --git a/src/protocol/transform.ts b/src/protocol/transform.ts index 5e06cc1..96b12ae 100644 --- a/src/protocol/transform.ts +++ b/src/protocol/transform.ts @@ -78,12 +78,26 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] { // with tool_calls". const droppedToolCallIds = new Set(); + // A tool result that carries images adds a follow-up user message (the `tool` + // role is a plain string in the OpenAI shape). It must not be emitted between + // the tool results of one assistant turn: an upstream rejects + // `assistant(tool_calls) -> tool(a) -> user -> tool(b)` with + // "An assistant message with 'tool_calls' must be followed by tool messages + // responding to each 'tool_call_id'". Queue it and flush once the run of + // consecutive tool results ends. + let pendingImageMessages: QoderMessage[] = []; + const flushPendingImageMessages = (): void => { + if (pendingImageMessages.length === 0) return; + normalizedMessages.push(...pendingImageMessages); + pendingImageMessages = []; + }; + for (const msg of messages) { // Skip error or aborted messages - if ( + const skipAbortedAssistant = msg.role === "assistant" && - ((msg as AssistantMessage).stopReason === "error" || (msg as AssistantMessage).stopReason === "aborted") - ) { + ((msg as AssistantMessage).stopReason === "error" || (msg as AssistantMessage).stopReason === "aborted"); + if (skipAbortedAssistant) { const am = msg as AssistantMessage; if (Array.isArray(am.content)) { for (const block of am.content) { @@ -93,7 +107,6 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] { } } } - continue; } // Drop the result too, otherwise it refers to a tool_calls entry that is @@ -102,6 +115,10 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] { continue; } + // End the current tool-result run (kept open above) before the next turn. + if (msg.role !== "toolResult") flushPendingImageMessages(); + if (skipAbortedAssistant) continue; + if (msg.role === "user") { let content: QoderContent = ""; if (typeof msg.content === "string") { @@ -196,7 +213,7 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] { // model from reading a bare image as something the human just sent. const images = getContentImages(tr); if (images.length > 0) { - normalizedMessages.push({ + pendingImageMessages.push({ role: "user", content: [ { @@ -215,5 +232,6 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] { } } + flushPendingImageMessages(); return normalizedMessages; }