fix: forward images returned by tool calls to the model

A tool result can carry images. pi's `read` tool returns a text note plus an
`image` block for png/jpg/gif/webp/bmp, and screenshot tools do the same.

transformMessagesForQoder() built the `tool` message with getContentText(),
which maps every non-text block to "". The images were dropped silently: the
TUI rendered the picture, the model received only "Read image file [image/png]"
and answered that it could not see images — then fell back to scraping the DOM.
Nothing reported a problem, and the model catalogue advertises input
["text", "image"], so the capability check upstream of this had no reason to
warn either.

The OpenAI-shaped `tool` role has nowhere to put them: its content is a plain
string. So the images now follow as a separate user message, using the same
image_url part the user branch already builds, with a short label so the model
does not read a bare image as something the human just sent.

Tool results without images are unchanged and still produce exactly one
message.
This commit is contained in:
Oyster Lee
2026-08-03 06:20:10 +08:00
parent 3af6bde995
commit a9a3521b9b
2 changed files with 104 additions and 0 deletions
+68
View File
@@ -208,6 +208,74 @@ describe("transformMessagesForQoder", () => {
});
});
it("forwards images returned by a tool call", () => {
// pi's `read` tool returns a text note plus an image block for a png. The
// `tool` role is a plain string in the OpenAI shape, so the image has to
// follow as a user message; before this it was dropped and the model saw
// only the note, then reported that it could not see images.
const msgs = [
{
role: "toolResult",
toolCallId: "call_1",
content: [
{ type: "text", text: "Read image file [image/png]" },
{ type: "image", data: "abc123", mimeType: "image/png" },
],
},
] as unknown as Message[];
const result = transformMessagesForQoder(msgs);
expect(result).toHaveLength(2);
expect(result[0]).toEqual({
role: "tool",
tool_call_id: "call_1",
content: "Read image file [image/png]",
});
expect(result[1].role).toBe("user");
const parts = result[1].content as Array<{ type: string; text?: string; image_url?: { url: string } }>;
expect(parts[0]).toEqual({
type: "text",
text: "[1 image returned by the previous tool call]",
});
expect(parts[1]).toEqual({
type: "image_url",
image_url: { url: "data:image/png;base64,abc123" },
});
});
it("forwards several images from one tool call", () => {
const msgs = [
{
role: "toolResult",
toolCallId: "call_1",
content: [
{ type: "text", text: "two shots" },
{ type: "image", data: "one", mimeType: "image/png" },
{ type: "image", data: "two", mimeType: "image/jpeg" },
],
},
] as unknown as Message[];
const result = transformMessagesForQoder(msgs);
const parts = result[1].content as Array<{ type: string; text?: string; image_url?: { url: string } }>;
expect(parts[0].text).toBe("[2 images returned by the previous tool call]");
expect(parts[1].image_url?.url).toBe("data:image/png;base64,one");
expect(parts[2].image_url?.url).toBe("data:image/jpeg;base64,two");
});
it("adds no extra message when a tool result has no images", () => {
// The common case by far; it must stay a single `tool` message.
const msgs = [
{
role: "toolResult",
toolCallId: "call_1",
content: [{ type: "text", text: "plain text result" }],
},
] as unknown as Message[];
const result = transformMessagesForQoder(msgs);
expect(result).toHaveLength(1);
expect(result[0].role).toBe("tool");
});
it("handles assistant message with string content", () => {
const msgs = [
{
+36
View File
@@ -52,6 +52,12 @@ export function getContentText(msg: Message): string {
return "";
}
/** The image blocks of a message, in order. Empty when there are none. */
export function getContentImages(msg: Message): ImageContent[] {
if (!Array.isArray(msg.content)) return [];
return msg.content.filter((c): c is ImageContent => c.type === "image");
}
export function transformTools(tools: Tool[]): QoderTool[] {
return tools.map((t) => ({
type: "function",
@@ -155,6 +161,36 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] {
tool_call_id: tr.toolCallId,
content: getContentText(tr),
});
// A tool result may carry images — pi's `read` tool returns a text note
// plus an `image` block for png/jpg/gif/webp/bmp, and screenshot tools do
// the same. getContentText() maps every non-text block to "", so those
// images were dropped silently: the TUI rendered the picture while the
// model received only "Read image file [image/png]" and reported that it
// could not see images.
//
// The OpenAI-shaped `tool` role has nowhere to put them — its content is
// a plain string — so they follow as a separate user message, the same
// shape the user branch above already builds. The leading label keeps the
// model from reading a bare image as something the human just sent.
const images = getContentImages(tr);
if (images.length > 0) {
normalizedMessages.push({
role: "user",
content: [
{
type: "text",
text: `[${images.length} image${images.length === 1 ? "" : "s"} returned by the previous tool call]`,
},
...images.map(
(img): QoderImagePart => ({
type: "image_url",
image_url: { url: `data:${img.mimeType};base64,${img.data}` },
}),
),
],
});
}
}
}