diff --git a/src/__tests__/transform.test.ts b/src/__tests__/transform.test.ts index 36bf99b..567f631 100644 --- a/src/__tests__/transform.test.ts +++ b/src/__tests__/transform.test.ts @@ -208,6 +208,74 @@ describe("transformMessagesForQoder", () => { }); }); + it("forwards images returned by a tool call", () => { + // pi's `read` tool returns a text note plus an image block for a png. The + // `tool` role is a plain string in the OpenAI shape, so the image has to + // follow as a user message; before this it was dropped and the model saw + // only the note, then reported that it could not see images. + const msgs = [ + { + role: "toolResult", + toolCallId: "call_1", + content: [ + { type: "text", text: "Read image file [image/png]" }, + { type: "image", data: "abc123", mimeType: "image/png" }, + ], + }, + ] as unknown as Message[]; + const result = transformMessagesForQoder(msgs); + + expect(result).toHaveLength(2); + expect(result[0]).toEqual({ + role: "tool", + tool_call_id: "call_1", + content: "Read image file [image/png]", + }); + expect(result[1].role).toBe("user"); + const parts = result[1].content as Array<{ type: string; text?: string; image_url?: { url: string } }>; + expect(parts[0]).toEqual({ + type: "text", + text: "[1 image returned by the previous tool call]", + }); + expect(parts[1]).toEqual({ + type: "image_url", + image_url: { url: "data:image/png;base64,abc123" }, + }); + }); + + it("forwards several images from one tool call", () => { + const msgs = [ + { + role: "toolResult", + toolCallId: "call_1", + content: [ + { type: "text", text: "two shots" }, + { type: "image", data: "one", mimeType: "image/png" }, + { type: "image", data: "two", mimeType: "image/jpeg" }, + ], + }, + ] as unknown as Message[]; + const result = transformMessagesForQoder(msgs); + const parts = result[1].content as Array<{ type: string; text?: string; image_url?: { url: string } }>; + expect(parts[0].text).toBe("[2 images returned by the previous tool call]"); + expect(parts[1].image_url?.url).toBe("data:image/png;base64,one"); + expect(parts[2].image_url?.url).toBe("data:image/jpeg;base64,two"); + }); + + it("adds no extra message when a tool result has no images", () => { + // The common case by far; it must stay a single `tool` message. + const msgs = [ + { + role: "toolResult", + toolCallId: "call_1", + content: [{ type: "text", text: "plain text result" }], + }, + ] as unknown as Message[]; + const result = transformMessagesForQoder(msgs); + expect(result).toHaveLength(1); + expect(result[0].role).toBe("tool"); + }); + it("handles assistant message with string content", () => { const msgs = [ { diff --git a/src/transform.ts b/src/transform.ts index 3ccf014..6933b72 100644 --- a/src/transform.ts +++ b/src/transform.ts @@ -52,6 +52,12 @@ export function getContentText(msg: Message): string { return ""; } +/** The image blocks of a message, in order. Empty when there are none. */ +export function getContentImages(msg: Message): ImageContent[] { + if (!Array.isArray(msg.content)) return []; + return msg.content.filter((c): c is ImageContent => c.type === "image"); +} + export function transformTools(tools: Tool[]): QoderTool[] { return tools.map((t) => ({ type: "function", @@ -155,6 +161,36 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] { tool_call_id: tr.toolCallId, content: getContentText(tr), }); + + // A tool result may carry images — pi's `read` tool returns a text note + // plus an `image` block for png/jpg/gif/webp/bmp, and screenshot tools do + // the same. getContentText() maps every non-text block to "", so those + // images were dropped silently: the TUI rendered the picture while the + // model received only "Read image file [image/png]" and reported that it + // could not see images. + // + // The OpenAI-shaped `tool` role has nowhere to put them — its content is + // a plain string — so they follow as a separate user message, the same + // shape the user branch above already builds. The leading label keeps the + // model from reading a bare image as something the human just sent. + const images = getContentImages(tr); + if (images.length > 0) { + normalizedMessages.push({ + role: "user", + content: [ + { + type: "text", + text: `[${images.length} image${images.length === 1 ? "" : "s"} returned by the previous tool call]`, + }, + ...images.map( + (img): QoderImagePart => ({ + type: "image_url", + image_url: { url: `data:${img.mimeType};base64,${img.data}` }, + }), + ), + ], + }); + } } }