From a9a3521b9b5a0a059e391814ab5dda6e504ef1a6 Mon Sep 17 00:00:00 2001 From: Oyster Lee Date: Mon, 3 Aug 2026 06:20:10 +0800 Subject: [PATCH 1/3] fix: forward images returned by tool calls to the model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tool result can carry images. pi's `read` tool returns a text note plus an `image` block for png/jpg/gif/webp/bmp, and screenshot tools do the same. transformMessagesForQoder() built the `tool` message with getContentText(), which maps every non-text block to "". The images were dropped silently: the TUI rendered the picture, the model received only "Read image file [image/png]" and answered that it could not see images — then fell back to scraping the DOM. Nothing reported a problem, and the model catalogue advertises input ["text", "image"], so the capability check upstream of this had no reason to warn either. The OpenAI-shaped `tool` role has nowhere to put them: its content is a plain string. So the images now follow as a separate user message, using the same image_url part the user branch already builds, with a short label so the model does not read a bare image as something the human just sent. Tool results without images are unchanged and still produce exactly one message. --- src/__tests__/transform.test.ts | 68 +++++++++++++++++++++++++++++++++ src/transform.ts | 36 +++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/src/__tests__/transform.test.ts b/src/__tests__/transform.test.ts index 36bf99b..567f631 100644 --- a/src/__tests__/transform.test.ts +++ b/src/__tests__/transform.test.ts @@ -208,6 +208,74 @@ describe("transformMessagesForQoder", () => { }); }); + it("forwards images returned by a tool call", () => { + // pi's `read` tool returns a text note plus an image block for a png. The + // `tool` role is a plain string in the OpenAI shape, so the image has to + // follow as a user message; before this it was dropped and the model saw + // only the note, then reported that it could not see images. + const msgs = [ + { + role: "toolResult", + toolCallId: "call_1", + content: [ + { type: "text", text: "Read image file [image/png]" }, + { type: "image", data: "abc123", mimeType: "image/png" }, + ], + }, + ] as unknown as Message[]; + const result = transformMessagesForQoder(msgs); + + expect(result).toHaveLength(2); + expect(result[0]).toEqual({ + role: "tool", + tool_call_id: "call_1", + content: "Read image file [image/png]", + }); + expect(result[1].role).toBe("user"); + const parts = result[1].content as Array<{ type: string; text?: string; image_url?: { url: string } }>; + expect(parts[0]).toEqual({ + type: "text", + text: "[1 image returned by the previous tool call]", + }); + expect(parts[1]).toEqual({ + type: "image_url", + image_url: { url: "data:image/png;base64,abc123" }, + }); + }); + + it("forwards several images from one tool call", () => { + const msgs = [ + { + role: "toolResult", + toolCallId: "call_1", + content: [ + { type: "text", text: "two shots" }, + { type: "image", data: "one", mimeType: "image/png" }, + { type: "image", data: "two", mimeType: "image/jpeg" }, + ], + }, + ] as unknown as Message[]; + const result = transformMessagesForQoder(msgs); + const parts = result[1].content as Array<{ type: string; text?: string; image_url?: { url: string } }>; + expect(parts[0].text).toBe("[2 images returned by the previous tool call]"); + expect(parts[1].image_url?.url).toBe("data:image/png;base64,one"); + expect(parts[2].image_url?.url).toBe("data:image/jpeg;base64,two"); + }); + + it("adds no extra message when a tool result has no images", () => { + // The common case by far; it must stay a single `tool` message. + const msgs = [ + { + role: "toolResult", + toolCallId: "call_1", + content: [{ type: "text", text: "plain text result" }], + }, + ] as unknown as Message[]; + const result = transformMessagesForQoder(msgs); + expect(result).toHaveLength(1); + expect(result[0].role).toBe("tool"); + }); + it("handles assistant message with string content", () => { const msgs = [ { diff --git a/src/transform.ts b/src/transform.ts index 3ccf014..6933b72 100644 --- a/src/transform.ts +++ b/src/transform.ts @@ -52,6 +52,12 @@ export function getContentText(msg: Message): string { return ""; } +/** The image blocks of a message, in order. Empty when there are none. */ +export function getContentImages(msg: Message): ImageContent[] { + if (!Array.isArray(msg.content)) return []; + return msg.content.filter((c): c is ImageContent => c.type === "image"); +} + export function transformTools(tools: Tool[]): QoderTool[] { return tools.map((t) => ({ type: "function", @@ -155,6 +161,36 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] { tool_call_id: tr.toolCallId, content: getContentText(tr), }); + + // A tool result may carry images — pi's `read` tool returns a text note + // plus an `image` block for png/jpg/gif/webp/bmp, and screenshot tools do + // the same. getContentText() maps every non-text block to "", so those + // images were dropped silently: the TUI rendered the picture while the + // model received only "Read image file [image/png]" and reported that it + // could not see images. + // + // The OpenAI-shaped `tool` role has nowhere to put them — its content is + // a plain string — so they follow as a separate user message, the same + // shape the user branch above already builds. The leading label keeps the + // model from reading a bare image as something the human just sent. + const images = getContentImages(tr); + if (images.length > 0) { + normalizedMessages.push({ + role: "user", + content: [ + { + type: "text", + text: `[${images.length} image${images.length === 1 ? "" : "s"} returned by the previous tool call]`, + }, + ...images.map( + (img): QoderImagePart => ({ + type: "image_url", + image_url: { url: `data:${img.mimeType};base64,${img.data}` }, + }), + ), + ], + }); + } } } From b4416909f5356501c69c7add70875a3d7ae4b659 Mon Sep 17 00:00:00 2001 From: Oyster Lee Date: Mon, 3 Aug 2026 06:46:52 +0800 Subject: [PATCH 2/3] fix: emit a tool call that arrives with no arguments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tool call whose arguments never arrive — a no-argument tool, or a model that sends id and name and then stops — was accumulated into toolCallsState but never added to the message, because the content block was only created inside `if (tc.function?.arguments)`. The finalizer then checked `toolCallsState.length > 0` and set stopReason "toolUse" on a message containing no tool call at all. pi's agent loop continues only on tool calls, so it had nothing to execute and the turn ended: mid-task, no tool run, no error, nothing in the transcript except the assistant saying what it was about to do. Observed against a real session where the model announced "let me consult the advisor" four times in a row and the run then stopped dead; its final message on disk is stopReason "toolUse" with a single text block and no toolCall. Two changes: - open the block as soon as the call is identifiable (id or name), and keep its id/name in step if they arrive in a later delta, since the finalizer only rewrites `arguments`; - guard the stopReason on blocks that actually reached the message rather than on the state array being non-empty, so a genuinely malformed stream reports "stop" instead of an unactionable "toolUse". Tests: no-argument call, identity arriving after the block is open, and a malformed delta that must not claim toolUse. --- src/__tests__/stream.test.ts | 67 ++++++++++++++++++++++++++++++++++++ src/stream.ts | 43 ++++++++++++++++++----- 2 files changed, 101 insertions(+), 9 deletions(-) diff --git a/src/__tests__/stream.test.ts b/src/__tests__/stream.test.ts index 1389509..268583a 100644 --- a/src/__tests__/stream.test.ts +++ b/src/__tests__/stream.test.ts @@ -5,6 +5,7 @@ import type { AssistantMessageEventStream, Context, Model, + ToolCall, } from "@earendil-works/pi-ai"; import { afterEach, describe, expect, it, vi } from "vitest"; import { streamQoder } from "../stream.js"; @@ -223,4 +224,70 @@ describe("streamQoder", () => { const toolCall = msg.content.find((c) => c.type === "toolCall"); expect(toolCall).toBeDefined(); }); + + it("emits a tool call that arrives with no arguments", async () => { + // A no-argument tool, or a model that sends id+name and stops. The block + // used to be created only inside `if (tc.function?.arguments)`, so this + // produced a toolCallsState entry and NO content block — and the finalizer + // then set stopReason "toolUse" on a message with no tool call in it. pi's + // agent loop had nothing to execute and the turn ended silently, mid-task. + const sse = + sseEnvelope( + chunk({ + tool_calls: [{ index: 0, id: "call_1", function: { name: "advisor", arguments: "" } }], + }), + ) + + sseEnvelope(finishChunk("tool_calls")) + + DONE_SSE; + globalThis.fetch = mockFetch(sse); + const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" }); + const events = await consume(stream); + + const done = events.find((e) => e.type === "done"); + const msg = (done as { message: AssistantMessage }).message; + const toolCall = msg.content.find((c) => c.type === "toolCall") as ToolCall | undefined; + expect(toolCall, "a named tool call must reach the message even with no arguments").toBeDefined(); + expect(toolCall?.name).toBe("advisor"); + expect(toolCall?.id).toBe("call_1"); + expect(toolCall?.arguments).toEqual({}); + expect(msg.stopReason).toBe("toolUse"); + }); + + it("picks up an id and name that arrive after the block is open", async () => { + // Streamed the other way round: arguments first, identity later. + const sse = + sseEnvelope(chunk({ tool_calls: [{ index: 0, function: { name: "bash", arguments: '{"comm' } }] })) + + sseEnvelope(chunk({ tool_calls: [{ index: 0, id: "call_9", function: { arguments: 'and":"ls"}' } }] })) + + sseEnvelope(finishChunk("tool_calls")) + + DONE_SSE; + globalThis.fetch = mockFetch(sse); + const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" }); + const events = await consume(stream); + + const done = events.find((e) => e.type === "done"); + const msg = (done as { message: AssistantMessage }).message; + const toolCall = msg.content.find((c) => c.type === "toolCall") as ToolCall | undefined; + expect(toolCall?.id).toBe("call_9"); + expect(toolCall?.name).toBe("bash"); + expect(toolCall?.arguments).toEqual({ command: "ls" }); + }); + + it("does not claim toolUse when no tool call reached the message", async () => { + // A malformed stream: a tool_calls delta with neither id nor name. Better a + // clean "stop" than a message that says toolUse and carries nothing, which + // the agent loop cannot act on and cannot report. + const sse = + sseEnvelope(chunk({ content: "thinking about it", role: "assistant" })) + + sseEnvelope(chunk({ tool_calls: [{ index: 0, function: {} }] })) + + sseEnvelope(finishChunk("stop")) + + DONE_SSE; + globalThis.fetch = mockFetch(sse); + const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" }); + const events = await consume(stream); + + const done = events.find((e) => e.type === "done"); + const msg = (done as { message: AssistantMessage }).message; + expect(msg.content.find((c) => c.type === "toolCall")).toBeUndefined(); + expect(msg.stopReason).toBe("stop"); + }); }); diff --git a/src/stream.ts b/src/stream.ts index 360b366..fd78f54 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -407,17 +407,39 @@ export function streamQoder( const state = toolCallsState[idx]; if (tc.id) state.id = tc.id; if (tc.function?.name) state.name = tc.function.name; + + // Open the block as soon as the call is IDENTIFIABLE, not + // when its first argument byte arrives. A call whose + // arguments are absent or an empty string — a no-argument + // tool, or a model that sends id+name and then stops — used + // to create a toolCallsState entry and no content block, so + // the finalizer below saw a non-empty state array, set + // stopReason "toolUse", and handed back a message with no + // tool call in it. The agent loop then had nothing to run + // and the turn simply ended, mid-task and without an error. + if (state.emittedStart === undefined && (state.id || state.name)) { + state.emittedStart = true; + state.contentIndex = output.content.length; + output.content.push({ + type: "toolCall", + id: state.id, + name: state.name, + arguments: {}, + } satisfies ToolCall); + stream.push({ type: "toolcall_start", contentIndex: state.contentIndex, partial: output }); + } + + // id and name can arrive after the block is open; keep it + // in step, since the finalizer only rewrites `arguments`. + if (state.emittedStart) { + const block = output.content[state.contentIndex] as ToolCall; + block.id = state.id; + block.name = state.name; + } + if (tc.function?.arguments) { const argDelta = tc.function.arguments; state.arguments += argDelta; - - if (state.emittedStart === undefined) { - state.emittedStart = true; - state.contentIndex = output.content.length; - const block: ToolCall = { type: "toolCall", id: state.id, name: state.name, arguments: {} }; - output.content.push(block); - stream.push({ type: "toolcall_start", contentIndex: state.contentIndex, partial: output }); - } stream.push({ type: "toolcall_delta", contentIndex: state.contentIndex, @@ -487,7 +509,10 @@ export function streamQoder( } } - if (toolCallsState.length > 0) { + // Guarded on blocks that actually reached the message, not on the state + // array being non-empty. Claiming "toolUse" for a message carrying no + // tool call is what turned a malformed stream into a silent dead end. + if (toolCallsState.some((state) => state?.emittedStart)) { output.stopReason = "toolUse"; } // Otherwise keep whatever finish_reason set upstream (defaults to "stop"). From 6956615f6664b51ba87a84042fa4ee2d2e2d2c08 Mon Sep 17 00:00:00 2001 From: Loongphy Date: Fri, 31 Jul 2026 06:37:45 +0800 Subject: [PATCH 3/3] fix: pi.extensions path should point to dist/index.js instead of src/index.ts The "files" field in package.json only includes "dist" and "README.md", so src/index.ts is not available in the published npm package. This was introduced in commit 1b110a6 ("fix index") which changed the path from the correct "./dist/index.js" to "./src/index.ts". --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2e619d7..a0f4a39 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ }, "pi": { "extensions": [ - "./src/index.ts" + "./dist/index.js" ] }, "devDependencies": {