Merge pull request #14 from OysterD3/fix/forward-tool-result-images
fix: forward tool-result images, and emit tool calls that arrive with no arguments
This commit is contained in:
@@ -5,6 +5,7 @@ import type {
|
||||
AssistantMessageEventStream,
|
||||
Context,
|
||||
Model,
|
||||
ToolCall,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { streamQoder } from "../stream.js";
|
||||
@@ -223,4 +224,70 @@ describe("streamQoder", () => {
|
||||
const toolCall = msg.content.find((c) => c.type === "toolCall");
|
||||
expect(toolCall).toBeDefined();
|
||||
});
|
||||
|
||||
it("emits a tool call that arrives with no arguments", async () => {
|
||||
// A no-argument tool, or a model that sends id+name and stops. The block
|
||||
// used to be created only inside `if (tc.function?.arguments)`, so this
|
||||
// produced a toolCallsState entry and NO content block — and the finalizer
|
||||
// then set stopReason "toolUse" on a message with no tool call in it. pi's
|
||||
// agent loop had nothing to execute and the turn ended silently, mid-task.
|
||||
const sse =
|
||||
sseEnvelope(
|
||||
chunk({
|
||||
tool_calls: [{ index: 0, id: "call_1", function: { name: "advisor", arguments: "" } }],
|
||||
}),
|
||||
) +
|
||||
sseEnvelope(finishChunk("tool_calls")) +
|
||||
DONE_SSE;
|
||||
globalThis.fetch = mockFetch(sse);
|
||||
const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" });
|
||||
const events = await consume(stream);
|
||||
|
||||
const done = events.find((e) => e.type === "done");
|
||||
const msg = (done as { message: AssistantMessage }).message;
|
||||
const toolCall = msg.content.find((c) => c.type === "toolCall") as ToolCall | undefined;
|
||||
expect(toolCall, "a named tool call must reach the message even with no arguments").toBeDefined();
|
||||
expect(toolCall?.name).toBe("advisor");
|
||||
expect(toolCall?.id).toBe("call_1");
|
||||
expect(toolCall?.arguments).toEqual({});
|
||||
expect(msg.stopReason).toBe("toolUse");
|
||||
});
|
||||
|
||||
it("picks up an id and name that arrive after the block is open", async () => {
|
||||
// Streamed the other way round: arguments first, identity later.
|
||||
const sse =
|
||||
sseEnvelope(chunk({ tool_calls: [{ index: 0, function: { name: "bash", arguments: '{"comm' } }] })) +
|
||||
sseEnvelope(chunk({ tool_calls: [{ index: 0, id: "call_9", function: { arguments: 'and":"ls"}' } }] })) +
|
||||
sseEnvelope(finishChunk("tool_calls")) +
|
||||
DONE_SSE;
|
||||
globalThis.fetch = mockFetch(sse);
|
||||
const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" });
|
||||
const events = await consume(stream);
|
||||
|
||||
const done = events.find((e) => e.type === "done");
|
||||
const msg = (done as { message: AssistantMessage }).message;
|
||||
const toolCall = msg.content.find((c) => c.type === "toolCall") as ToolCall | undefined;
|
||||
expect(toolCall?.id).toBe("call_9");
|
||||
expect(toolCall?.name).toBe("bash");
|
||||
expect(toolCall?.arguments).toEqual({ command: "ls" });
|
||||
});
|
||||
|
||||
it("does not claim toolUse when no tool call reached the message", async () => {
|
||||
// A malformed stream: a tool_calls delta with neither id nor name. Better a
|
||||
// clean "stop" than a message that says toolUse and carries nothing, which
|
||||
// the agent loop cannot act on and cannot report.
|
||||
const sse =
|
||||
sseEnvelope(chunk({ content: "thinking about it", role: "assistant" })) +
|
||||
sseEnvelope(chunk({ tool_calls: [{ index: 0, function: {} }] })) +
|
||||
sseEnvelope(finishChunk("stop")) +
|
||||
DONE_SSE;
|
||||
globalThis.fetch = mockFetch(sse);
|
||||
const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" });
|
||||
const events = await consume(stream);
|
||||
|
||||
const done = events.find((e) => e.type === "done");
|
||||
const msg = (done as { message: AssistantMessage }).message;
|
||||
expect(msg.content.find((c) => c.type === "toolCall")).toBeUndefined();
|
||||
expect(msg.stopReason).toBe("stop");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -208,6 +208,74 @@ describe("transformMessagesForQoder", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("forwards images returned by a tool call", () => {
|
||||
// pi's `read` tool returns a text note plus an image block for a png. The
|
||||
// `tool` role is a plain string in the OpenAI shape, so the image has to
|
||||
// follow as a user message; before this it was dropped and the model saw
|
||||
// only the note, then reported that it could not see images.
|
||||
const msgs = [
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_1",
|
||||
content: [
|
||||
{ type: "text", text: "Read image file [image/png]" },
|
||||
{ type: "image", data: "abc123", mimeType: "image/png" },
|
||||
],
|
||||
},
|
||||
] as unknown as Message[];
|
||||
const result = transformMessagesForQoder(msgs);
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0]).toEqual({
|
||||
role: "tool",
|
||||
tool_call_id: "call_1",
|
||||
content: "Read image file [image/png]",
|
||||
});
|
||||
expect(result[1].role).toBe("user");
|
||||
const parts = result[1].content as Array<{ type: string; text?: string; image_url?: { url: string } }>;
|
||||
expect(parts[0]).toEqual({
|
||||
type: "text",
|
||||
text: "[1 image returned by the previous tool call]",
|
||||
});
|
||||
expect(parts[1]).toEqual({
|
||||
type: "image_url",
|
||||
image_url: { url: "data:image/png;base64,abc123" },
|
||||
});
|
||||
});
|
||||
|
||||
it("forwards several images from one tool call", () => {
|
||||
const msgs = [
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_1",
|
||||
content: [
|
||||
{ type: "text", text: "two shots" },
|
||||
{ type: "image", data: "one", mimeType: "image/png" },
|
||||
{ type: "image", data: "two", mimeType: "image/jpeg" },
|
||||
],
|
||||
},
|
||||
] as unknown as Message[];
|
||||
const result = transformMessagesForQoder(msgs);
|
||||
const parts = result[1].content as Array<{ type: string; text?: string; image_url?: { url: string } }>;
|
||||
expect(parts[0].text).toBe("[2 images returned by the previous tool call]");
|
||||
expect(parts[1].image_url?.url).toBe("data:image/png;base64,one");
|
||||
expect(parts[2].image_url?.url).toBe("data:image/jpeg;base64,two");
|
||||
});
|
||||
|
||||
it("adds no extra message when a tool result has no images", () => {
|
||||
// The common case by far; it must stay a single `tool` message.
|
||||
const msgs = [
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_1",
|
||||
content: [{ type: "text", text: "plain text result" }],
|
||||
},
|
||||
] as unknown as Message[];
|
||||
const result = transformMessagesForQoder(msgs);
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].role).toBe("tool");
|
||||
});
|
||||
|
||||
it("handles assistant message with string content", () => {
|
||||
const msgs = [
|
||||
{
|
||||
|
||||
+34
-9
@@ -407,17 +407,39 @@ export function streamQoder(
|
||||
const state = toolCallsState[idx];
|
||||
if (tc.id) state.id = tc.id;
|
||||
if (tc.function?.name) state.name = tc.function.name;
|
||||
|
||||
// Open the block as soon as the call is IDENTIFIABLE, not
|
||||
// when its first argument byte arrives. A call whose
|
||||
// arguments are absent or an empty string — a no-argument
|
||||
// tool, or a model that sends id+name and then stops — used
|
||||
// to create a toolCallsState entry and no content block, so
|
||||
// the finalizer below saw a non-empty state array, set
|
||||
// stopReason "toolUse", and handed back a message with no
|
||||
// tool call in it. The agent loop then had nothing to run
|
||||
// and the turn simply ended, mid-task and without an error.
|
||||
if (state.emittedStart === undefined && (state.id || state.name)) {
|
||||
state.emittedStart = true;
|
||||
state.contentIndex = output.content.length;
|
||||
output.content.push({
|
||||
type: "toolCall",
|
||||
id: state.id,
|
||||
name: state.name,
|
||||
arguments: {},
|
||||
} satisfies ToolCall);
|
||||
stream.push({ type: "toolcall_start", contentIndex: state.contentIndex, partial: output });
|
||||
}
|
||||
|
||||
// id and name can arrive after the block is open; keep it
|
||||
// in step, since the finalizer only rewrites `arguments`.
|
||||
if (state.emittedStart) {
|
||||
const block = output.content[state.contentIndex] as ToolCall;
|
||||
block.id = state.id;
|
||||
block.name = state.name;
|
||||
}
|
||||
|
||||
if (tc.function?.arguments) {
|
||||
const argDelta = tc.function.arguments;
|
||||
state.arguments += argDelta;
|
||||
|
||||
if (state.emittedStart === undefined) {
|
||||
state.emittedStart = true;
|
||||
state.contentIndex = output.content.length;
|
||||
const block: ToolCall = { type: "toolCall", id: state.id, name: state.name, arguments: {} };
|
||||
output.content.push(block);
|
||||
stream.push({ type: "toolcall_start", contentIndex: state.contentIndex, partial: output });
|
||||
}
|
||||
stream.push({
|
||||
type: "toolcall_delta",
|
||||
contentIndex: state.contentIndex,
|
||||
@@ -487,7 +509,10 @@ export function streamQoder(
|
||||
}
|
||||
}
|
||||
|
||||
if (toolCallsState.length > 0) {
|
||||
// Guarded on blocks that actually reached the message, not on the state
|
||||
// array being non-empty. Claiming "toolUse" for a message carrying no
|
||||
// tool call is what turned a malformed stream into a silent dead end.
|
||||
if (toolCallsState.some((state) => state?.emittedStart)) {
|
||||
output.stopReason = "toolUse";
|
||||
}
|
||||
// Otherwise keep whatever finish_reason set upstream (defaults to "stop").
|
||||
|
||||
@@ -52,6 +52,12 @@ export function getContentText(msg: Message): string {
|
||||
return "";
|
||||
}
|
||||
|
||||
/** The image blocks of a message, in order. Empty when there are none. */
|
||||
export function getContentImages(msg: Message): ImageContent[] {
|
||||
if (!Array.isArray(msg.content)) return [];
|
||||
return msg.content.filter((c): c is ImageContent => c.type === "image");
|
||||
}
|
||||
|
||||
export function transformTools(tools: Tool[]): QoderTool[] {
|
||||
return tools.map((t) => ({
|
||||
type: "function",
|
||||
@@ -155,6 +161,36 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] {
|
||||
tool_call_id: tr.toolCallId,
|
||||
content: getContentText(tr),
|
||||
});
|
||||
|
||||
// A tool result may carry images — pi's `read` tool returns a text note
|
||||
// plus an `image` block for png/jpg/gif/webp/bmp, and screenshot tools do
|
||||
// the same. getContentText() maps every non-text block to "", so those
|
||||
// images were dropped silently: the TUI rendered the picture while the
|
||||
// model received only "Read image file [image/png]" and reported that it
|
||||
// could not see images.
|
||||
//
|
||||
// The OpenAI-shaped `tool` role has nowhere to put them — its content is
|
||||
// a plain string — so they follow as a separate user message, the same
|
||||
// shape the user branch above already builds. The leading label keeps the
|
||||
// model from reading a bare image as something the human just sent.
|
||||
const images = getContentImages(tr);
|
||||
if (images.length > 0) {
|
||||
normalizedMessages.push({
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `[${images.length} image${images.length === 1 ? "" : "s"} returned by the previous tool call]`,
|
||||
},
|
||||
...images.map(
|
||||
(img): QoderImagePart => ({
|
||||
type: "image_url",
|
||||
image_url: { url: `data:${img.mimeType};base64,${img.data}` },
|
||||
}),
|
||||
),
|
||||
],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user