fix: keep a turn's tool results together when one returns an image
The OpenAI-shaped `tool` role content is a plain string, so a tool result carrying images (pi's read/screenshot tools) follows as an extra user message. It was emitted immediately after its own tool message, which broke multi-call turns: assistant(tool_calls=[a, b]) -> tool(a) -> user(image) -> tool(b) Qoder rejects that with "An assistant message with 'tool_calls' must be followed by tool messages responding to each 'tool_call_id' (insufficient tool messages following tool_calls message)", leaving the session unable to send any further request. Queue the image follow-up and flush it once the run of consecutive tool results ends, so every tool_call_id is answered before the user message.
This commit is contained in:
@@ -276,6 +276,42 @@ describe("transformMessagesForQoder", () => {
|
|||||||
expect(result[0].role).toBe("tool");
|
expect(result[0].role).toBe("tool");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps every tool result of one assistant turn together when a sibling returns an image", () => {
|
||||||
|
// Regression: the image follow-up must not land between the tool results of
|
||||||
|
// a multi-call assistant turn. `assistant(tool_calls) -> tool(a) -> user ->
|
||||||
|
// tool(b)` makes Qoder reject the request with "An assistant message with
|
||||||
|
// 'tool_calls' must be followed by tool messages responding to each
|
||||||
|
// 'tool_call_id' (insufficient tool messages following tool_calls message)".
|
||||||
|
const msgs = [
|
||||||
|
{
|
||||||
|
role: "assistant",
|
||||||
|
content: [
|
||||||
|
{ type: "toolCall", id: "call_a", name: "read", arguments: { path: "/tmp/a.png" } },
|
||||||
|
{ type: "toolCall", id: "call_b", name: "read", arguments: { path: "/tmp/b.md" } },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: "toolResult",
|
||||||
|
toolCallId: "call_a",
|
||||||
|
content: [
|
||||||
|
{ type: "text", text: "Read image file [image/png]" },
|
||||||
|
{ type: "image", data: "abc123", mimeType: "image/png" },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
role: "toolResult",
|
||||||
|
toolCallId: "call_b",
|
||||||
|
content: [{ type: "text", text: "# notes" }],
|
||||||
|
},
|
||||||
|
] as unknown as Message[];
|
||||||
|
const result = transformMessagesForQoder(msgs);
|
||||||
|
|
||||||
|
expect(result.map((m) => m.role)).toEqual(["assistant", "tool", "tool", "user"]);
|
||||||
|
expect(result[1].tool_call_id).toBe("call_a");
|
||||||
|
expect(result[2].tool_call_id).toBe("call_b");
|
||||||
|
expect(result[3].content).toEqual(expect.arrayContaining([expect.objectContaining({ type: "image_url" })]));
|
||||||
|
});
|
||||||
|
|
||||||
it("handles assistant message with string content", () => {
|
it("handles assistant message with string content", () => {
|
||||||
const msgs = [
|
const msgs = [
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -78,12 +78,26 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] {
|
|||||||
// with tool_calls".
|
// with tool_calls".
|
||||||
const droppedToolCallIds = new Set<string>();
|
const droppedToolCallIds = new Set<string>();
|
||||||
|
|
||||||
|
// A tool result that carries images adds a follow-up user message (the `tool`
|
||||||
|
// role is a plain string in the OpenAI shape). It must not be emitted between
|
||||||
|
// the tool results of one assistant turn: an upstream rejects
|
||||||
|
// `assistant(tool_calls) -> tool(a) -> user -> tool(b)` with
|
||||||
|
// "An assistant message with 'tool_calls' must be followed by tool messages
|
||||||
|
// responding to each 'tool_call_id'". Queue it and flush once the run of
|
||||||
|
// consecutive tool results ends.
|
||||||
|
let pendingImageMessages: QoderMessage[] = [];
|
||||||
|
const flushPendingImageMessages = (): void => {
|
||||||
|
if (pendingImageMessages.length === 0) return;
|
||||||
|
normalizedMessages.push(...pendingImageMessages);
|
||||||
|
pendingImageMessages = [];
|
||||||
|
};
|
||||||
|
|
||||||
for (const msg of messages) {
|
for (const msg of messages) {
|
||||||
// Skip error or aborted messages
|
// Skip error or aborted messages
|
||||||
if (
|
const skipAbortedAssistant =
|
||||||
msg.role === "assistant" &&
|
msg.role === "assistant" &&
|
||||||
((msg as AssistantMessage).stopReason === "error" || (msg as AssistantMessage).stopReason === "aborted")
|
((msg as AssistantMessage).stopReason === "error" || (msg as AssistantMessage).stopReason === "aborted");
|
||||||
) {
|
if (skipAbortedAssistant) {
|
||||||
const am = msg as AssistantMessage;
|
const am = msg as AssistantMessage;
|
||||||
if (Array.isArray(am.content)) {
|
if (Array.isArray(am.content)) {
|
||||||
for (const block of am.content) {
|
for (const block of am.content) {
|
||||||
@@ -93,7 +107,6 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Drop the result too, otherwise it refers to a tool_calls entry that is
|
// Drop the result too, otherwise it refers to a tool_calls entry that is
|
||||||
@@ -102,6 +115,10 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// End the current tool-result run (kept open above) before the next turn.
|
||||||
|
if (msg.role !== "toolResult") flushPendingImageMessages();
|
||||||
|
if (skipAbortedAssistant) continue;
|
||||||
|
|
||||||
if (msg.role === "user") {
|
if (msg.role === "user") {
|
||||||
let content: QoderContent = "";
|
let content: QoderContent = "";
|
||||||
if (typeof msg.content === "string") {
|
if (typeof msg.content === "string") {
|
||||||
@@ -196,7 +213,7 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] {
|
|||||||
// model from reading a bare image as something the human just sent.
|
// model from reading a bare image as something the human just sent.
|
||||||
const images = getContentImages(tr);
|
const images = getContentImages(tr);
|
||||||
if (images.length > 0) {
|
if (images.length > 0) {
|
||||||
normalizedMessages.push({
|
pendingImageMessages.push({
|
||||||
role: "user",
|
role: "user",
|
||||||
content: [
|
content: [
|
||||||
{
|
{
|
||||||
@@ -215,5 +232,6 @@ export function transformMessagesForQoder(messages: Message[]): QoderMessage[] {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
flushPendingImageMessages();
|
||||||
return normalizedMessages;
|
return normalizedMessages;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user