A tool result can carry images. pi's `read` tool returns a text note plus an `image` block for png/jpg/gif/webp/bmp, and screenshot tools do the same. transformMessagesForQoder() built the `tool` message with getContentText(), which maps every non-text block to "". The images were dropped silently: the TUI rendered the picture, the model received only "Read image file [image/png]" and answered that it could not see images — then fell back to scraping the DOM. Nothing reported a problem, and the model catalogue advertises input ["text", "image"], so the capability check upstream of this had no reason to warn either. The OpenAI-shaped `tool` role has nowhere to put them: its content is a plain string. So the images now follow as a separate user message, using the same image_url part the user branch already builds, with a short label so the model does not read a bare image as something the human just sent. Tool results without images are unchanged and still produce exactly one message.
199 lines
6.4 KiB
TypeScript
199 lines
6.4 KiB
TypeScript
import type {
|
|
AssistantMessage,
|
|
ImageContent,
|
|
Message,
|
|
TextContent,
|
|
ThinkingContent,
|
|
Tool,
|
|
ToolCall,
|
|
ToolResultMessage,
|
|
} from "@earendil-works/pi-ai";
|
|
|
|
/** OpenAI-style tool definition sent to the Qoder API. */
|
|
interface QoderTool {
|
|
type: "function";
|
|
function: {
|
|
name: string;
|
|
description?: string;
|
|
parameters?: unknown;
|
|
};
|
|
}
|
|
|
|
/** OpenAI-style tool call within an assistant message. */
|
|
interface QoderToolCall {
|
|
id?: string;
|
|
type: "function";
|
|
function: { name?: string; arguments: string };
|
|
}
|
|
|
|
type QoderTextPart = { type: "text"; text: string };
|
|
type QoderImagePart = { type: "image_url"; image_url: { url: string } };
|
|
type QoderContent = string | Array<QoderTextPart | QoderImagePart>;
|
|
|
|
/** OpenAI-style message sent to the Qoder API. */
|
|
interface QoderMessage {
|
|
role: "user" | "assistant" | "tool" | "system";
|
|
content: QoderContent | null;
|
|
tool_calls?: QoderToolCall[];
|
|
tool_call_id?: string;
|
|
}
|
|
|
|
export function getContentText(msg: Message): string {
|
|
if (typeof msg.content === "string") return msg.content;
|
|
if (Array.isArray(msg.content)) {
|
|
return msg.content
|
|
.map((c) => {
|
|
if (c.type === "text") return (c as TextContent).text;
|
|
if (c.type === "thinking") return (c as ThinkingContent).thinking;
|
|
return "";
|
|
})
|
|
.join("");
|
|
}
|
|
return "";
|
|
}
|
|
|
|
/** The image blocks of a message, in order. Empty when there are none. */
|
|
export function getContentImages(msg: Message): ImageContent[] {
|
|
if (!Array.isArray(msg.content)) return [];
|
|
return msg.content.filter((c): c is ImageContent => c.type === "image");
|
|
}
|
|
|
|
export function transformTools(tools: Tool[]): QoderTool[] {
|
|
return tools.map((t) => ({
|
|
type: "function",
|
|
function: {
|
|
name: t.name,
|
|
description: t.description,
|
|
parameters: t.parameters,
|
|
},
|
|
}));
|
|
}
|
|
|
|
export function transformMessagesForQoder(messages: Message[]): QoderMessage[] {
|
|
const normalizedMessages: QoderMessage[] = [];
|
|
|
|
for (const msg of messages) {
|
|
// Skip error or aborted messages
|
|
if (
|
|
msg.role === "assistant" &&
|
|
((msg as AssistantMessage).stopReason === "error" || (msg as AssistantMessage).stopReason === "aborted")
|
|
) {
|
|
continue;
|
|
}
|
|
|
|
if (msg.role === "user") {
|
|
let content: QoderContent = "";
|
|
if (typeof msg.content === "string") {
|
|
content = msg.content;
|
|
} else if (Array.isArray(msg.content)) {
|
|
const hasImage = msg.content.some((c) => c.type === "image");
|
|
if (hasImage) {
|
|
content = msg.content
|
|
.map((c): QoderTextPart | QoderImagePart | null => {
|
|
if (c.type === "text") {
|
|
return { type: "text", text: (c as TextContent).text };
|
|
}
|
|
if (c.type === "image") {
|
|
const img = c as ImageContent;
|
|
return {
|
|
type: "image_url",
|
|
image_url: {
|
|
url: `data:${img.mimeType};base64,${img.data}`,
|
|
},
|
|
};
|
|
}
|
|
return null;
|
|
})
|
|
.filter((p): p is QoderTextPart | QoderImagePart => p !== null);
|
|
} else {
|
|
content = getContentText(msg);
|
|
}
|
|
}
|
|
normalizedMessages.push({
|
|
role: "user",
|
|
content,
|
|
});
|
|
} else if (msg.role === "assistant") {
|
|
const am = msg as AssistantMessage;
|
|
let content = "";
|
|
const toolCalls: QoderToolCall[] = [];
|
|
|
|
if (Array.isArray(am.content)) {
|
|
for (const block of am.content) {
|
|
if (block.type === "text") {
|
|
content += (block as TextContent).text;
|
|
} else if (block.type === "thinking") {
|
|
// Include thinking tags if reasoning is on
|
|
content += `<thinking>${(block as ThinkingContent).thinking}</thinking>\n\n`;
|
|
} else if (block.type === "toolCall") {
|
|
const tc = block as ToolCall;
|
|
toolCalls.push({
|
|
id: tc.id,
|
|
type: "function",
|
|
function: {
|
|
name: tc.name,
|
|
arguments: typeof tc.arguments === "string" ? tc.arguments : JSON.stringify(tc.arguments),
|
|
},
|
|
});
|
|
}
|
|
}
|
|
} else {
|
|
content = am.content || "";
|
|
}
|
|
|
|
// Qoder's gateway drops assistant messages whose content is null, which
|
|
// orphans the following tool_result and makes dmodel/ultimate upstreams
|
|
// reject the request ("tool must follow a message with tool_calls").
|
|
// When an assistant turn has tool calls but no text/thinking, inject a
|
|
// single-space placeholder so the gateway keeps the message.
|
|
const mapped: QoderMessage = {
|
|
role: "assistant",
|
|
content: content || (toolCalls.length > 0 ? " " : null),
|
|
};
|
|
if (toolCalls.length > 0) {
|
|
mapped.tool_calls = toolCalls;
|
|
}
|
|
normalizedMessages.push(mapped);
|
|
} else if (msg.role === "toolResult") {
|
|
const tr = msg as ToolResultMessage;
|
|
normalizedMessages.push({
|
|
role: "tool",
|
|
tool_call_id: tr.toolCallId,
|
|
content: getContentText(tr),
|
|
});
|
|
|
|
// A tool result may carry images — pi's `read` tool returns a text note
|
|
// plus an `image` block for png/jpg/gif/webp/bmp, and screenshot tools do
|
|
// the same. getContentText() maps every non-text block to "", so those
|
|
// images were dropped silently: the TUI rendered the picture while the
|
|
// model received only "Read image file [image/png]" and reported that it
|
|
// could not see images.
|
|
//
|
|
// The OpenAI-shaped `tool` role has nowhere to put them — its content is
|
|
// a plain string — so they follow as a separate user message, the same
|
|
// shape the user branch above already builds. The leading label keeps the
|
|
// model from reading a bare image as something the human just sent.
|
|
const images = getContentImages(tr);
|
|
if (images.length > 0) {
|
|
normalizedMessages.push({
|
|
role: "user",
|
|
content: [
|
|
{
|
|
type: "text",
|
|
text: `[${images.length} image${images.length === 1 ? "" : "s"} returned by the previous tool call]`,
|
|
},
|
|
...images.map(
|
|
(img): QoderImagePart => ({
|
|
type: "image_url",
|
|
image_url: { url: `data:${img.mimeType};base64,${img.data}` },
|
|
}),
|
|
),
|
|
],
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
return normalizedMessages;
|
|
}
|