Files
pi-provider-qoder/src/protocol/stream.ts
T
cat-shark e7d12b5532 fix: retry instead of stalling when DSML tool-call markup leaks
DeepSeek V4 sometimes expresses a tool call as its internal DSML markup
instead of filling `tool_calls`, and Qoder's gateway leaks the tag skeleton
rather than converting it. When no structured call arrives the turn used to
end as an empty `stop` and the agent stopped mid-task with no error.

Handle both shapes:

- Filter the delimited form (`||DSML||tool_calls` ...) on the reasoning and
  content channels, rebuilding a complete block into a structured call.
- Strip the delimiter-less form the gateway leaves behind
  (`<parameter name="command">…</parameter>`, `</invoke>`, `</tool_calls>`,
  a stray `</thinking>`) line-wise before the empty-turn guard reads the
  message, and make `hasVisibleText` ignore markup-only text. That form used
  to land in the *content* channel as a normal reply, so the guard saw
  visible text and never fired.

A turn that produces only leaked markup now fails with a retryable error
(matching pi's "ended without" pattern) instead of ending silently.
2026-09-24 17:17:18 +08:00

689 lines
28 KiB
TypeScript

import crypto from "node:crypto";
import * as PiAi from "@earendil-works/pi-ai";
import {
type Api,
type AssistantMessage,
type AssistantMessageEventStream,
type Context,
clampThinkingLevel,
type Model,
type SimpleStreamOptions,
type TextContent,
type ThinkingContent,
type ToolCall,
} from "@earendil-works/pi-ai";
import { resolveQoderIdentity } from "../auth/oauth.js";
import { getCachedModelConfig, MAX_OUTPUT_TOKENS } from "../catalog.js";
import { buildAuthHeaders, getMachineId } from "../cosy.js";
import { getQoderChatURL, getQoderRegionConfig } from "../region.js";
import { DsmlFilter, type DsmlToolCall, isLeakedToolMarkupLine, stripLeakedToolMarkup } from "./dsml.js";
import { qoderEncodeBody } from "./encoding.js";
import { stripThinkingTags, ThinkingTagParser } from "./thinking.js";
import { transformMessagesForQoder, transformTools } from "./transform.js";
interface ToolCallState {
arguments: string;
id: string;
name: string;
emittedStart?: boolean;
emittedEnd?: boolean;
contentIndex: number;
}
function stableHash(prefix: string, ...inputs: string[]): string {
const hash = crypto.createHash("sha256");
hash.update(prefix);
for (const input of inputs) {
hash.update("\0");
hash.update(input);
}
return hash.digest("hex").slice(0, 16);
}
function stableChatRecordID(
model: string,
messages: Array<{ role?: string; content?: unknown }>,
tools: unknown,
maxTokens: number,
): string {
const hash = crypto.createHash("sha256");
hash.update("qoder-record");
hash.update("\0");
hash.update(model);
for (const msg of messages) {
if (msg?.role) {
hash.update("\0");
hash.update(msg.role);
}
if (msg?.content) {
hash.update("\0");
hash.update(typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content));
}
}
if (tools) {
hash.update("\0");
hash.update(JSON.stringify(tools));
}
hash.update("\0");
hash.update(`mt=${maxTokens}`);
return hash.digest("hex").slice(0, 16);
}
function contentToText(content: unknown): string {
if (typeof content === "string") return content;
if (Array.isArray(content)) {
return content
.map((part) => {
if (typeof part === "string") return part;
if (part && typeof part === "object" && "text" in part) return (part as { text: string }).text;
return "";
})
.join("\n");
}
return "";
}
/**
* True when the assistant produced text a user can actually read. Leaked
* tool-call markup is protocol noise, not an answer: a turn whose only "text"
* is marker fragments must still reach the retryable empty-turn guard below.
*/
function hasVisibleText(message: AssistantMessage): boolean {
return message.content.some(
(block) =>
block.type === "text" &&
block.text.split("\n").some((line) => line.trim().length > 0 && !isLeakedToolMarkupLine(line.trim())),
);
}
export function streamQoder(
model: Model<Api>,
context: Context,
options?: SimpleStreamOptions,
): AssistantMessageEventStream {
const StreamCtor = (PiAi as unknown as { AssistantMessageEventStream: new () => AssistantMessageEventStream })
.AssistantMessageEventStream;
const stream = new StreamCtor();
const output: AssistantMessage = {
role: "assistant",
content: [],
api: model.api,
provider: model.provider,
model: model.id,
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
stopReason: "stop",
timestamp: Date.now(),
};
(async () => {
try {
const providerMode = model.provider === "qoder-cn" ? "cn" : "global";
const region = getQoderRegionConfig(providerMode);
const accessToken = options?.apiKey;
if (!accessToken) {
throw new Error(
providerMode === "cn"
? "Qoder CN credentials not set. Run /login qoder-cn or set QODERCN_PERSONAL_ACCESS_TOKEN."
: "Qoder credentials not set. Run /login qoder or set QODER_PERSONAL_ACCESS_TOKEN.",
);
}
// Resolve the real Qoder identity from the job token. OMP keeps login
// credentials in its own agent.db, not in ~/.pi/agent/auth.json, so a
// cache miss would otherwise send uid "qoder-user" and Qoder CN rejects
// it with "Login expired" (105).
const ident = await resolveQoderIdentity(accessToken, model.provider, providerMode);
const userID = ident.userID || "qoder-user";
const name = ident.name || region.userNameFallback;
const email = ident.email || region.userEmailFallback;
const machineID = ident.machineID || getMachineId();
// Both providers expose the upstream display_name (whitespace stripped)
// as the pi id. Read the original key from cached/static config so the
// gateway still receives identifiers such as `lite` or `qmodel`.
const modelConfig = getCachedModelConfig(model.id, providerMode);
if (!modelConfig?.key) {
throw new Error(`Unknown Qoder model id: ${model.id}`);
}
const qoderModel = modelConfig.key;
const isReasoning = !!modelConfig.is_reasoning;
const normalizedMessages = transformMessagesForQoder(context.messages);
// OMP may supply the system prompt as a single-element content array;
// Qoder MessagesInputDto#content is a String and rejects an array with
// "Execution failed: set property ... MessagesInputDto#content". Normalize.
const systemText = contentToText(context.systemPrompt || "");
let lastUserText = "";
for (let i = normalizedMessages.length - 1; i >= 0; i--) {
if (normalizedMessages[i].role === "user") {
const content = normalizedMessages[i].content;
lastUserText =
typeof content === "string"
? content
: Array.isArray(content)
? content.map((c) => ("text" in c ? c.text : "")).join("")
: "";
break;
}
}
// Use a stable session id when pi provides one (per agent session) so
// the Qoder server can maintain prompt cache affinity across consecutive
// requests. Fall back to a random id only when no sessionId is available.
const stablePart = stableHash("qoder-session", userID, qoderModel);
const sessionID = options?.sessionId
? `${stablePart}-${options.sessionId}`
: `${stablePart}-${crypto.randomUUID()}`;
// Qoder's catalog exposes no per-model output cap, so we use the
// documented upstream ceiling (MAX_OUTPUT_TOKENS = 131072, see models.ts)
// and let pi cap it lower when the caller sets options.maxTokens (e.g.
// compaction at 40K). This avoids truncating reasoning chains / long
// generations that the 32K default would cut off.
let maxTokens = MAX_OUTPUT_TOKENS;
if (options?.maxTokens && options.maxTokens < maxTokens) {
maxTokens = options.maxTokens;
}
const toolsRaw = context.tools && context.tools.length > 0 ? transformTools(context.tools) : undefined;
const recordID = stableChatRecordID(qoderModel, normalizedMessages, toolsRaw, maxTokens);
// Map pi's thinking level (options.reasoning) to Qoder's request fields.
// Confirmed from @qoder-ai/qodercli: the chat body carries `reasoning_effort`
// ("none"|"low"|"medium"|"high"|"xhigh"|"max") and `enable_thinking` (bool)
// inside `parameters`, alongside `max_tokens`.
//
// This mirrors the pattern the pi-ai OpenAI provider uses: clamp the
// requested level to what the model advertises via thinkingLevelMap, then
// map to the upstream effort name. clampThinkingLevel returns "off" when
// the level is unsupported or the user disabled thinking.
const requestedLevel = options?.reasoning;
const clamped = requestedLevel ? clampThinkingLevel(model, requestedLevel) : undefined;
const reasoningLevel = clamped === "off" ? undefined : clamped;
const parameters: Record<string, unknown> = { max_tokens: maxTokens };
if (reasoningLevel) {
parameters.enable_thinking = true;
// Effort-based models advertise concrete effort names in the map
// (low/medium/xhigh/max). Toggle-only models map every level to
// "enabled"/"disabled" and accept no effort value — only the on/off
// switch matters, so we send enable_thinking alone.
const mapped = model.thinkingLevelMap?.[reasoningLevel];
const effort = mapped && mapped !== "enabled" && mapped !== "disabled" ? mapped : reasoningLevel;
// Only send reasoning_effort when the upstream model actually exposes
// effort levels (thinking_config.enabled.efforts).
if (modelConfig?.thinking_config?.enabled?.efforts && typeof effort === "string") {
parameters.reasoning_effort = effort;
}
} else {
// No reasoning level selected (or clamped to off): explicitly disable
// thinking so the model does not reason by default.
parameters.enable_thinking = false;
}
const reqBody: Record<string, unknown> = {
request_id: crypto.randomUUID(),
request_set_id: recordID,
chat_record_id: recordID,
session_id: sessionID,
stream: true,
chat_task: "FREE_INPUT",
is_reply: true,
is_retry: false,
source: 1,
version: "3",
session_type: "qodercli",
agent_id: "agent_common",
task_id: "common",
code_language: "",
chat_prompt: "",
image_urls: null,
aliyun_user_type: "",
// Qoder's server ignores the top-level `system` field (verified: the
// model never sees it). Inject the system prompt as a leading
// role:system message instead, which the server does honor.
system: "",
messages: systemText ? [{ role: "system", content: systemText }, ...normalizedMessages] : normalizedMessages,
tools: toolsRaw || [],
parameters,
chat_context: {
chatPrompt: "",
imageUrls: null,
extra: {
context: [],
modelConfig: {
key: qoderModel,
is_reasoning: isReasoning,
},
originalContent: lastUserText,
},
features: [],
text: lastUserText,
},
model_config: modelConfig,
business: {
product: "cli",
version: "1.0.0",
type: "agent",
stage: "start",
id: crypto.randomUUID(),
name: lastUserText.substring(0, 30),
begin_at: Date.now(),
},
};
const bodyBytes = Buffer.from(JSON.stringify(reqBody));
const encodedBytes = qoderEncodeBody(bodyBytes);
const chatURL = getQoderChatURL(providerMode);
const headers = buildAuthHeaders(encodedBytes, chatURL, {
userID,
authToken: accessToken,
name,
email,
machineID,
});
const modelSource = modelConfig.source || "system";
const response = await fetch(chatURL, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "text/event-stream",
"Cache-Control": "no-cache",
"Accept-Encoding": "identity",
"X-Model-Key": qoderModel,
"X-Model-Source": modelSource,
...headers,
},
body: encodedBytes as unknown as BodyInit,
signal: options?.signal,
});
if (!response.ok) {
const errText = await response.text();
throw new Error(`Qoder API request failed: ${response.status} ${response.statusText}. Response: ${errText}`);
}
const reader = response.body?.getReader();
if (!reader) throw new Error("No response body");
const decoder = new TextDecoder();
let buffer = "";
let bufferStart = 0;
let contentBlockIndex = -1;
let thinkingBlockIndex = -1;
const toolCallsState: ToolCallState[] = [];
const thinkingEnabled = (options?.reasoning as unknown) !== false && (options?.reasoning as unknown) !== "off";
const thinkingParser = thinkingEnabled ? new ThinkingTagParser(output, stream) : null;
const appendThinking = (chunk: string): void => {
if (thinkingBlockIndex === -1) {
thinkingBlockIndex = output.content.length;
output.content.push({ type: "thinking", thinking: "" });
stream.push({ type: "thinking_start", contentIndex: thinkingBlockIndex, partial: output });
}
const block = output.content[thinkingBlockIndex] as ThinkingContent;
block.thinking += chunk;
stream.push({ type: "thinking_delta", contentIndex: thinkingBlockIndex, delta: chunk, partial: output });
};
const appendContent = (chunk: string): void => {
// End the API reasoning block as soon as real visible text arrives.
if (thinkingBlockIndex !== -1) {
const block = output.content[thinkingBlockIndex] as ThinkingContent;
stream.push({
type: "thinking_end",
contentIndex: thinkingBlockIndex,
content: block.thinking,
partial: output,
});
thinkingBlockIndex = -1;
}
if (thinkingParser) {
thinkingParser.processChunk(chunk);
return;
}
if (contentBlockIndex === -1) {
contentBlockIndex = output.content.length;
output.content.push({ type: "text", text: "" });
stream.push({ type: "text_start", contentIndex: contentBlockIndex, partial: output });
}
const block = output.content[contentBlockIndex] as TextContent;
block.text += chunk;
stream.push({ type: "text_delta", contentIndex: contentBlockIndex, delta: chunk, partial: output });
};
// DeepSeek V4 falls back to its internal DSML tool-call markup instead of
// the structured `tool_calls` field now and then, and the leak can land in
// either channel. Strip it before it reaches the message and keep any
// rebuilt call for the finalizer, which only uses it when the gateway
// itself sent no structured tool call.
const reasoningDsml = new DsmlFilter(appendThinking);
// Visible content is only filtered for line-leading markers: a real leak
// always lands on its own line, while an answer that *explains* the markup
// quotes it inline and must stay readable.
const contentDsml = new DsmlFilter(appendContent, { preserveInlineMarkers: true });
stream.push({ type: "start", partial: output });
// `data: [DONE]` is the end of the response. Break the read loop too, not
// just the line loop: Qoder's gateway keeps the HTTP body open after the
// sentinel, so waiting for `done` from reader.read() hung until the
// server or the OS eventually closed the socket. The full reply had
// already been streamed by then, so the agent looked stuck with no error.
let sawDone = false;
while (!sawDone) {
const { done, value } = await reader.read();
if (done) break;
// Drop consumed prefix before appending so we do not keep growing a
// dead head of the string across chunks.
if (bufferStart > 0) {
buffer = buffer.substring(bufferStart);
bufferStart = 0;
}
buffer += decoder.decode(value, { stream: true });
while (true) {
const lineEnd = buffer.indexOf("\n", bufferStart);
if (lineEnd === -1) break;
const line = buffer.substring(bufferStart, lineEnd).trim();
bufferStart = lineEnd + 1;
if (!line.startsWith("data:")) continue;
const dataStr = line.substring(5).trim();
if (dataStr === "[DONE]") {
sawDone = true;
break;
}
try {
const envelope = JSON.parse(dataStr);
if (envelope.statusCodeValue && envelope.statusCodeValue !== 200) {
throw new Error(`Upstream status ${envelope.statusCodeValue}: ${envelope.body}`);
}
const innerStr = envelope.body;
// The gateway sends the sentinel wrapped in an envelope
// (`body: "[DONE]"`) as well as bare, and both mean the reply is
// over, so both have to end the read loop.
if (innerStr === "[DONE]") {
sawDone = true;
break;
}
if (!innerStr) continue;
const inner = JSON.parse(innerStr);
if (inner.id) output.responseId = inner.id as string;
if (inner.model) output.responseModel = inner.model as string;
if (inner.usage) {
const u = inner.usage as {
prompt_tokens?: number;
completion_tokens?: number;
total_tokens?: number;
completion_tokens_details?: { reasoning_tokens?: number };
prompt_tokens_details?: {
cacheable_tokens?: number;
cached_tokens?: number;
cache_write_tokens?: number;
};
};
// pi-core computes `promptTokens = input + cacheRead + cacheWrite`
// (Anthropic convention: `input` EXCLUDES cached/written tokens).
// Qoder follows OpenAI semantics where `prompt_tokens` INCLUDES
// `cached_tokens`, so subtract cacheRead (and cache_write_tokens
// when reported) to match the contract pi-ai's own OpenAI
// provider uses. `cacheable_tokens` is a capacity metric, not a
// write count (it is 0 even on first-turn writes), so it is NOT
// mapped to cacheWrite.
const promptTokens = u.prompt_tokens ?? 0;
const cacheReadTokens = u.prompt_tokens_details?.cached_tokens ?? 0;
const cacheWriteTokens = u.prompt_tokens_details?.cache_write_tokens ?? 0;
output.usage.input = Math.max(0, promptTokens - cacheReadTokens - cacheWriteTokens);
output.usage.output = u.completion_tokens ?? 0;
output.usage.totalTokens = u.total_tokens ?? 0;
output.usage.cacheRead = cacheReadTokens;
output.usage.cacheWrite = cacheWriteTokens;
}
if (inner.choices && inner.choices.length > 0) {
const choice = inner.choices[0];
const delta = choice.delta;
if (delta) {
// 1. Process reasoning/thinking content (API reasoning)
if (delta.reasoning_content) {
// Qoder's backend sometimes routes a literal `<thinking>`
// opener into reasoning_content (with the matching
// `</thinking>` closer landing in the content stream). Strip
// tag artifacts so the thinking block stays clean, matching
// the SDK's ContentBlock model. The DSML filter then drops
// leaked tool-call markup before it shows up as "thinking".
reasoningDsml.push(stripThinkingTags(delta.reasoning_content));
}
// 2. Process text content
if (delta.content) {
contentDsml.push(delta.content);
}
// 3. Process tool calls
if (delta.tool_calls && Array.isArray(delta.tool_calls)) {
for (const tc of delta.tool_calls) {
const idx = tc.index ?? 0;
if (!toolCallsState[idx]) {
toolCallsState[idx] = { arguments: "", id: "", name: "", contentIndex: 0 };
}
const state = toolCallsState[idx];
if (tc.id) state.id = tc.id;
if (tc.function?.name) state.name = tc.function.name;
// Open the block as soon as the call is IDENTIFIABLE, not
// when its first argument byte arrives. A call whose
// arguments are absent or an empty string — a no-argument
// tool, or a model that sends id+name and then stops — used
// to create a toolCallsState entry and no content block, so
// the finalizer below saw a non-empty state array, set
// stopReason "toolUse", and handed back a message with no
// tool call in it. The agent loop then had nothing to run
// and the turn simply ended, mid-task and without an error.
if (state.emittedStart === undefined && (state.id || state.name)) {
state.emittedStart = true;
state.contentIndex = output.content.length;
output.content.push({
type: "toolCall",
id: state.id,
name: state.name,
arguments: {},
} satisfies ToolCall);
stream.push({ type: "toolcall_start", contentIndex: state.contentIndex, partial: output });
}
// id and name can arrive after the block is open; keep it
// in step, since the finalizer only rewrites `arguments`.
if (state.emittedStart) {
const block = output.content[state.contentIndex] as ToolCall;
block.id = state.id;
block.name = state.name;
}
if (tc.function?.arguments) {
const argDelta = tc.function.arguments;
state.arguments += argDelta;
stream.push({
type: "toolcall_delta",
contentIndex: state.contentIndex,
delta: argDelta,
partial: output,
});
}
}
}
}
if (choice.finish_reason) {
// Preserve the real upstream finish_reason (e.g. "length",
// "content_filter") instead of forcing "stop" later.
output.stopReason = choice.finish_reason as AssistantMessage["stopReason"];
}
}
} catch (e) {
// A single malformed SSE line shouldn't kill the stream — skip it.
// But a genuine upstream error (thrown below) must propagate to the
// outer catch and surface as stopReason="error", not be swallowed.
if (e instanceof SyntaxError) {
if (process.env.QODER_DEBUG) {
console.error("[pi-provider-qoder] skipping malformed SSE line:", dataStr.slice(0, 200));
}
continue;
}
throw e;
}
}
}
// Stop reading and let the connection go once the reply is complete.
// Without this the body stays open until the server times it out.
await reader.cancel().catch(() => {});
// Flush the DSML filters first so anything they held back (and the
// thinking-tag parser's own buffer) still lands in the message.
reasoningDsml.finalize();
contentDsml.finalize();
if (thinkingParser) {
thinkingParser.finalize();
}
// Drop delimiter-less leaked tool-call markup before the guard below reads
// the message. Without this the fragment looks like visible text and the
// turn ends as a silent `stop` (see stripLeakedToolMarkup).
let sawLeakedMarkup = false;
for (const block of output.content) {
if (block.type !== "text" && block.type !== "thinking") continue;
const result = stripLeakedToolMarkup(block.type === "text" ? block.text : block.thinking);
if (!result.stripped) continue;
if (block.type === "text") block.text = result.text;
else block.thinking = result.text;
sawLeakedMarkup = true;
}
if (thinkingBlockIndex !== -1) {
const block = output.content[thinkingBlockIndex] as ThinkingContent;
stream.push({
type: "thinking_end",
contentIndex: thinkingBlockIndex,
content: block.thinking,
partial: output,
});
}
for (const state of toolCallsState) {
if (state?.emittedStart && !state.emittedEnd) {
state.emittedEnd = true;
let args = {};
try {
args = JSON.parse(state.arguments || "{}");
} catch {}
const block = output.content[state.contentIndex] as ToolCall;
block.arguments = args;
stream.push({
type: "toolcall_end",
contentIndex: state.contentIndex,
toolCall: {
type: "toolCall",
id: state.id,
name: state.name,
arguments: args,
},
partial: output,
});
}
}
// Guarded on blocks that actually reached the message, not on the state
// array being non-empty. Claiming "toolUse" for a message carrying no
// tool call is what turned a malformed stream into a silent dead end.
const structuredToolCalls = toolCallsState.some((state) => state?.emittedStart);
// The gateway dropped the structured call, but the raw DSML markup carried
// a complete one: rebuild it instead of losing the turn.
const rebuiltToolCalls: DsmlToolCall[] = structuredToolCalls
? []
: [...reasoningDsml.toolCalls, ...contentDsml.toolCalls];
for (const call of rebuiltToolCalls) {
const callContentIndex = output.content.length;
const toolCall: ToolCall = {
type: "toolCall",
id: call.id,
name: call.name,
arguments: call.arguments,
};
output.content.push(toolCall);
stream.push({ type: "toolcall_start", contentIndex: callContentIndex, partial: output });
stream.push({
type: "toolcall_delta",
contentIndex: callContentIndex,
delta: JSON.stringify(call.arguments),
partial: output,
});
stream.push({ type: "toolcall_end", contentIndex: callContentIndex, toolCall, partial: output });
}
if (structuredToolCalls || rebuiltToolCalls.length > 0) {
output.stopReason = "toolUse";
} else if (output.stopReason === "stop" && !options?.signal?.aborted && !hasVisibleText(output)) {
// Either the reply was empty, or the model emitted a tool call as raw
// DSML and the gateway never converted it (only stray closing markers
// arrive in that case, so nothing is reconstructable). Both mean the
// turn produced nothing to act on: report an error so the host retries
// instead of treating the empty message as a finished answer. The text
// below must keep matching pi's retryable-error pattern ("ended
// without").
output.stopReason = "error";
output.errorMessage =
"Qoder stream ended without a tool call or visible text" +
(reasoningDsml.sawMarkup || contentDsml.sawMarkup || sawLeakedMarkup
? " (raw DeepSeek DSML tool-call markup leaked into the stream)"
: "") +
". Retrying the request.";
stream.push({ type: "error", reason: "error", error: output });
stream.end();
return;
}
// Otherwise keep whatever finish_reason set upstream (defaults to "stop").
// Never overwrite a meaningful finish_reason ("length", "content_filter",
// ...) with "stop".
stream.push({
type: "done",
reason: output.stopReason as Extract<AssistantMessage["stopReason"], "stop" | "length" | "toolUse">,
message: output,
});
stream.end();
} catch (e: unknown) {
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
output.errorMessage = e instanceof Error ? e.message : String(e);
stream.push({ type: "error", reason: output.stopReason, error: output });
try {
stream.end();
} catch {}
}
})();
return stream;
}