fix: strip thinking tag artifacts leaked into reasoning_content

Qoder's backend sometimes routes a literal <thinking> opener into
reasoning_content while the matching closer lands in the content
stream, leaving tag fragments in the thinking block. Strip them during
streaming and add live repro scripts.
This commit is contained in:
liyanan
2026-07-28 10:21:28 +08:00
parent 85ec02ca8a
commit 47141df66e
5 changed files with 761 additions and 29 deletions
+378
View File
@@ -0,0 +1,378 @@
/**
* Live reproduction of the `</thinking>` leak against the real Qoder gateway.
*
* What this verifies:
* 1. Whether the server actually splits a `<thinking>...</thinking>` pair
* across `delta.reasoning_content` (opener) and `delta.content` (closer)
* — the M1 condition that caused `</thinking>` to leak into visible text.
* 2. That the NEW ThinkingTagParser + stripThinkingTags produce clean output
* (no literal tags) on the real captured stream.
* 3. For comparison, that the OLD parser logic leaks `</thinking>` into text
* on the same stream.
*
* Usage: npx tsx scripts/live-thinking-leak-test.ts [model] [prompt]
* model defaults to "efficient" (the model that leaked historically)
*/
import { readFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import type { AssistantMessage, AssistantMessageEvent, AssistantMessageEventStream } from "@earendil-works/pi-ai";
import { buildAuthHeaders, getQoderChatURL, getQoderMode } from "../src/cosy.js";
import { qoderEncodeBody } from "../src/qoder-encoding.js";
import { transformMessagesForQoder } from "../src/transform.js";
import {
THINKING_TAG_VARIANTS,
ThinkingTagParser,
stripThinkingTags,
} from "../src/thinking-parser.js";
const AUTH_FILE = join(homedir(), ".pi", "agent", "auth.json");
const MODEL = process.argv[2] || "efficient";
const PROMPT =
process.argv[3] ||
"A bat and a ball cost $1.10 in total. The bat costs $1.00 more than the ball. How much does the ball cost? Reason carefully step by step before answering.";
interface QoderCreds {
access: string;
userID: string;
name: string;
email: string;
machineID?: string;
}
function loadCreds(): QoderCreds {
const auth = JSON.parse(readFileSync(AUTH_FILE, "utf-8"));
const c = auth.qoder;
if (!c?.access) throw new Error("No qoder credentials in ~/.pi/agent/auth.json — run `pi login` first");
return c;
}
function uuid() {
return globalThis.crypto.randomUUID();
}
function buildRequestBody(model: string, messages: ReturnType<typeof transformMessagesForQoder>, isReasoning: boolean) {
return {
request_id: uuid(),
request_set_id: uuid(),
chat_record_id: uuid(),
session_id: `qoder-session-${model}-${uuid()}`,
stream: true,
chat_task: "FREE_INPUT",
is_reply: true,
is_retry: false,
source: 1,
version: "3",
session_type: "qodercli",
agent_id: "agent_common",
task_id: "common",
code_language: "",
chat_prompt: "",
image_urls: null,
aliyun_user_type: "",
system: "",
messages,
tools: [],
parameters: { max_tokens: 1024 },
chat_context: {
chatPrompt: "",
imageUrls: null,
extra: { context: [], modelConfig: { key: model, is_reasoning: isReasoning }, originalContent: PROMPT },
features: [],
text: PROMPT,
},
model_config: { key: model, is_reasoning: isReasoning, source: "system" },
business: { product: "cli", version: "1.0.0", type: "agent", stage: "start", id: uuid(), name: PROMPT.slice(0, 30), begin_at: Date.now() },
};
}
interface RawStream {
reasoningChunks: string[];
contentChunks: string[];
finishReason: string | null;
rawHead: string;
}
/** POST and collect every raw delta.reasoning_content / delta.content chunk. */
async function captureRawStream(model: string, isReasoning: boolean): Promise<RawStream> {
const mode = getQoderMode();
const url = getQoderChatURL(mode);
const creds = loadCreds();
const messages = transformMessagesForQoder([{ role: "user", content: PROMPT } as never]);
const body = buildRequestBody(model, messages, isReasoning);
const encodedBody = qoderEncodeBody(Buffer.from(JSON.stringify(body)));
const encodedBytes = Buffer.from(encodedBody, "utf8");
const headers = buildAuthHeaders(encodedBytes, url, {
userID: creds.userID,
authToken: creds.access,
name: creds.name,
email: creds.email,
machineID: creds.machineID,
});
console.log(`\nPOST ${url} model=${model} is_reasoning=${isReasoning}`);
const res = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "text/event-stream",
"Cache-Control": "no-cache",
"Accept-Encoding": "identity",
"X-Model-Key": model,
"X-Model-Source": "system",
...headers,
},
body: encodedBytes,
});
const out: RawStream = { reasoningChunks: [], contentChunks: [], finishReason: null, rawHead: "" };
if (!res.ok || !res.body) {
const text = await res.text().catch(() => "");
out.rawHead = `HTTP ${res.status} ${res.statusText}: ${text.slice(0, 400)}`;
return out;
}
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buf = "";
let envelopeBuffer = "";
let headWritten = false;
while (true) {
const { done, value } = await reader.read();
if (done) break;
buf += decoder.decode(value, { stream: true });
let nl: number;
while ((nl = buf.indexOf("\n")) !== -1) {
const line = buf.slice(0, nl);
buf = buf.slice(nl + 1);
envelopeBuffer += line + "\n";
// SSE lines look like: data:{"statusCodeValue":200,"body":"<json string>"}
const m = line.match(/^data:(.*)$/);
if (!m) continue;
const payload = m[1].trim();
if (!payload || payload === "[DONE]") continue;
try {
const envelope = JSON.parse(payload);
if (!headWritten) {
out.rawHead = `envelope statusCodeValue=${envelope.statusCodeValue} (first line ok)`;
headWritten = true;
}
if (envelope.statusCodeValue && envelope.statusCodeValue !== 200) {
out.rawHead = `Upstream ${envelope.statusCodeValue}: ${String(envelope.body).slice(0, 300)}`;
return out;
}
const innerStr = envelope.body;
if (!innerStr || innerStr === "[DONE]") continue;
const inner = JSON.parse(innerStr);
const choice = inner.choices?.[0];
const delta = choice?.delta;
if (delta?.reasoning_content) out.reasoningChunks.push(delta.reasoning_content);
if (delta?.content) out.contentChunks.push(delta.content);
if (choice?.finish_reason) out.finishReason = choice.finish_reason;
} catch {
// non-JSON line, skip
}
}
}
return out;
}
// ── OLD parser logic (faithful reimplementation of the pre-fix behavior) ──
function getTrailingPrefixLength(text: string, tag: string): number {
const max = Math.min(text.length, tag.length - 1);
for (let len = max; len > 0; len--) {
if (text.endsWith(tag.slice(0, len))) return len;
}
return 0;
}
function getMaxTrailingPrefix(text: string, tags: string[]): number {
let max = 0;
for (const t of tags) max = Math.max(max, getTrailingPrefixLength(text, t));
return max;
}
/** Old processBeforeThinking: openers only, NO orphan-closer handling. */
function oldParseContent(chunks: string[]): { thinking: string; text: string } {
let buffer = "";
let inThinking = false;
let activeClose = THINKING_TAG_VARIANTS[0].close;
let thinking = "";
let text = "";
for (const chunk of chunks) {
buffer += chunk;
let progress = true;
while (buffer.length > 0 && progress) {
progress = false;
if (!inThinking) {
let bestPos = -1;
let bestV: (typeof THINKING_TAG_VARIANTS)[number] | null = null;
for (const v of THINKING_TAG_VARIANTS) {
const p = buffer.indexOf(v.open);
if (p !== -1 && (bestPos === -1 || p < bestPos)) {
bestPos = p;
bestV = v;
}
}
if (bestV) {
if (bestPos > 0) text += buffer.slice(0, bestPos);
buffer = buffer.slice(bestPos + bestV.open.length);
activeClose = bestV.close;
inThinking = true;
progress = true;
continue;
}
// no opener → emit safe text holding back open-tag prefixes only
const trailing = getMaxTrailingPrefix(buffer, THINKING_TAG_VARIANTS.map((v) => v.open));
const safe = buffer.length - trailing;
if (safe > 0) {
text += buffer.slice(0, safe);
buffer = buffer.slice(safe);
progress = true;
}
} else {
const ep = buffer.indexOf(activeClose);
if (ep !== -1) {
if (ep > 0) thinking += buffer.slice(0, ep);
buffer = buffer.slice(ep + activeClose.length);
inThinking = false;
if (buffer.startsWith("\n\n")) buffer = buffer.slice(2);
progress = true;
continue;
}
const trailing = getTrailingPrefixLength(buffer, activeClose);
const safe = buffer.length - trailing;
if (safe > 0) {
thinking += buffer.slice(0, safe);
buffer = buffer.slice(safe);
progress = true;
}
}
}
}
if (inThinking) thinking += buffer;
else text += buffer;
return { thinking, text };
}
// ── NEW parser (the actual shipped class) ──
function newParseContent(chunks: string[]): { thinking: string; text: string } {
const output: AssistantMessage = {
role: "assistant",
content: [],
api: "qoder-api" as never,
provider: "qoder",
model: "efficient",
usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } },
stopReason: "stop",
timestamp: Date.now(),
} as AssistantMessage;
const events: AssistantMessageEvent[] = [];
const stream = {
push: (e: AssistantMessageEvent) => events.push(e),
end: () => {},
[Symbol.iterator]: function* () {},
[Symbol.asyncIterator]: function* () {},
events,
} as unknown as AssistantMessageEventStream;
const parser = new ThinkingTagParser(output, stream);
for (const c of chunks) parser.processChunk(c);
parser.finalize();
let thinking = "";
let text = "";
for (const block of output.content) {
if (block.type === "thinking") thinking += (block as { thinking: string }).thinking;
else if (block.type === "text") text += (block as { text: string }).text;
}
return { thinking, text };
}
function containsAnyTag(s: string, which: "open" | "close"): string[] {
const found: string[] = [];
for (const v of THINKING_TAG_VARIANTS) {
const tag = which === "open" ? v.open : v.close;
if (tag.length > 0 && s.includes(tag)) found.push(tag);
}
return found;
}
function banner(s: string) {
console.log("\n" + "─".repeat(70));
console.log(s);
console.log("─".repeat(70));
}
async function main() {
banner(`Live repro: </thinking> leak | model=${MODEL}`);
console.log(`prompt: ${PROMPT.slice(0, 100)}${PROMPT.length > 100 ? "…" : ""}`);
// Try is_reasoning=false first (matches the historical leak config), then true.
for (const isReasoning of [false, true]) {
const raw = await captureRawStream(MODEL, isReasoning);
if (raw.rawHead && raw.reasoningChunks.length === 0 && raw.contentChunks.length === 0) {
console.log(`\n!! no deltas captured: ${raw.rawHead}`);
continue;
}
const reasoning = raw.reasoningChunks.join("");
const content = raw.contentChunks.join("");
console.log(
`\ncaptured: ${raw.reasoningChunks.length} reasoning chunks (${reasoning.length} chars), ` +
`${raw.contentChunks.length} content chunks (${content.length} chars), finish=${raw.finishReason}`,
);
if (reasoning.length === 0 && content.length === 0) {
console.log("(empty stream — nothing to analyze)");
continue;
}
// ── Detect M1: server split a tag pair across the two channels ──
const openInReasoning = containsAnyTag(reasoning, "open");
const closeInContent = containsAnyTag(content, "close");
const openInContent = containsAnyTag(content, "open");
const closeInReasoning = containsAnyTag(reasoning, "close");
const m1Split = openInReasoning.length > 0 && closeInContent.length > 0;
banner("M1 detection (tag split across reasoning_content / content)");
console.log(`reasoning_content has openers : ${openInReasoning.length ? openInReasoning.join(" ") : "(none)"}`);
console.log(`content has closers : ${closeInContent.length ? closeInContent.join(" ") : "(none)"}`);
console.log(`content has openers : ${openInContent.length ? openInContent.join(" ") : "(none)"}`);
console.log(`reasoning_content has closers: ${closeInReasoning.length ? closeInReasoning.join(" ") : "(none)"}`);
console.log(`\n>>> M1 split reproduced: ${m1Split ? "YES — server splits the tag pair across channels" : "NO (model did not emit split tags this run)"}`);
if (content.length > 0) {
banner("CONTENT channel — OLD vs NEW parser");
const oldC = oldParseContent(raw.contentChunks);
const newC = newParseContent(raw.contentChunks);
const oldLeakTags = [...containsAnyTag(oldC.text, "close"), ...containsAnyTag(oldC.text, "open")];
const newLeakTags = [...containsAnyTag(newC.text, "close"), ...containsAnyTag(newC.text, "open")];
console.log(`OLD text output : ${JSON.stringify(oldC.text.slice(0, 120))}${oldC.text.length > 120 ? "…" : ""}`);
console.log(`OLD leaks tags : ${oldLeakTags.length ? oldLeakTags.join(" ") : "(none) ← clean"}`);
console.log(`NEW text output : ${JSON.stringify(newC.text.slice(0, 120))}${newC.text.length > 120 ? "…" : ""}`);
console.log(`NEW leaks tags : ${newLeakTags.length ? newLeakTags.join(" ") : "(none) ← clean"}`);
}
if (reasoning.length > 0) {
banner("REASONING_CONTENT channel — OLD (verbatim) vs NEW (stripThinkingTags)");
const oldR = reasoning; // old stream.ts appended verbatim
const newR = stripThinkingTags(reasoning);
const oldRLeaks = containsAnyTag(oldR, "open").concat(containsAnyTag(oldR, "close"));
const newRLeaks = containsAnyTag(newR, "open").concat(containsAnyTag(newR, "close"));
console.log(`OLD reasoning head: ${JSON.stringify(oldR.slice(0, 80))}…`);
console.log(`OLD leaks tags : ${oldRLeaks.length ? oldRLeaks.join(" ") : "(none) ← clean"}`);
console.log(`NEW reasoning head: ${JSON.stringify(newR.slice(0, 80))}…`);
console.log(`NEW leaks tags : ${newRLeaks.length ? newRLeaks.join(" ") : "(none) ← clean"}`);
}
// If we got a real stream, don't bother retrying the other is_reasoning flag.
if (reasoning.length > 0 || content.length > 0) break;
}
banner("done");
}
main().catch((e) => {
console.error("FATAL:", e);
process.exit(1);
});