/**
* Live reproduction of the `` leak against the real Qoder gateway.
*
* What this verifies:
* 1. Whether the server actually splits a `...` pair
* across `delta.reasoning_content` (opener) and `delta.content` (closer)
* — the M1 condition that caused `` to leak into visible text.
* 2. That the NEW ThinkingTagParser + stripThinkingTags produce clean output
* (no literal tags) on the real captured stream.
* 3. For comparison, that the OLD parser logic leaks `` into text
* on the same stream.
*
* Usage: npx tsx scripts/live-thinking-leak-test.ts [model] [prompt]
* model defaults to "efficient" (the model that leaked historically)
*/
import { readFileSync } from "node:fs";
import { homedir } from "node:os";
import { join } from "node:path";
import type { AssistantMessage, AssistantMessageEvent, AssistantMessageEventStream } from "@earendil-works/pi-ai";
import { buildAuthHeaders, getQoderChatURL, getQoderMode } from "../src/cosy.js";
import { qoderEncodeBody } from "../src/qoder-encoding.js";
import { transformMessagesForQoder } from "../src/transform.js";
import {
THINKING_TAG_VARIANTS,
ThinkingTagParser,
stripThinkingTags,
} from "../src/thinking-parser.js";
const AUTH_FILE = join(homedir(), ".pi", "agent", "auth.json");
const MODEL = process.argv[2] || "efficient";
const PROMPT =
process.argv[3] ||
"A bat and a ball cost $1.10 in total. The bat costs $1.00 more than the ball. How much does the ball cost? Reason carefully step by step before answering.";
interface QoderCreds {
access: string;
userID: string;
name: string;
email: string;
machineID?: string;
}
function loadCreds(): QoderCreds {
const auth = JSON.parse(readFileSync(AUTH_FILE, "utf-8"));
const c = auth.qoder;
if (!c?.access) throw new Error("No qoder credentials in ~/.pi/agent/auth.json — run `pi login` first");
return c;
}
function uuid() {
return globalThis.crypto.randomUUID();
}
function buildRequestBody(model: string, messages: ReturnType, isReasoning: boolean) {
return {
request_id: uuid(),
request_set_id: uuid(),
chat_record_id: uuid(),
session_id: `qoder-session-${model}-${uuid()}`,
stream: true,
chat_task: "FREE_INPUT",
is_reply: true,
is_retry: false,
source: 1,
version: "3",
session_type: "qodercli",
agent_id: "agent_common",
task_id: "common",
code_language: "",
chat_prompt: "",
image_urls: null,
aliyun_user_type: "",
system: "",
messages,
tools: [],
parameters: { max_tokens: 1024 },
chat_context: {
chatPrompt: "",
imageUrls: null,
extra: { context: [], modelConfig: { key: model, is_reasoning: isReasoning }, originalContent: PROMPT },
features: [],
text: PROMPT,
},
model_config: { key: model, is_reasoning: isReasoning, source: "system" },
business: { product: "cli", version: "1.0.0", type: "agent", stage: "start", id: uuid(), name: PROMPT.slice(0, 30), begin_at: Date.now() },
};
}
interface RawStream {
reasoningChunks: string[];
contentChunks: string[];
finishReason: string | null;
rawHead: string;
}
/** POST and collect every raw delta.reasoning_content / delta.content chunk. */
async function captureRawStream(model: string, isReasoning: boolean): Promise {
const mode = getQoderMode();
const url = getQoderChatURL(mode);
const creds = loadCreds();
const messages = transformMessagesForQoder([{ role: "user", content: PROMPT } as never]);
const body = buildRequestBody(model, messages, isReasoning);
const encodedBody = qoderEncodeBody(Buffer.from(JSON.stringify(body)));
const encodedBytes = Buffer.from(encodedBody, "utf8");
const headers = buildAuthHeaders(encodedBytes, url, {
userID: creds.userID,
authToken: creds.access,
name: creds.name,
email: creds.email,
machineID: creds.machineID,
});
console.log(`\nPOST ${url} model=${model} is_reasoning=${isReasoning}`);
const res = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
Accept: "text/event-stream",
"Cache-Control": "no-cache",
"Accept-Encoding": "identity",
"X-Model-Key": model,
"X-Model-Source": "system",
...headers,
},
body: encodedBytes,
});
const out: RawStream = { reasoningChunks: [], contentChunks: [], finishReason: null, rawHead: "" };
if (!res.ok || !res.body) {
const text = await res.text().catch(() => "");
out.rawHead = `HTTP ${res.status} ${res.statusText}: ${text.slice(0, 400)}`;
return out;
}
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buf = "";
let envelopeBuffer = "";
let headWritten = false;
while (true) {
const { done, value } = await reader.read();
if (done) break;
buf += decoder.decode(value, { stream: true });
let nl: number;
while ((nl = buf.indexOf("\n")) !== -1) {
const line = buf.slice(0, nl);
buf = buf.slice(nl + 1);
envelopeBuffer += line + "\n";
// SSE lines look like: data:{"statusCodeValue":200,"body":""}
const m = line.match(/^data:(.*)$/);
if (!m) continue;
const payload = m[1].trim();
if (!payload || payload === "[DONE]") continue;
try {
const envelope = JSON.parse(payload);
if (!headWritten) {
out.rawHead = `envelope statusCodeValue=${envelope.statusCodeValue} (first line ok)`;
headWritten = true;
}
if (envelope.statusCodeValue && envelope.statusCodeValue !== 200) {
out.rawHead = `Upstream ${envelope.statusCodeValue}: ${String(envelope.body).slice(0, 300)}`;
return out;
}
const innerStr = envelope.body;
if (!innerStr || innerStr === "[DONE]") continue;
const inner = JSON.parse(innerStr);
const choice = inner.choices?.[0];
const delta = choice?.delta;
if (delta?.reasoning_content) out.reasoningChunks.push(delta.reasoning_content);
if (delta?.content) out.contentChunks.push(delta.content);
if (choice?.finish_reason) out.finishReason = choice.finish_reason;
} catch {
// non-JSON line, skip
}
}
}
return out;
}
// ── OLD parser logic (faithful reimplementation of the pre-fix behavior) ──
function getTrailingPrefixLength(text: string, tag: string): number {
const max = Math.min(text.length, tag.length - 1);
for (let len = max; len > 0; len--) {
if (text.endsWith(tag.slice(0, len))) return len;
}
return 0;
}
function getMaxTrailingPrefix(text: string, tags: string[]): number {
let max = 0;
for (const t of tags) max = Math.max(max, getTrailingPrefixLength(text, t));
return max;
}
/** Old processBeforeThinking: openers only, NO orphan-closer handling. */
function oldParseContent(chunks: string[]): { thinking: string; text: string } {
let buffer = "";
let inThinking = false;
let activeClose = THINKING_TAG_VARIANTS[0].close;
let thinking = "";
let text = "";
for (const chunk of chunks) {
buffer += chunk;
let progress = true;
while (buffer.length > 0 && progress) {
progress = false;
if (!inThinking) {
let bestPos = -1;
let bestV: (typeof THINKING_TAG_VARIANTS)[number] | null = null;
for (const v of THINKING_TAG_VARIANTS) {
const p = buffer.indexOf(v.open);
if (p !== -1 && (bestPos === -1 || p < bestPos)) {
bestPos = p;
bestV = v;
}
}
if (bestV) {
if (bestPos > 0) text += buffer.slice(0, bestPos);
buffer = buffer.slice(bestPos + bestV.open.length);
activeClose = bestV.close;
inThinking = true;
progress = true;
continue;
}
// no opener → emit safe text holding back open-tag prefixes only
const trailing = getMaxTrailingPrefix(buffer, THINKING_TAG_VARIANTS.map((v) => v.open));
const safe = buffer.length - trailing;
if (safe > 0) {
text += buffer.slice(0, safe);
buffer = buffer.slice(safe);
progress = true;
}
} else {
const ep = buffer.indexOf(activeClose);
if (ep !== -1) {
if (ep > 0) thinking += buffer.slice(0, ep);
buffer = buffer.slice(ep + activeClose.length);
inThinking = false;
if (buffer.startsWith("\n\n")) buffer = buffer.slice(2);
progress = true;
continue;
}
const trailing = getTrailingPrefixLength(buffer, activeClose);
const safe = buffer.length - trailing;
if (safe > 0) {
thinking += buffer.slice(0, safe);
buffer = buffer.slice(safe);
progress = true;
}
}
}
}
if (inThinking) thinking += buffer;
else text += buffer;
return { thinking, text };
}
// ── NEW parser (the actual shipped class) ──
function newParseContent(chunks: string[]): { thinking: string; text: string } {
const output: AssistantMessage = {
role: "assistant",
content: [],
api: "qoder-api" as never,
provider: "qoder",
model: "efficient",
usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } },
stopReason: "stop",
timestamp: Date.now(),
} as AssistantMessage;
const events: AssistantMessageEvent[] = [];
const stream = {
push: (e: AssistantMessageEvent) => events.push(e),
end: () => {},
[Symbol.iterator]: function* () {},
[Symbol.asyncIterator]: function* () {},
events,
} as unknown as AssistantMessageEventStream;
const parser = new ThinkingTagParser(output, stream);
for (const c of chunks) parser.processChunk(c);
parser.finalize();
let thinking = "";
let text = "";
for (const block of output.content) {
if (block.type === "thinking") thinking += (block as { thinking: string }).thinking;
else if (block.type === "text") text += (block as { text: string }).text;
}
return { thinking, text };
}
function containsAnyTag(s: string, which: "open" | "close"): string[] {
const found: string[] = [];
for (const v of THINKING_TAG_VARIANTS) {
const tag = which === "open" ? v.open : v.close;
if (tag.length > 0 && s.includes(tag)) found.push(tag);
}
return found;
}
function banner(s: string) {
console.log("\n" + "─".repeat(70));
console.log(s);
console.log("─".repeat(70));
}
async function main() {
banner(`Live repro: leak | model=${MODEL}`);
console.log(`prompt: ${PROMPT.slice(0, 100)}${PROMPT.length > 100 ? "…" : ""}`);
// Try is_reasoning=false first (matches the historical leak config), then true.
for (const isReasoning of [false, true]) {
const raw = await captureRawStream(MODEL, isReasoning);
if (raw.rawHead && raw.reasoningChunks.length === 0 && raw.contentChunks.length === 0) {
console.log(`\n!! no deltas captured: ${raw.rawHead}`);
continue;
}
const reasoning = raw.reasoningChunks.join("");
const content = raw.contentChunks.join("");
console.log(
`\ncaptured: ${raw.reasoningChunks.length} reasoning chunks (${reasoning.length} chars), ` +
`${raw.contentChunks.length} content chunks (${content.length} chars), finish=${raw.finishReason}`,
);
if (reasoning.length === 0 && content.length === 0) {
console.log("(empty stream — nothing to analyze)");
continue;
}
// ── Detect M1: server split a tag pair across the two channels ──
const openInReasoning = containsAnyTag(reasoning, "open");
const closeInContent = containsAnyTag(content, "close");
const openInContent = containsAnyTag(content, "open");
const closeInReasoning = containsAnyTag(reasoning, "close");
const m1Split = openInReasoning.length > 0 && closeInContent.length > 0;
banner("M1 detection (tag split across reasoning_content / content)");
console.log(`reasoning_content has openers : ${openInReasoning.length ? openInReasoning.join(" ") : "(none)"}`);
console.log(`content has closers : ${closeInContent.length ? closeInContent.join(" ") : "(none)"}`);
console.log(`content has openers : ${openInContent.length ? openInContent.join(" ") : "(none)"}`);
console.log(`reasoning_content has closers: ${closeInReasoning.length ? closeInReasoning.join(" ") : "(none)"}`);
console.log(`\n>>> M1 split reproduced: ${m1Split ? "YES — server splits the tag pair across channels" : "NO (model did not emit split tags this run)"}`);
if (content.length > 0) {
banner("CONTENT channel — OLD vs NEW parser");
const oldC = oldParseContent(raw.contentChunks);
const newC = newParseContent(raw.contentChunks);
const oldLeakTags = [...containsAnyTag(oldC.text, "close"), ...containsAnyTag(oldC.text, "open")];
const newLeakTags = [...containsAnyTag(newC.text, "close"), ...containsAnyTag(newC.text, "open")];
console.log(`OLD text output : ${JSON.stringify(oldC.text.slice(0, 120))}${oldC.text.length > 120 ? "…" : ""}`);
console.log(`OLD leaks tags : ${oldLeakTags.length ? oldLeakTags.join(" ") : "(none) ← clean"}`);
console.log(`NEW text output : ${JSON.stringify(newC.text.slice(0, 120))}${newC.text.length > 120 ? "…" : ""}`);
console.log(`NEW leaks tags : ${newLeakTags.length ? newLeakTags.join(" ") : "(none) ← clean"}`);
}
if (reasoning.length > 0) {
banner("REASONING_CONTENT channel — OLD (verbatim) vs NEW (stripThinkingTags)");
const oldR = reasoning; // old stream.ts appended verbatim
const newR = stripThinkingTags(reasoning);
const oldRLeaks = containsAnyTag(oldR, "open").concat(containsAnyTag(oldR, "close"));
const newRLeaks = containsAnyTag(newR, "open").concat(containsAnyTag(newR, "close"));
console.log(`OLD reasoning head: ${JSON.stringify(oldR.slice(0, 80))}…`);
console.log(`OLD leaks tags : ${oldRLeaks.length ? oldRLeaks.join(" ") : "(none) ← clean"}`);
console.log(`NEW reasoning head: ${JSON.stringify(newR.slice(0, 80))}…`);
console.log(`NEW leaks tags : ${newRLeaks.length ? newRLeaks.join(" ") : "(none) ← clean"}`);
}
// If we got a real stream, don't bother retrying the other is_reasoning flag.
if (reasoning.length > 0 || content.length > 0) break;
}
banner("done");
}
main().catch((e) => {
console.error("FATAL:", e);
process.exit(1);
});