fix(stream): stop silently swallowing upstream errors and session-lock stalls
Three stability fixes for the streaming layer. Together they eliminate
the "agent silently stops mid-task" failure mode users hit during agentic
loops.
1. Surface upstream errors instead of swallowing them
The SSE parsing loop wrapped every line in try { ... } catch {}, which
silently dropped the throw raised when Qoder returned an error envelope
(e.g. 406 Session blocked, provider_error). The stream then completed
with empty content and stopReason "stop", so pi saw a clean stop instead
of an error and never retried or informed the user.
The catch now only skips genuinely malformed SSE lines (SyntaxError) and
re-throws everything else so the outer handler can set stopReason "error"
+ errorMessage. A QODER_DEBUG env var logs skipped lines for diagnosis.
2. Stop overwriting the real finish_reason with "stop"
At end-of-stream the code unconditionally did:
output.stopReason = toolCalls.length ? "toolUse" : "stop"
clobbering any meaningful finish_reason ("length", "content_filter", ...)
captured earlier. Now only toolUse is forced when tool calls are present;
otherwise the upstream finish_reason is preserved (defaults to "stop").
The done event reason cast now includes "length".
3. Give each request a fresh server-side session id
session_id was stableHash("qoder-session", userID, qoderModel) —
constant across all requests for a given user+model. Qoder enforces a
per-session lock and replies 406 Session blocked when a second request
arrives before the previous one fully drains server-side, which happens
routinely in tight agentic tool loops. Appending crypto.randomUUID()
makes every request use a fresh session. We send full conversation
history on each request, so losing server-side session continuity is
safe.
4. Capture upstream usage, responseId and responseModel
The provider reported all-zero usage and no responseId/responseModel.
Now the finish chunk usage (prompt/completion/total tokens, cache
read/write) is mapped into output.usage, and id/model into
responseId/responseModel, so pi sees real token accounting and can
correlate responses.
Add stream.test.ts with regression coverage for the 406 path, the
finish_reason preservation, the toolUse stop reason, and usage/response
metadata capture.
This commit is contained in:
@@ -0,0 +1,221 @@
|
|||||||
|
import type {
|
||||||
|
Api,
|
||||||
|
AssistantMessage,
|
||||||
|
AssistantMessageEvent,
|
||||||
|
AssistantMessageEventStream,
|
||||||
|
Context,
|
||||||
|
Model,
|
||||||
|
} from "@earendil-works/pi-ai";
|
||||||
|
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||||
|
import { streamQoder } from "../stream.js";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build a single SSE `data:` line carrying a Qoder envelope:
|
||||||
|
* { headers, body: <JSON string>, statusCodeValue, statusCode }
|
||||||
|
* The server wraps the OpenAI-style chunk inside `body` as a JSON string.
|
||||||
|
*/
|
||||||
|
function sseEnvelope(body: object, statusCodeValue = 200, statusCode = "OK"): string {
|
||||||
|
return (
|
||||||
|
"data:" +
|
||||||
|
JSON.stringify({
|
||||||
|
headers: { "Content-Type": ["application/json"] },
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
statusCodeValue,
|
||||||
|
statusCode,
|
||||||
|
}) +
|
||||||
|
"\n\n"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const DONE_SSE =
|
||||||
|
"data:" +
|
||||||
|
JSON.stringify({
|
||||||
|
headers: { "Content-Type": ["application/json"] },
|
||||||
|
body: "[DONE]",
|
||||||
|
statusCodeValue: 200,
|
||||||
|
statusCode: "OK",
|
||||||
|
}) +
|
||||||
|
"\n\n";
|
||||||
|
|
||||||
|
function chunk(delta: object, extra: object = {}): object {
|
||||||
|
return {
|
||||||
|
choices: [{ delta, index: 0 }],
|
||||||
|
created: 1,
|
||||||
|
id: "test-id",
|
||||||
|
model: "auto",
|
||||||
|
object: "chat.completion.chunk",
|
||||||
|
...extra,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function finishChunk(finish_reason: string, extra: object = {}): object {
|
||||||
|
return {
|
||||||
|
choices: [{ finish_reason, index: 0 }],
|
||||||
|
created: 1,
|
||||||
|
id: "test-id",
|
||||||
|
model: "auto",
|
||||||
|
object: "chat.completion.chunk",
|
||||||
|
usage: { completion_tokens: 1, prompt_tokens: 1, total_tokens: 2 },
|
||||||
|
...extra,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const SUCCESS_SSE =
|
||||||
|
sseEnvelope(chunk({ role: "assistant" })) +
|
||||||
|
sseEnvelope(chunk({ reasoning_content: "The user wants OK.", role: "assistant" })) +
|
||||||
|
sseEnvelope(chunk({ content: "OK", role: "assistant" })) +
|
||||||
|
sseEnvelope(finishChunk("stop")) +
|
||||||
|
DONE_SSE;
|
||||||
|
|
||||||
|
const BLOCKED_SSE = sseEnvelope(
|
||||||
|
{ code: "provider_error", message: "Session blocked", request_id: "r", type: "provider_error" },
|
||||||
|
406,
|
||||||
|
"Not Acceptable",
|
||||||
|
);
|
||||||
|
|
||||||
|
function mockFetch(body: string): typeof fetch {
|
||||||
|
const response = new Response(body, {
|
||||||
|
status: 200,
|
||||||
|
headers: { "content-type": "text/event-stream" },
|
||||||
|
});
|
||||||
|
return vi.fn(async () => response) as unknown as typeof fetch;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeModel(): Model<Api> {
|
||||||
|
return { id: "ultimate", api: "qoder-api" as Api, provider: "qoder" } as Model<Api>;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeContext(): Context {
|
||||||
|
return {
|
||||||
|
systemPrompt: "test",
|
||||||
|
messages: [{ role: "user", content: "hi" }],
|
||||||
|
tools: [],
|
||||||
|
} as unknown as Context;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function consume(stream: AssistantMessageEventStream): Promise<AssistantMessageEvent[]> {
|
||||||
|
const events: AssistantMessageEvent[] = [];
|
||||||
|
for await (const ev of stream) {
|
||||||
|
events.push(ev);
|
||||||
|
if (ev.type === "done" || ev.type === "error") break;
|
||||||
|
}
|
||||||
|
return events;
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("streamQoder", () => {
|
||||||
|
const originalFetch = globalThis.fetch;
|
||||||
|
afterEach(() => {
|
||||||
|
globalThis.fetch = originalFetch;
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("parses a successful SSE stream into text + stop", async () => {
|
||||||
|
globalThis.fetch = mockFetch(SUCCESS_SSE);
|
||||||
|
const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" });
|
||||||
|
const events = await consume(stream);
|
||||||
|
|
||||||
|
const done = events.find((e) => e.type === "done");
|
||||||
|
expect(done, "expected a done event").toBeDefined();
|
||||||
|
const msg = (done as { message: AssistantMessage }).message;
|
||||||
|
expect(msg.stopReason).toBe("stop");
|
||||||
|
const text = msg.content.find((c) => c.type === "text");
|
||||||
|
expect(text && "text" in text ? text.text : "").toBe("OK");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("surfaces an upstream 406 'Session blocked' as an error event, not a silent stop", async () => {
|
||||||
|
globalThis.fetch = mockFetch(BLOCKED_SSE);
|
||||||
|
const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" });
|
||||||
|
const events = await consume(stream);
|
||||||
|
|
||||||
|
const err = events.find((e) => e.type === "error");
|
||||||
|
expect(err, "expected an error event").toBeDefined();
|
||||||
|
const msg = (err as { error: AssistantMessage }).error;
|
||||||
|
expect(msg.stopReason).toBe("error");
|
||||||
|
expect(msg.errorMessage).toMatch(/Session blocked/);
|
||||||
|
expect(msg.errorMessage).toMatch(/406/);
|
||||||
|
// Must NOT emit a silent done/stop.
|
||||||
|
expect(events.find((e) => e.type === "done")).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("preserves finish_reason=length instead of overwriting to stop", async () => {
|
||||||
|
const sse =
|
||||||
|
sseEnvelope(chunk({ content: "partial", role: "assistant" })) + sseEnvelope(finishChunk("length")) + DONE_SSE;
|
||||||
|
globalThis.fetch = mockFetch(sse);
|
||||||
|
const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" });
|
||||||
|
const events = await consume(stream);
|
||||||
|
|
||||||
|
const done = events.find((e) => e.type === "done");
|
||||||
|
const msg = (done as { message: AssistantMessage }).message;
|
||||||
|
expect(msg.stopReason).toBe("length");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("captures usage, responseId and responseModel from the finish chunk", async () => {
|
||||||
|
const sse =
|
||||||
|
sseEnvelope(chunk({ content: "OK", role: "assistant" })) +
|
||||||
|
sseEnvelope(
|
||||||
|
finishChunk("stop", {
|
||||||
|
id: "chatcmpl-abc123",
|
||||||
|
model: "qmodel_latest",
|
||||||
|
usage: {
|
||||||
|
prompt_tokens: 42,
|
||||||
|
completion_tokens: 7,
|
||||||
|
total_tokens: 49,
|
||||||
|
completion_tokens_details: { reasoning_tokens: 3 },
|
||||||
|
prompt_tokens_details: { cacheable_tokens: 10, cached_tokens: 5 },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
) +
|
||||||
|
DONE_SSE;
|
||||||
|
globalThis.fetch = mockFetch(sse);
|
||||||
|
const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" });
|
||||||
|
const events = await consume(stream);
|
||||||
|
|
||||||
|
const done = events.find((e) => e.type === "done");
|
||||||
|
const msg = (done as { message: AssistantMessage }).message;
|
||||||
|
expect(msg.responseId).toBe("chatcmpl-abc123");
|
||||||
|
expect(msg.responseModel).toBe("qmodel_latest");
|
||||||
|
expect(msg.usage.input).toBe(42);
|
||||||
|
expect(msg.usage.output).toBe(7);
|
||||||
|
expect(msg.usage.totalTokens).toBe(49);
|
||||||
|
expect(msg.usage.cacheRead).toBe(5);
|
||||||
|
expect(msg.usage.cacheWrite).toBe(10);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("emits a done event with reason=length when finish_reason is length", async () => {
|
||||||
|
const sse =
|
||||||
|
sseEnvelope(chunk({ content: "partial", role: "assistant" })) + sseEnvelope(finishChunk("length")) + DONE_SSE;
|
||||||
|
globalThis.fetch = mockFetch(sse);
|
||||||
|
const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" });
|
||||||
|
const events = await consume(stream);
|
||||||
|
|
||||||
|
const done = events.find((e) => e.type === "done");
|
||||||
|
expect(done, "expected a done event").toBeDefined();
|
||||||
|
expect((done as { reason: string }).reason).toBe("length");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("reports a tool_use stop reason when the stream emits tool calls", async () => {
|
||||||
|
const sse =
|
||||||
|
sseEnvelope(
|
||||||
|
chunk({
|
||||||
|
tool_calls: [
|
||||||
|
{
|
||||||
|
index: 0,
|
||||||
|
id: "call_1",
|
||||||
|
function: { name: "bash", arguments: '{"command":"ls"}' },
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
) +
|
||||||
|
sseEnvelope(finishChunk("tool_calls")) +
|
||||||
|
DONE_SSE;
|
||||||
|
globalThis.fetch = mockFetch(sse);
|
||||||
|
const stream = streamQoder(makeModel(), makeContext(), { apiKey: "fake" });
|
||||||
|
const events = await consume(stream);
|
||||||
|
|
||||||
|
const done = events.find((e) => e.type === "done");
|
||||||
|
const msg = (done as { message: AssistantMessage }).message;
|
||||||
|
expect(msg.stopReason).toBe("toolUse");
|
||||||
|
const toolCall = msg.content.find((c) => c.type === "toolCall");
|
||||||
|
expect(toolCall).toBeDefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
+45
-6
@@ -153,7 +153,12 @@ export function streamQoder(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const sessionID = stableHash("qoder-session", userID, qoderModel);
|
// Each request gets a fresh server-side session id. Reusing a stable
|
||||||
|
// session id across back-to-back agentic requests triggers Qoder's
|
||||||
|
// per-session lock and the server replies `406 Session blocked` mid
|
||||||
|
// stream. We send full conversation history on every request, so a unique
|
||||||
|
// session id is safe and avoids the lock entirely.
|
||||||
|
const sessionID = `${stableHash("qoder-session", userID, qoderModel)}-${crypto.randomUUID()}`;
|
||||||
|
|
||||||
let maxTokens = 32768;
|
let maxTokens = 32768;
|
||||||
if (maxOutputTokens > 0) {
|
if (maxOutputTokens > 0) {
|
||||||
@@ -294,6 +299,22 @@ export function streamQoder(
|
|||||||
if (!innerStr || innerStr === "[DONE]") continue;
|
if (!innerStr || innerStr === "[DONE]") continue;
|
||||||
|
|
||||||
const inner = JSON.parse(innerStr);
|
const inner = JSON.parse(innerStr);
|
||||||
|
if (inner.id) output.responseId = inner.id as string;
|
||||||
|
if (inner.model) output.responseModel = inner.model as string;
|
||||||
|
if (inner.usage) {
|
||||||
|
const u = inner.usage as {
|
||||||
|
prompt_tokens?: number;
|
||||||
|
completion_tokens?: number;
|
||||||
|
total_tokens?: number;
|
||||||
|
completion_tokens_details?: { reasoning_tokens?: number };
|
||||||
|
prompt_tokens_details?: { cacheable_tokens?: number; cached_tokens?: number };
|
||||||
|
};
|
||||||
|
output.usage.input = u.prompt_tokens ?? 0;
|
||||||
|
output.usage.output = u.completion_tokens ?? 0;
|
||||||
|
output.usage.totalTokens = u.total_tokens ?? 0;
|
||||||
|
output.usage.cacheRead = u.prompt_tokens_details?.cached_tokens ?? 0;
|
||||||
|
output.usage.cacheWrite = u.prompt_tokens_details?.cacheable_tokens ?? 0;
|
||||||
|
}
|
||||||
if (inner.choices && inner.choices.length > 0) {
|
if (inner.choices && inner.choices.length > 0) {
|
||||||
const choice = inner.choices[0];
|
const choice = inner.choices[0];
|
||||||
const delta = choice.delta;
|
const delta = choice.delta;
|
||||||
@@ -382,10 +403,23 @@ export function streamQoder(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (choice.finish_reason) {
|
if (choice.finish_reason) {
|
||||||
output.stopReason = choice.finish_reason;
|
// Preserve the real upstream finish_reason (e.g. "length",
|
||||||
|
// "content_filter") instead of forcing "stop" later.
|
||||||
|
output.stopReason = choice.finish_reason as AssistantMessage["stopReason"];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch {}
|
} catch (e) {
|
||||||
|
// A single malformed SSE line shouldn't kill the stream — skip it.
|
||||||
|
// But a genuine upstream error (thrown below) must propagate to the
|
||||||
|
// outer catch and surface as stopReason="error", not be swallowed.
|
||||||
|
if (e instanceof SyntaxError) {
|
||||||
|
if (process.env.QODER_DEBUG) {
|
||||||
|
console.error("[pi-provider-qoder] skipping malformed SSE line:", dataStr.slice(0, 200));
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -428,10 +462,15 @@ export function streamQoder(
|
|||||||
|
|
||||||
if (toolCallsState.length > 0) {
|
if (toolCallsState.length > 0) {
|
||||||
output.stopReason = "toolUse";
|
output.stopReason = "toolUse";
|
||||||
} else {
|
|
||||||
output.stopReason = "stop";
|
|
||||||
}
|
}
|
||||||
stream.push({ type: "done", reason: output.stopReason as "stop" | "toolUse", message: output });
|
// Otherwise keep whatever finish_reason set upstream (defaults to "stop").
|
||||||
|
// Never overwrite a meaningful finish_reason ("length", "content_filter",
|
||||||
|
// ...) with "stop".
|
||||||
|
stream.push({
|
||||||
|
type: "done",
|
||||||
|
reason: output.stopReason as Extract<AssistantMessage["stopReason"], "stop" | "length" | "toolUse">,
|
||||||
|
message: output,
|
||||||
|
});
|
||||||
stream.end();
|
stream.end();
|
||||||
} catch (e: unknown) {
|
} catch (e: unknown) {
|
||||||
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
output.stopReason = options?.signal?.aborted ? "aborted" : "error";
|
||||||
|
|||||||
Reference in New Issue
Block a user