feat(core): add retry for transient HTTP and stream-level errors

Add retry mechanism driven by pi settings.json retry.provider config
(timeoutMs, maxRetries, maxRetryDelayMs).

HTTP-level retries handle 429/5xx with exponential backoff and jitter,
respecting Retry-After headers (seconds and HTTP-date formats).

Stream-level retries handle cases where the API returns 200 OK but
sends an error event in the stream body. Retries only when no content
has been emitted yet.

Per-attempt timeout via AbortController with automatic retry. Clean
abort propagation through the retry loop.
This commit is contained in:
jizhejiang
2026-06-02 11:38:10 +08:00
parent f48291f252
commit c3107dbc1c
7 changed files with 610 additions and 1995 deletions
+4
View File
@@ -1,5 +1,9 @@
# Changelog # Changelog
## Unreleased
- Add retry mechanism for transient HTTP errors (429, 5xx) and stream-level errors, configurable via pi `settings.json` `retry.provider` fields (`timeoutMs`, `maxRetries`, `maxRetryDelayMs`). Supports exponential backoff with jitter and `Retry-After` header.
## 0.3.1 - 2026-05-29 ## 0.3.1 - 2026-05-29
- Bump CLI version header to `0.29.0` for Command Code API parity. - Bump CLI version header to `0.29.0` for Command Code API parity.
+8 -1923
View File
File diff suppressed because it is too large Load Diff
+2 -1
View File
@@ -28,7 +28,7 @@
"LICENSE" "LICENSE"
], ],
"scripts": { "scripts": {
"test": "npm run typecheck && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-pricing.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-stream.ts && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "test": "npm run typecheck && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-pricing.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-stream.ts && tsx tests/test-retry.ts && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs",
"typecheck": "tsc --noEmit", "typecheck": "tsc --noEmit",
"format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'",
"format": "prettier --write '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'",
@@ -38,6 +38,7 @@
"test:oauth": "tsx tests/test-oauth.ts", "test:oauth": "tsx tests/test-oauth.ts",
"test:abort": "tsx tests/test-abort.ts", "test:abort": "tsx tests/test-abort.ts",
"test:stream": "tsx tests/test-stream.ts", "test:stream": "tsx tests/test-stream.ts",
"test:retry": "tsx tests/test-retry.ts",
"test:pi-local": "node tests/test-pi-local.mjs", "test:pi-local": "node tests/test-pi-local.mjs",
"test:smoke": "node tests/test-smoke.mjs" "test:smoke": "node tests/test-smoke.mjs"
}, },
+133 -9
View File
@@ -42,6 +42,37 @@ export const DEFAULT_API_BASE = "https://api.commandcode.ai"
export const COMMAND_CODE_CLI_VERSION = "0.29.0" export const COMMAND_CODE_CLI_VERSION = "0.29.0"
const DEFAULT_GENERATE_MAX_TOKENS = 64_000 const DEFAULT_GENERATE_MAX_TOKENS = 64_000
const DEFAULT_MAX_RETRIES = 2
const DEFAULT_MAX_RETRY_DELAY_MS = 60_000
const BASE_RETRY_DELAY_MS = 500
function isRetryableStatus(status: number): boolean {
return status === 429 || (status >= 500 && status < 600)
}
function parseRetryAfterSeconds(value: string | null): number | undefined {
if (!value) return undefined
const seconds = Number(value)
if (Number.isFinite(seconds) && seconds >= 0) return seconds
const date = Date.parse(value)
if (!Number.isNaN(date)) return Math.max(0, (date - Date.now()) / 1000)
return undefined
}
function retryDelayMs(
attempt: number,
retryAfterHeader: string | null,
maxDelayMs: number,
): number {
const retryAfterMs = parseRetryAfterSeconds(retryAfterHeader)
if (retryAfterMs !== undefined) {
if (retryAfterMs * 1000 > maxDelayMs) return -1
return retryAfterMs * 1000
}
const exponential = BASE_RETRY_DELAY_MS * 2 ** attempt
const jitter = exponential * 0.2 * Math.random()
return Math.min(exponential + jitter, maxDelayMs)
}
function defaultUsage(): Usage { function defaultUsage(): Usage {
return { return {
@@ -104,6 +135,22 @@ export function createStreamCommandCode(deps: CoreDependencies) {
const cwd = deps.cwd ?? (() => process.cwd()) const cwd = deps.cwd ?? (() => process.cwd())
const now = deps.now ?? (() => Date.now()) const now = deps.now ?? (() => Date.now())
const uuid = deps.uuid ?? (() => randomUUID()) const uuid = deps.uuid ?? (() => randomUUID())
const delay =
deps.delay ??
((ms: number, signal: AbortSignal) => {
if (signal.aborted) return Promise.reject(abortError())
return new Promise<void>((resolve, reject) => {
const id = setTimeout(() => {
signal.removeEventListener("abort", onAbort)
resolve()
}, ms)
const onAbort = () => {
clearTimeout(id)
reject(abortError())
}
signal.addEventListener("abort", onAbort, { once: true })
})
})
function raceAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> { function raceAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
if (signal.aborted) return Promise.reject(abortError()) if (signal.aborted) return Promise.reject(abortError())
@@ -386,10 +433,10 @@ export function createStreamCommandCode(deps: CoreDependencies) {
) )
if (nextBody !== undefined) body = nextBody if (nextBody !== undefined) body = nextBody
const response = await raceAbort( const maxRetries = options?.maxRetries ?? DEFAULT_MAX_RETRIES
fetchImpl(`${apiBase}/alpha/generate`, { const maxRetryDelayMs = options?.maxRetryDelayMs ?? DEFAULT_MAX_RETRY_DELAY_MS
method: "POST", const timeoutMs = options?.timeoutMs
headers: { const requestHeaders = {
"Content-Type": "application/json", "Content-Type": "application/json",
Authorization: `Bearer ${apiKey}`, Authorization: `Bearer ${apiKey}`,
"x-command-code-version": COMMAND_CODE_CLI_VERSION, "x-command-code-version": COMMAND_CODE_CLI_VERSION,
@@ -398,12 +445,59 @@ export function createStreamCommandCode(deps: CoreDependencies) {
"x-taste-learning": "true", "x-taste-learning": "true",
"x-co-flag": "false", "x-co-flag": "false",
...options?.headers, ...options?.headers,
}, }
body: JSON.stringify(body), const bodyStr = JSON.stringify(body)
signal: controller.signal,
}), let response!: Response
controller.signal, retryLoop: for (let attempt = 0; ; attempt++) {
const attemptController = new AbortController()
let attemptTimedOut = false
let attemptTimeoutId: ReturnType<typeof setTimeout> | undefined
if (timeoutMs !== undefined) {
attemptTimeoutId = setTimeout(() => {
attemptTimedOut = true
attemptController.abort()
}, timeoutMs)
}
const onOuterAbort = () => attemptController.abort()
controller.signal.addEventListener("abort", onOuterAbort, { once: true })
try {
response = await fetchImpl(`${apiBase}/alpha/generate`, {
method: "POST",
headers: requestHeaders,
body: bodyStr,
signal: attemptController.signal,
})
} catch (fetchError: unknown) {
if (controller.signal.aborted) throw abortError("Aborted")
if (
timeoutMs !== undefined &&
attemptController.signal.aborted &&
attempt < maxRetries
) {
continue
}
throw fetchError
} finally {
controller.signal.removeEventListener("abort", onOuterAbort)
if (attemptTimeoutId !== undefined) clearTimeout(attemptTimeoutId)
}
// --- HTTP-level retry ---
if (!response.ok && isRetryableStatus(response.status) && attempt < maxRetries) {
const retryAfter = response.headers.get("retry-after")
const waitMs = retryDelayMs(attempt, retryAfter, maxRetryDelayMs)
if (waitMs < 0) {
const requestedSeconds = parseRetryAfterSeconds(retryAfter) ?? 0
throw new Error(
`Retry-After delay ${requestedSeconds}s exceeds max ${maxRetryDelayMs}ms`,
) )
}
await response.text().catch(() => "")
if (waitMs > 0) await delay(waitMs, controller.signal)
continue retryLoop
}
await raceAbort( await raceAbort(
Promise.resolve( Promise.resolve(
@@ -426,12 +520,14 @@ export function createStreamCommandCode(deps: CoreDependencies) {
throw new Error(`Command Code API error ${response.status}: ${errBody.slice(0, 500)}`) throw new Error(`Command Code API error ${response.status}: ${errBody.slice(0, 500)}`)
} }
// --- Read response stream ---
reader = response.body?.getReader() reader = response.body?.getReader()
if (!reader) throw new Error("No response body") if (!reader) throw new Error("No response body")
const decoder = new TextDecoder() const decoder = new TextDecoder()
let buffer = "" let buffer = ""
try {
readLoop: for (;;) { readLoop: for (;;) {
if (controller.signal.aborted) throw abortError("Aborted") if (controller.signal.aborted) throw abortError("Aborted")
const { done, value } = await raceAbort(reader.read(), controller.signal) const { done, value } = await raceAbort(reader.read(), controller.signal)
@@ -451,7 +547,33 @@ export function createStreamCommandCode(deps: CoreDependencies) {
if (finished) break readLoop if (finished) break readLoop
} }
} }
} catch (streamError: unknown) {
// Stream-level error (e.g. API returned 200 OK but sent an error event)
// or per-attempt timeout during stream reading.
await reader.cancel().catch(() => {})
try {
reader.releaseLock()
} catch {}
reader = undefined
if (controller.signal.aborted) throw streamError
const canRetry =
(output.content.length === 0 || attemptTimedOut) && attempt < maxRetries
if (canRetry) {
// Reset state for the next attempt.
output.content.length = 0
output.stopReason = "stop"
output.errorMessage = undefined
finished = false
const waitMs = attemptTimedOut ? 0 : retryDelayMs(attempt, null, maxRetryDelayMs)
if (waitMs > 0) await delay(waitMs, controller.signal)
continue retryLoop
}
throw streamError
}
// Stream completed successfully.
endTextBlock() endTextBlock()
endThinking() endThinking()
@@ -461,6 +583,8 @@ export function createStreamCommandCode(deps: CoreDependencies) {
message: output, message: output,
}) })
stream.end() stream.end()
break retryLoop
}
} catch (error: unknown) { } catch (error: unknown) {
const reason: ErrorReason = controller.signal.aborted ? "aborted" : "error" const reason: ErrorReason = controller.signal.aborted ? "aborted" : "error"
output.stopReason = reason output.stopReason = reason
+19
View File
@@ -89,6 +89,23 @@ export interface StreamOptions {
maxTokens?: number maxTokens?: number
onPayload?: (payload: unknown, model: ModelLike) => unknown | Promise<unknown> onPayload?: (payload: unknown, model: ModelLike) => unknown | Promise<unknown>
onResponse?: (response: ProviderResponseInfo, model: ModelLike) => void | Promise<void> onResponse?: (response: ProviderResponseInfo, model: ModelLike) => void | Promise<void>
/**
* HTTP request timeout in milliseconds.
* Applied per-attempt; on timeout the request is retried if retries remain.
*/
timeoutMs?: number
/**
* Maximum retry attempts for transient HTTP errors (429, 5xx).
* Default: 2.
*/
maxRetries?: number
/**
* Maximum delay in milliseconds to wait for a retry when the server requests
* a long wait via Retry-After. If the server's requested delay exceeds this
* value, the request fails immediately. Default: 60000 (60 seconds).
* Set to 0 to disable the cap.
*/
maxRetryDelayMs?: number
} }
export type AssistantMessageEvent = export type AssistantMessageEvent =
@@ -153,4 +170,6 @@ export interface CoreDependencies {
now?: () => number now?: () => number
uuid?: () => string uuid?: () => string
homeDir?: () => string homeDir?: () => string
/** Injectable delay for retry backoff. Defaults to setTimeout. */
delay?: (ms: number, signal: AbortSignal) => Promise<void>
} }
+20 -5
View File
@@ -112,6 +112,7 @@ export function createTestDeps(overrides: Partial<CoreDependencies> = {}): TestD
now: () => new Date("2026-05-05T12:00:00Z").getTime(), now: () => new Date("2026-05-05T12:00:00Z").getTime(),
uuid: () => "00000000-0000-4000-8000-000000000000", uuid: () => "00000000-0000-4000-8000-000000000000",
cwd: () => "/repo", cwd: () => "/repo",
delay: async () => {},
...overrides, ...overrides,
}) })
return { streamCommandCode, calculatedUsages } return { streamCommandCode, calculatedUsages }
@@ -124,12 +125,15 @@ type SuccessPlan = {
chunks?: string[] chunks?: string[]
delays?: number[] delays?: number[]
hangAfterLast?: boolean hangAfterLast?: boolean
/** Delay in ms before the server starts sending the response. */
responseDelay?: number
} }
type ErrorPlan = { type ErrorPlan = {
type: "error" type: "error"
status: number status: number
body: string body: string
headers?: Record<string, string>
} }
export type ResponsePlan = SuccessPlan | ErrorPlan export type ResponsePlan = SuccessPlan | ErrorPlan
@@ -146,6 +150,7 @@ function headersToRecord(headers: IncomingHttpHeaders): Record<string, string> {
export interface MockCommandCodeServer { export interface MockCommandCodeServer {
baseUrl(): string baseUrl(): string
mockResponse(plan: ResponsePlan): void mockResponse(plan: ResponsePlan): void
mockResponseQueue(plans: ResponsePlan[]): void
reset(): void reset(): void
close(): Promise<void> close(): Promise<void>
lastRequestBody(): unknown lastRequestBody(): unknown
@@ -155,7 +160,7 @@ export interface MockCommandCodeServer {
} }
export async function startMockCommandCodeServer(): Promise<MockCommandCodeServer> { export async function startMockCommandCodeServer(): Promise<MockCommandCodeServer> {
let nextPlan: ResponsePlan = { type: "success", events: [] } let planQueue: ResponsePlan[] = [{ type: "success", events: [] }]
let lastBody: unknown let lastBody: unknown
let lastHeaders: Record<string, string> = {} let lastHeaders: Record<string, string> = {}
let requests = 0 let requests = 0
@@ -183,9 +188,12 @@ export async function startMockCommandCodeServer(): Promise<MockCommandCodeServe
lastBody = undefined lastBody = undefined
} }
const plan = nextPlan // Pop the first plan from the queue; keep the last one as fallback.
const plan = planQueue.length > 1 ? planQueue.shift()! : planQueue[0]
if (plan.type === "error") { if (plan.type === "error") {
res.writeHead(plan.status, { "Content-Type": "text/plain" }) const headers: Record<string, string> = { "Content-Type": "text/plain", ...plan.headers }
res.writeHead(plan.status, headers)
res.end(plan.body) res.end(plan.body)
return return
} }
@@ -223,7 +231,11 @@ export async function startMockCommandCodeServer(): Promise<MockCommandCodeServe
} }
} }
if (plan.responseDelay) {
setTimeout(sendNext, plan.responseDelay)
} else {
sendNext() sendNext()
}
}) })
}) })
@@ -238,10 +250,13 @@ export async function startMockCommandCodeServer(): Promise<MockCommandCodeServe
return { return {
baseUrl: () => `http://127.0.0.1:${port}`, baseUrl: () => `http://127.0.0.1:${port}`,
mockResponse(plan: ResponsePlan) { mockResponse(plan: ResponsePlan) {
nextPlan = plan planQueue = [plan]
},
mockResponseQueue(plans: ResponsePlan[]) {
planQueue = [...plans]
}, },
reset() { reset() {
nextPlan = { type: "success", events: [] } planQueue = [{ type: "success", events: [] }]
lastBody = undefined lastBody = undefined
lastHeaders = {} lastHeaders = {}
requests = 0 requests = 0
+367
View File
@@ -0,0 +1,367 @@
/**
* Tests for retry and timeout behaviour driven by pi settings.json
* retry config (timeoutMs, maxRetries, maxRetryDelayMs).
*/
import assert from "node:assert/strict"
import { after, before, beforeEach, describe, it } from "node:test"
import type { AssistantMessageEvent } from "../src/core.ts"
import {
collectEvents,
createTestDeps,
makeContext,
makeModel,
startMockCommandCodeServer,
type MockCommandCodeServer,
} from "./helpers.ts"
let server: MockCommandCodeServer
before(async () => {
server = await startMockCommandCodeServer()
})
after(async () => {
await server.close()
})
beforeEach(() => {
server.reset()
})
function eventTypes(events: readonly AssistantMessageEvent[]): string[] {
return events.map((event) => event.type)
}
describe("streamCommandCode — retry on transient errors", () => {
it("retries on 429 and succeeds on the second attempt", async () => {
server.mockResponseQueue([
{ type: "error", status: 429, body: "rate limited" },
{
type: "success",
events: [
JSON.stringify({ type: "text-delta", text: "ok" }),
JSON.stringify({ type: "finish", finishReason: "stop" }),
],
},
])
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }),
)
assert.equal(server.requestCount(), 2)
assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "text_end", "done"])
const done = events.at(-1)
if (done?.type !== "done") throw new Error("expected done")
assert.equal(done.reason, "stop")
})
it("retries on 500 and succeeds on the second attempt", async () => {
server.mockResponseQueue([
{ type: "error", status: 500, body: "internal server error" },
{
type: "success",
events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
},
])
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }),
)
assert.equal(server.requestCount(), 2)
assert.equal(events.at(-1)?.type, "done")
})
it("does NOT retry on 400 (non-retryable client error)", async () => {
server.mockResponse({ type: "error", status: 400, body: "bad request" })
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }),
)
assert.equal(server.requestCount(), 1)
assert.deepEqual(eventTypes(events), ["start", "error"])
const last = events.at(-1)
if (last?.type !== "error") throw new Error("expected error")
assert.match(last.error.errorMessage ?? "", /400/)
})
it("exhausts maxRetries and emits an error", async () => {
server.mockResponse({ type: "error", status: 503, body: "unavailable" })
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), {
apiKey: "mock-key",
maxRetries: 3,
}),
)
// initial attempt + 3 retries = 4 total
assert.equal(server.requestCount(), 4)
assert.deepEqual(eventTypes(events), ["start", "error"])
const last503 = events.at(-1)
if (last503?.type !== "error") throw new Error("expected error")
assert.match(last503.error.errorMessage ?? "", /503/)
})
})
describe("streamCommandCode — Retry-After header", () => {
it("respects Retry-After delay in seconds", async () => {
let delayCalled = false
server.mockResponseQueue([
{
type: "error",
status: 429,
body: "rate limited",
headers: { "retry-after": "2" },
},
{
type: "success",
events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
},
])
const { streamCommandCode } = createTestDeps({
apiBase: server.baseUrl(),
delay: async (ms: number) => {
delayCalled = true
assert.equal(ms, 2000)
},
})
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }),
)
assert.equal(server.requestCount(), 2)
assert.equal(events.at(-1)?.type, "done")
assert.ok(delayCalled, "delay should have been called with Retry-After value")
})
it("fails immediately when Retry-After exceeds maxRetryDelayMs", async () => {
server.mockResponse({
type: "error",
status: 429,
body: "rate limited",
headers: { "retry-after": "300" },
})
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), {
apiKey: "mock-key",
maxRetryDelayMs: 10_000,
}),
)
assert.equal(server.requestCount(), 1)
assert.deepEqual(eventTypes(events), ["start", "error"])
const lastMax = events.at(-1)
if (lastMax?.type !== "error") throw new Error("expected error")
assert.match(lastMax.error.errorMessage ?? "", /exceeds max/)
})
})
describe("streamCommandCode — timeout", () => {
it("retries on per-attempt timeout and succeeds", async () => {
server.mockResponseQueue([
{
type: "success",
events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
hangAfterLast: true,
responseDelay: 200,
},
{
type: "success",
events: [
JSON.stringify({ type: "text-delta", text: "fast" }),
JSON.stringify({ type: "finish", finishReason: "stop" }),
],
},
])
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), {
apiKey: "mock-key",
timeoutMs: 50,
}),
5_000,
)
assert.equal(server.requestCount(), 2)
assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "text_end", "done"])
})
it("emits error when all retry attempts time out", async () => {
server.mockResponse({
type: "success",
events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
hangAfterLast: true,
responseDelay: 200,
})
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), {
apiKey: "mock-key",
timeoutMs: 50,
maxRetries: 1,
}),
5_000,
)
// initial + 1 retry = 2
assert.equal(server.requestCount(), 2)
assert.deepEqual(eventTypes(events), ["start", "error"])
})
})
describe("streamCommandCode — abort cancels retry loop", () => {
it("user abort stops retries immediately", async () => {
server.mockResponse({ type: "error", status: 500, body: "error" })
const controller = new AbortController()
const { streamCommandCode } = createTestDeps({
apiBase: server.baseUrl(),
delay: async (_ms: number, signal: AbortSignal) => {
// Abort during the retry delay
controller.abort()
// Simulate the real delay which rejects on abort
return new Promise<void>((_, reject) => {
if (signal.aborted) reject(new DOMException("Aborted", "AbortError"))
signal.addEventListener("abort", () => reject(new DOMException("Aborted", "AbortError")))
})
},
})
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), {
apiKey: "mock-key",
signal: controller.signal,
maxRetries: 10,
}),
)
// Should only have made 1 request (the initial one), then aborted during delay
assert.equal(server.requestCount(), 1)
assert.deepEqual(eventTypes(events), ["start", "error"])
const error = events.at(-1)
if (error?.type !== "error") throw new Error("expected error")
assert.equal(error.reason, "aborted")
})
})
describe("streamCommandCode — retry defaults", () => {
it("uses default maxRetries of 2 when not specified", async () => {
server.mockResponse({ type: "error", status: 500, body: "error" })
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
await collectEvents(streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }))
// initial + 2 retries = 3
assert.equal(server.requestCount(), 3)
})
it("respects maxRetries: 0 (no retries)", async () => {
server.mockResponse({ type: "error", status: 500, body: "error" })
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), {
apiKey: "mock-key",
maxRetries: 0,
}),
)
assert.equal(server.requestCount(), 1)
assert.deepEqual(eventTypes(events), ["start", "error"])
})
})
describe("streamCommandCode — stream-level error retry", () => {
it("retries when API returns 200 OK but stream contains an error event", async () => {
server.mockResponseQueue([
{
type: "success",
events: [
JSON.stringify({
type: "error",
error: "Service temporarily unavailable. Please try again shortly.",
}),
],
},
{
type: "success",
events: [
JSON.stringify({ type: "text-delta", text: "ok" }),
JSON.stringify({ type: "finish", finishReason: "stop" }),
],
},
])
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }),
)
assert.equal(server.requestCount(), 2)
assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "text_end", "done"])
})
it("exhausts retries on persistent stream-level errors", async () => {
server.mockResponse({
type: "success",
events: [
JSON.stringify({
type: "error",
error: "Service temporarily unavailable. Please try again shortly.",
}),
],
})
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), {
apiKey: "mock-key",
maxRetries: 3,
}),
)
// initial + 3 retries = 4
assert.equal(server.requestCount(), 4)
assert.deepEqual(eventTypes(events), ["start", "error"])
const last = events.at(-1)
if (last?.type !== "error") throw new Error("expected error")
assert.match(last.error.errorMessage ?? "", /temporarily unavailable/)
})
it("does NOT retry stream error when content was already emitted", async () => {
server.mockResponseQueue([
{
type: "success",
events: [
JSON.stringify({ type: "text-delta", text: "partial" }),
JSON.stringify({
type: "error",
error: "Service temporarily unavailable",
}),
],
},
])
const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
const events = await collectEvents(
streamCommandCode(makeModel(), makeContext(), { apiKey: "mock-key" }),
)
// Only 1 request — no retry because content was already emitted.
assert.equal(server.requestCount(), 1)
assert.deepEqual(eventTypes(events), ["start", "text_start", "text_delta", "error"])
})
})