feat(core): add retry for transient HTTP and stream-level errors

Add retry mechanism driven by pi settings.json retry.provider config (timeoutMs, maxRetries, maxRetryDelayMs). HTTP-level retries handle 429/5xx with exponential backoff and jitter, respecting Retry-After headers (seconds and HTTP-date formats). Stream-level retries handle cases where the API returns 200 OK but sends an error event in the stream body. Retries only when no content has been emitted yet. Per-attempt timeout via AbortController with automatic retry. Clean abort propagation through the retry loop.
2026-06-02 11:38:10 +08:00
parent f48291f252
commit c3107dbc1c
7 changed files with 610 additions and 1995 deletions
@@ -42,6 +42,37 @@ export const DEFAULT_API_BASE = "https://api.commandcode.ai"
 export const COMMAND_CODE_CLI_VERSION = "0.29.0"

 const DEFAULT_GENERATE_MAX_TOKENS = 64_000
+const DEFAULT_MAX_RETRIES = 2
+const DEFAULT_MAX_RETRY_DELAY_MS = 60_000
+const BASE_RETRY_DELAY_MS = 500
+
+function isRetryableStatus(status: number): boolean {
+  return status === 429 || (status >= 500 && status < 600)
+}
+
+function parseRetryAfterSeconds(value: string | null): number | undefined {
+  if (!value) return undefined
+  const seconds = Number(value)
+  if (Number.isFinite(seconds) && seconds >= 0) return seconds
+  const date = Date.parse(value)
+  if (!Number.isNaN(date)) return Math.max(0, (date - Date.now()) / 1000)
+  return undefined
+}
+
+function retryDelayMs(
+  attempt: number,
+  retryAfterHeader: string | null,
+  maxDelayMs: number,
+): number {
+  const retryAfterMs = parseRetryAfterSeconds(retryAfterHeader)
+  if (retryAfterMs !== undefined) {
+    if (retryAfterMs * 1000 > maxDelayMs) return -1
+    return retryAfterMs * 1000
+  }
+  const exponential = BASE_RETRY_DELAY_MS * 2 ** attempt
+  const jitter = exponential * 0.2 * Math.random()
+  return Math.min(exponential + jitter, maxDelayMs)
+}

 function defaultUsage(): Usage {
  return {
@@ -104,6 +135,22 @@ export function createStreamCommandCode(deps: CoreDependencies) {
  const cwd = deps.cwd ?? (() => process.cwd())
  const now = deps.now ?? (() => Date.now())
  const uuid = deps.uuid ?? (() => randomUUID())
+  const delay =
+    deps.delay ??
+    ((ms: number, signal: AbortSignal) => {
+      if (signal.aborted) return Promise.reject(abortError())
+      return new Promise<void>((resolve, reject) => {
+        const id = setTimeout(() => {
+          signal.removeEventListener("abort", onAbort)
+          resolve()
+        }, ms)
+        const onAbort = () => {
+          clearTimeout(id)
+          reject(abortError())
+        }
+        signal.addEventListener("abort", onAbort, { once: true })
+      })
+    })

  function raceAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
    if (signal.aborted) return Promise.reject(abortError())
@@ -386,81 +433,158 @@ export function createStreamCommandCode(deps: CoreDependencies) {
        )
        if (nextBody !== undefined) body = nextBody

-        const response = await raceAbort(
-          fetchImpl(`${apiBase}/alpha/generate`, {
-            method: "POST",
-            headers: {
-              "Content-Type": "application/json",
-              Authorization: `Bearer ${apiKey}`,
-              "x-command-code-version": COMMAND_CODE_CLI_VERSION,
-              "x-cli-environment": "production",
-              "x-project-slug": projectSlugFromPath(workingDir),
-              "x-taste-learning": "true",
-              "x-co-flag": "false",
-              ...options?.headers,
-            },
-            body: JSON.stringify(body),
-            signal: controller.signal,
-          }),
-          controller.signal,
-        )
+        const maxRetries = options?.maxRetries ?? DEFAULT_MAX_RETRIES
+        const maxRetryDelayMs = options?.maxRetryDelayMs ?? DEFAULT_MAX_RETRY_DELAY_MS
+        const timeoutMs = options?.timeoutMs
+        const requestHeaders = {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${apiKey}`,
+          "x-command-code-version": COMMAND_CODE_CLI_VERSION,
+          "x-cli-environment": "production",
+          "x-project-slug": projectSlugFromPath(workingDir),
+          "x-taste-learning": "true",
+          "x-co-flag": "false",
+          ...options?.headers,
+        }
+        const bodyStr = JSON.stringify(body)

-        await raceAbort(
-          Promise.resolve(
-            options?.onResponse?.(
-              {
-                status: response.status,
-                headers: headersToRecord(response.headers),
-              },
-              model,
+        let response!: Response
+        retryLoop: for (let attempt = 0; ; attempt++) {
+          const attemptController = new AbortController()
+          let attemptTimedOut = false
+          let attemptTimeoutId: ReturnType<typeof setTimeout> | undefined
+          if (timeoutMs !== undefined) {
+            attemptTimeoutId = setTimeout(() => {
+              attemptTimedOut = true
+              attemptController.abort()
+            }, timeoutMs)
+          }
+          const onOuterAbort = () => attemptController.abort()
+          controller.signal.addEventListener("abort", onOuterAbort, { once: true })
+
+          try {
+            response = await fetchImpl(`${apiBase}/alpha/generate`, {
+              method: "POST",
+              headers: requestHeaders,
+              body: bodyStr,
+              signal: attemptController.signal,
+            })
+          } catch (fetchError: unknown) {
+            if (controller.signal.aborted) throw abortError("Aborted")
+            if (
+              timeoutMs !== undefined &&
+              attemptController.signal.aborted &&
+              attempt < maxRetries
+            ) {
+              continue
+            }
+            throw fetchError
+          } finally {
+            controller.signal.removeEventListener("abort", onOuterAbort)
+            if (attemptTimeoutId !== undefined) clearTimeout(attemptTimeoutId)
+          }
+
+          // --- HTTP-level retry ---
+          if (!response.ok && isRetryableStatus(response.status) && attempt < maxRetries) {
+            const retryAfter = response.headers.get("retry-after")
+            const waitMs = retryDelayMs(attempt, retryAfter, maxRetryDelayMs)
+            if (waitMs < 0) {
+              const requestedSeconds = parseRetryAfterSeconds(retryAfter) ?? 0
+              throw new Error(
+                `Retry-After delay ${requestedSeconds}s exceeds max ${maxRetryDelayMs}ms`,
+              )
+            }
+            await response.text().catch(() => "")
+            if (waitMs > 0) await delay(waitMs, controller.signal)
+            continue retryLoop
+          }
+
+          await raceAbort(
+            Promise.resolve(
+              options?.onResponse?.(
+                {
+                  status: response.status,
+                  headers: headersToRecord(response.headers),
+                },
+                model,
+              ),
            ),
-          ),
-          controller.signal,
-        )
-
-        if (!response.ok) {
-          const errBody = await raceAbort(
-            response.text().catch(() => ""),
            controller.signal,
          )
-          throw new Error(`Command Code API error ${response.status}: ${errBody.slice(0, 500)}`)
-        }

-        reader = response.body?.getReader()
-        if (!reader) throw new Error("No response body")
-
-        const decoder = new TextDecoder()
-        let buffer = ""
-
-        readLoop: for (;;) {
-          if (controller.signal.aborted) throw abortError("Aborted")
-          const { done, value } = await raceAbort(reader.read(), controller.signal)
-          if (done) {
-            if (buffer.trim()) handleEvent(parseStreamEventLine(buffer))
-            break
+          if (!response.ok) {
+            const errBody = await raceAbort(
+              response.text().catch(() => ""),
+              controller.signal,
+            )
+            throw new Error(`Command Code API error ${response.status}: ${errBody.slice(0, 500)}`)
          }
-          if (controller.signal.aborted) throw abortError("Aborted")

-          buffer += decoder.decode(value, { stream: true })
-          const lines = buffer.split("\n")
-          buffer = lines.pop() ?? ""
+          // --- Read response stream ---
+          reader = response.body?.getReader()
+          if (!reader) throw new Error("No response body")

-          for (const line of lines) {
-            if (controller.signal.aborted) throw abortError("Aborted")
-            handleEvent(parseStreamEventLine(line))
-            if (finished) break readLoop
+          const decoder = new TextDecoder()
+          let buffer = ""
+
+          try {
+            readLoop: for (;;) {
+              if (controller.signal.aborted) throw abortError("Aborted")
+              const { done, value } = await raceAbort(reader.read(), controller.signal)
+              if (done) {
+                if (buffer.trim()) handleEvent(parseStreamEventLine(buffer))
+                break
+              }
+              if (controller.signal.aborted) throw abortError("Aborted")
+
+              buffer += decoder.decode(value, { stream: true })
+              const lines = buffer.split("\n")
+              buffer = lines.pop() ?? ""
+
+              for (const line of lines) {
+                if (controller.signal.aborted) throw abortError("Aborted")
+                handleEvent(parseStreamEventLine(line))
+                if (finished) break readLoop
+              }
+            }
+          } catch (streamError: unknown) {
+            // Stream-level error (e.g. API returned 200 OK but sent an error event)
+            // or per-attempt timeout during stream reading.
+            await reader.cancel().catch(() => {})
+            try {
+              reader.releaseLock()
+            } catch {}
+            reader = undefined
+
+            if (controller.signal.aborted) throw streamError
+
+            const canRetry =
+              (output.content.length === 0 || attemptTimedOut) && attempt < maxRetries
+            if (canRetry) {
+              // Reset state for the next attempt.
+              output.content.length = 0
+              output.stopReason = "stop"
+              output.errorMessage = undefined
+              finished = false
+              const waitMs = attemptTimedOut ? 0 : retryDelayMs(attempt, null, maxRetryDelayMs)
+              if (waitMs > 0) await delay(waitMs, controller.signal)
+              continue retryLoop
+            }
+            throw streamError
          }
+
+          // Stream completed successfully.
+          endTextBlock()
+          endThinking()
+
+          stream.push({
+            type: "done",
+            reason: successStopReason(output.stopReason),
+            message: output,
+          })
+          stream.end()
+          break retryLoop
        }
-
-        endTextBlock()
-        endThinking()
-
-        stream.push({
-          type: "done",
-          reason: successStopReason(output.stopReason),
-          message: output,
-        })
-        stream.end()
      } catch (error: unknown) {
        const reason: ErrorReason = controller.signal.aborted ? "aborted" : "error"
        output.stopReason = reason
@@ -89,6 +89,23 @@ export interface StreamOptions {
  maxTokens?: number
  onPayload?: (payload: unknown, model: ModelLike) => unknown | Promise<unknown>
  onResponse?: (response: ProviderResponseInfo, model: ModelLike) => void | Promise<void>
+  /**
+   * HTTP request timeout in milliseconds.
+   * Applied per-attempt; on timeout the request is retried if retries remain.
+   */
+  timeoutMs?: number
+  /**
+   * Maximum retry attempts for transient HTTP errors (429, 5xx).
+   * Default: 2.
+   */
+  maxRetries?: number
+  /**
+   * Maximum delay in milliseconds to wait for a retry when the server requests
+   * a long wait via Retry-After. If the server's requested delay exceeds this
+   * value, the request fails immediately. Default: 60000 (60 seconds).
+   * Set to 0 to disable the cap.
+   */
+  maxRetryDelayMs?: number
 }

 export type AssistantMessageEvent =
@@ -153,4 +170,6 @@ export interface CoreDependencies {
  now?: () => number
  uuid?: () => string
  homeDir?: () => string
+  /** Injectable delay for retry backoff. Defaults to setTimeout. */
+  delay?: (ms: number, signal: AbortSignal) => Promise<void>
 }