perf(models): start from the cached catalog and refresh in the background

Every host start awaited a full catalog request before the provider
registered, costing one HTTPS round-trip (up to the discovery timeout on a
hanging connection) even when a cache written seconds earlier was on disk.

Register the cached catalog immediately and run the live refresh in the
background; the live result re-registers the provider when it arrives. A
first start without a cache still awaits the live catalog. The background
refresh is aborted on session_shutdown so print mode does not wait for it.

Closes #63

(cherry picked from commit 142191f9420fe90460cdc3cbae70e26d0c000860)

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Start from the cached model catalog and refresh it in the background instead of blocking host startup on the catalog request; a first start without a cache still waits for the live catalog.
 - Register the `commandcode-custom` API in the `@earendil-works/pi-ai/compat` registry so sibling extensions that stream with the active Command Code model no longer fail with `No API provider registered for api: commandcode-custom` on plain pi.
 - Assert structural catalog invariants in the model tests so the daily catalog sync no longer fails on every upstream change.
 - Display the monthly renewal date and remaining days in `/commandcode-quota`.

README.md

@@ -118,7 +118,7 @@ https://api.commandcode.ai/provider/v1/models
 
 The last successful catalog is cached at `<agent-dir>/commandcode-models.json`. For pi this is `~/.pi/agent/commandcode-models.json` by default. Compatible hosts such as OMP use their own agent directory.
 
-If the endpoint is temporarily unavailable, the provider uses the cached catalog. On a first offline start without a cache, pi still loads, but Command Code models remain unavailable until the connection is restored and `/commandcode-refresh` succeeds.
+When a valid cache exists, the provider registers the cached catalog immediately and refreshes it from the endpoint in the background, so startup does not wait for the network. The refreshed catalog replaces the cached one as soon as it arrives; `/commandcode-status` reports `source: cache` until then. If the endpoint is temporarily unavailable, the cached catalog stays active. On a first start without a cache, the provider waits for the live catalog; if that fails offline, pi still loads, but Command Code models remain unavailable until the connection is restored and `/commandcode-refresh` succeeds.
 
 While pi is running, use these provider commands without restarting:
 

index.ts

@@ -29,6 +29,7 @@ import {
   DEFAULT_PROVIDER_API_BASE,
   getModelsTimeoutMs,
   inputModalitiesForModel,
+  loadCachedCommandCodeModels,
   loadCommandCodeModels,
   MODEL_EFFORTS,
   thinkingMetadataForModel,
@@ -159,15 +160,21 @@ export default async function (pi: ExtensionAPI) {
   const runtime = createCommandCodeRuntime<ProviderConfig, ExtensionCommandContext>(pi, {
     endpoint: modelsUrl,
     cachePath: modelsCachePath,
-    loadModels: () =>
+    loadModels: (signal) =>
       loadCommandCodeModels({
         url: modelsUrl,
         cachePath: modelsCachePath,
         timeoutMs: modelsTimeoutMs,
+        signal,
       }),
+    loadCachedModels: () => loadCachedCommandCodeModels(modelsCachePath),
     createProviderConfig: (models) => createProviderConfig(models, apiBase, transport.stream),
     getTransport: transport.getTransport,
   })
 
+  pi.on("session_shutdown", () => {
+    runtime.dispose()
+  })
+
   await runtime.initialize()
 }

src/models.ts

@@ -332,6 +332,17 @@ async function readCommandCodeModelsCache(cachePath: string): Promise<readonly C
   return commandCodeModelsFromCache(parsed)
 }
 
+/** Reads the cached catalog without touching the network; empty when missing or invalid. */
+export async function loadCachedCommandCodeModels(
+  cachePath: string,
+): Promise<readonly CommandCodeModel[]> {
+  try {
+    return await readCommandCodeModelsCache(cachePath)
+  } catch {
+    return []
+  }
+}
+
 async function writeCommandCodeModelsCache(
   cachePath: string,
   models: readonly CommandCodeModel[],

src/runtime.ts

@@ -26,7 +26,9 @@ export interface CommandCodeRuntimeApi<
 export interface CommandCodeRuntimeOptions<TProviderConfig> {
   endpoint: string
   cachePath: string
-  loadModels: () => Promise<LoadCommandCodeModelsResult>
+  loadModels: (signal: AbortSignal) => Promise<LoadCommandCodeModelsResult>
+  /** Cached catalog only; resolves to an empty list when no valid cache exists. */
+  loadCachedModels: () => Promise<readonly CommandCodeModel[]>
   createProviderConfig: (models: readonly CommandCodeModel[]) => TProviderConfig
   getTransport?: () => "unknown" | "provider" | "generate"
   now?: () => number
@@ -108,6 +110,7 @@ export class CommandCodeRuntime<TProviderConfig, TContext extends CommandCodeCom
   private status: CommandCodeRuntimeStatus
   private providerRegistered = false
   private refreshPromise: Promise<CommandCodeRefreshResult> | undefined
+  private readonly shutdown = new AbortController()
 
   constructor(
     private readonly pi: CommandCodeRuntimeApi<TProviderConfig, TContext>,
@@ -133,9 +136,34 @@ export class CommandCodeRuntime<TProviderConfig, TContext extends CommandCodeCom
     }
   }
 
+  /**
+   * Registers the cached catalog immediately and refreshes it in the
+   * background so host startup does not wait for the network. Without a
+   * valid cache the live refresh is awaited so models are available at once.
+   */
   async initialize(): Promise<void> {
     this.registerCommands()
-    await this.refresh()
+
+    const cached = await this.options.loadCachedModels()
+    if (cached.length === 0) {
+      await this.refresh()
+      return
+    }
+
+    this.pi.registerProvider("commandcode", this.options.createProviderConfig(cached))
+    this.providerRegistered = true
+    this.status = {
+      ...this.status,
+      source: "cache",
+      modelCount: cached.length,
+      lastSuccess: this.now(),
+    }
+    void this.refresh()
+  }
+
+  /** Aborts any background refresh so a stopping host does not wait for the network. */
+  dispose(): void {
+    this.shutdown.abort(new Error("Command Code provider shut down"))
   }
 
   refresh(): Promise<CommandCodeRefreshResult> {
@@ -156,7 +184,7 @@ export class CommandCodeRuntime<TProviderConfig, TContext extends CommandCodeCom
     }
 
     try {
-      const loaded = await this.options.loadModels()
+      const loaded = await this.options.loadModels(this.shutdown.signal)
       const warning = loaded.warning ? redactDiagnosticText(loaded.warning) : undefined
 
       const shouldRegister =
@@ -217,6 +245,14 @@ export class CommandCodeRuntime<TProviderConfig, TContext extends CommandCodeCom
         warning: preservedWarning,
       }
     } catch (error) {
+      if (this.shutdown.signal.aborted) {
+        this.status = { ...this.status, refreshing: false }
+        return {
+          refreshed: false,
+          source: this.status.source,
+          modelCount: this.status.modelCount,
+        }
+      }
       const warning = redactDiagnosticText(
         `Could not refresh the Command Code model catalog: ${errorMessage(error)}`,
       )

tests/test-pi-local.mjs

@@ -753,6 +753,37 @@ try {
   modelsDelayMs = 0
   delete env.COMMANDCODE_MODELS_TIMEOUT_MS
 
+  console.log("[pi-local] cached catalog starts without waiting for slow discovery")
+  const warmup = await runPi(
+    ["--no-extensions", "-e", EXT_PATH, "--list-models", "commandcode"],
+    20_000,
+  )
+  assert.equal(warmup.code, 0, warmup.stderr)
+  assert.doesNotThrow(() => accessSync(modelsCachePath, constants.R_OK))
+  modelsDelayMs = 5_000
+  requestCount = 0
+  const cachedStartedAt = Date.now()
+  const cachedPrint = await runPi(
+    [
+      "--no-extensions",
+      "-e",
+      EXT_PATH,
+      "-p",
+      "say mock token",
+      "--provider",
+      "commandcode",
+      "--model",
+      TEST_MODEL,
+    ],
+    30_000,
+  )
+  const cachedElapsedMs = Date.now() - cachedStartedAt
+  assert.equal(cachedPrint.code, 0, cachedPrint.stderr)
+  assert.match(cachedPrint.stdout, /mock-pi-ok/)
+  assert.equal(requestCount, 1)
+  assert.ok(cachedElapsedMs < 5_000, `cached start took ${cachedElapsedMs}ms`)
+  modelsDelayMs = 0
+
   console.log("[pi-local] print mode with reasoning and tool schemas")
   requestCount = 0
   const print = await runPi(

tests/test-runtime.ts

@@ -97,6 +97,7 @@ describe("Command Code runtime", () => {
       endpoint: "https://api.commandcode.ai/provider/v1/models?token=user_secret_value",
       cachePath: "/tmp/commandcode-models.json",
       loadModels: () => firstLoad.promise,
+      loadCachedModels: async () => [],
       createProviderConfig: (models) => ({ models }),
       getTransport: () => "provider",
       now: () => now,
@@ -105,6 +106,7 @@ describe("Command Code runtime", () => {
 
     const initialization = runtime.initialize()
     assert.deepEqual([...pi.commands.keys()], ["commandcode-refresh", "commandcode-status"])
+    await Promise.resolve()
     assert.equal(runtime.getStatus().refreshing, true)
     assert.equal(runtime.getStatus().lastAttempt, now)
 
@@ -142,6 +144,7 @@ describe("Command Code runtime", () => {
         if (!next) throw new Error("unexpected refresh")
         return next instanceof Promise ? next : next.promise
       },
+      loadCachedModels: async () => [],
       createProviderConfig: (models) => ({ models }),
       logWarning: (warning) => warnings.push(warning),
     })
@@ -189,6 +192,7 @@ describe("Command Code runtime", () => {
         if (!result) throw new Error("unexpected refresh")
         return result
       },
+      loadCachedModels: async () => [],
       createProviderConfig: (models) => ({ models }),
       logWarning: () => {},
     })
@@ -204,6 +208,123 @@ describe("Command Code runtime", () => {
     assert.deepEqual(pi.providers.at(-1)?.models, [FIRST_MODEL, SECOND_MODEL])
   })
 
+  it("registers the cached catalog immediately and refreshes it in the background", async () => {
+    const pi = new ExtensionAPITestDouble()
+    const liveLoad = deferred<LoadCommandCodeModelsResult>()
+    let now = 1_700_000_000_000
+
+    const runtime = createCommandCodeRuntime(pi, {
+      endpoint: "https://api.commandcode.ai/provider/v1/models",
+      cachePath: "/tmp/commandcode-models.json",
+      loadModels: () => liveLoad.promise,
+      loadCachedModels: async () => [FIRST_MODEL],
+      createProviderConfig: (models) => ({ models }),
+      now: () => now,
+      logWarning: () => {},
+    })
+
+    await runtime.initialize()
+    assert.equal(pi.providers.length, 1)
+    assert.deepEqual(pi.providers[0]?.models, [FIRST_MODEL])
+    assert.equal(runtime.getStatus().source, "cache")
+    assert.equal(runtime.getStatus().modelCount, 1)
+    assert.equal(runtime.getStatus().refreshing, true)
+
+    now += 1_000
+    liveLoad.resolve(loaded([FIRST_MODEL, SECOND_MODEL]))
+    await runtime.refresh()
+    assert.equal(pi.providers.length, 2)
+    assert.deepEqual(pi.providers[1]?.models, [FIRST_MODEL, SECOND_MODEL])
+    assert.equal(runtime.getStatus().source, "live")
+    assert.equal(runtime.getStatus().modelCount, 2)
+    assert.equal(runtime.getStatus().refreshing, false)
+  })
+
+  it("keeps the cached catalog when the background refresh fails", async () => {
+    const pi = new ExtensionAPITestDouble()
+    const warnings: string[] = []
+
+    const runtime = createCommandCodeRuntime(pi, {
+      endpoint: "https://api.commandcode.ai/provider/v1/models",
+      cachePath: "/tmp/commandcode-models.json",
+      loadModels: async () => {
+        throw new Error("offline")
+      },
+      loadCachedModels: async () => [FIRST_MODEL],
+      createProviderConfig: (models) => ({ models }),
+      logWarning: (message) => warnings.push(message),
+    })
+
+    await runtime.initialize()
+    await runtime.refresh()
+    assert.equal(pi.providers.length, 1)
+    assert.deepEqual(pi.providers[0]?.models, [FIRST_MODEL])
+    assert.equal(runtime.getStatus().source, "cache")
+    assert.equal(runtime.getStatus().modelCount, 1)
+    assert.match(runtime.getStatus().warning ?? "", /offline/)
+    assert.equal(warnings.length, 1)
+  })
+
+  it("aborts the background refresh on dispose without reporting a warning", async () => {
+    const pi = new ExtensionAPITestDouble()
+    const warnings: string[] = []
+    let refreshSignal: AbortSignal | undefined
+
+    const runtime = createCommandCodeRuntime(pi, {
+      endpoint: "https://api.commandcode.ai/provider/v1/models",
+      cachePath: "/tmp/commandcode-models.json",
+      loadModels: (signal) =>
+        new Promise((_resolve, reject) => {
+          refreshSignal = signal
+          signal.addEventListener("abort", () => reject(signal.reason), { once: true })
+        }),
+      loadCachedModels: async () => [FIRST_MODEL],
+      createProviderConfig: (models) => ({ models }),
+      logWarning: (message) => warnings.push(message),
+    })
+
+    await runtime.initialize()
+    const pending = runtime.refresh()
+    assert.equal(refreshSignal?.aborted, false)
+
+    runtime.dispose()
+    const result = await pending
+    assert.equal(refreshSignal?.aborted, true)
+    assert.equal(result.refreshed, false)
+    assert.equal(runtime.getStatus().refreshing, false)
+    assert.equal(runtime.getStatus().warning, undefined)
+    assert.deepEqual(warnings, [])
+    assert.deepEqual(pi.providers[0]?.models, [FIRST_MODEL])
+  })
+
+  it("awaits the live catalog when no cache exists", async () => {
+    const pi = new ExtensionAPITestDouble()
+    const liveLoad = deferred<LoadCommandCodeModelsResult>()
+
+    const runtime = createCommandCodeRuntime(pi, {
+      endpoint: "https://api.commandcode.ai/provider/v1/models",
+      cachePath: "/tmp/commandcode-models.json",
+      loadModels: () => liveLoad.promise,
+      loadCachedModels: async () => [],
+      createProviderConfig: (models) => ({ models }),
+      logWarning: () => {},
+    })
+
+    let initialized = false
+    const initialization = runtime.initialize().then(() => {
+      initialized = true
+    })
+    await Promise.resolve()
+    assert.equal(pi.providers.length, 0)
+    assert.equal(initialized, false)
+
+    liveLoad.resolve(loaded([FIRST_MODEL]))
+    await initialization
+    assert.equal(initialized, true)
+    assert.deepEqual(pi.providers[0]?.models, [FIRST_MODEL])
+    assert.equal(runtime.getStatus().source, "live")
+  })
+
   it("installs a cached catalog after an initially empty start", async () => {
     const pi = new ExtensionAPITestDouble()
     const results = [
@@ -221,6 +342,7 @@ describe("Command Code runtime", () => {
         if (!result) throw new Error("unexpected refresh")
         return result
       },
+      loadCachedModels: async () => [],
       createProviderConfig: (models) => ({ models }),
       logWarning: () => {},
     })
@@ -254,6 +376,7 @@ describe("Command Code runtime", () => {
         if (!result) throw new Error("unexpected refresh")
         return result
       },
+      loadCachedModels: async () => [],
       createProviderConfig: (models) => ({ models }),
       logWarning: () => {},
     })
@@ -280,6 +403,7 @@ describe("Command Code runtime", () => {
       loadModels: async () => {
         throw new Error("offline; api_key=user_initial_secret")
       },
+      loadCachedModels: async () => [],
       createProviderConfig: (models) => ({ models }),
       logWarning: () => {},
     })