From 59322e4ffabcb3df516ca5720261240de6985b0c Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Tue, 1 Sep 2026 23:18:14 +0200 Subject: [PATCH] perf(models): start from the cached catalog and refresh in the background Every host start awaited a full catalog request before the provider registered, costing one HTTPS round-trip (up to the discovery timeout on a hanging connection) even when a cache written seconds earlier was on disk. Register the cached catalog immediately and run the live refresh in the background; the live result re-registers the provider when it arrives. A first start without a cache still awaits the live catalog. The background refresh is aborted on session_shutdown so print mode does not wait for it. Closes #63 (cherry picked from commit 142191f9420fe90460cdc3cbae70e26d0c000860) --- CHANGELOG.md | 1 + README.md | 2 +- index.ts | 9 ++- src/models.ts | 11 ++++ src/runtime.ts | 42 +++++++++++++- tests/test-pi-local.mjs | 31 ++++++++++ tests/test-runtime.ts | 124 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 215 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab77472..1e85f6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Start from the cached model catalog and refresh it in the background instead of blocking host startup on the catalog request; a first start without a cache still waits for the live catalog. - Register the `commandcode-custom` API in the `@earendil-works/pi-ai/compat` registry so sibling extensions that stream with the active Command Code model no longer fail with `No API provider registered for api: commandcode-custom` on plain pi. - Assert structural catalog invariants in the model tests so the daily catalog sync no longer fails on every upstream change. - Display the monthly renewal date and remaining days in `/commandcode-quota`. diff --git a/README.md b/README.md index 25be44c..26be812 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ https://api.commandcode.ai/provider/v1/models The last successful catalog is cached at `/commandcode-models.json`. For pi this is `~/.pi/agent/commandcode-models.json` by default. Compatible hosts such as OMP use their own agent directory. -If the endpoint is temporarily unavailable, the provider uses the cached catalog. On a first offline start without a cache, pi still loads, but Command Code models remain unavailable until the connection is restored and `/commandcode-refresh` succeeds. +When a valid cache exists, the provider registers the cached catalog immediately and refreshes it from the endpoint in the background, so startup does not wait for the network. The refreshed catalog replaces the cached one as soon as it arrives; `/commandcode-status` reports `source: cache` until then. If the endpoint is temporarily unavailable, the cached catalog stays active. On a first start without a cache, the provider waits for the live catalog; if that fails offline, pi still loads, but Command Code models remain unavailable until the connection is restored and `/commandcode-refresh` succeeds. While pi is running, use these provider commands without restarting: diff --git a/index.ts b/index.ts index 6dce140..12af76c 100644 --- a/index.ts +++ b/index.ts @@ -29,6 +29,7 @@ import { DEFAULT_PROVIDER_API_BASE, getModelsTimeoutMs, inputModalitiesForModel, + loadCachedCommandCodeModels, loadCommandCodeModels, MODEL_EFFORTS, thinkingMetadataForModel, @@ -159,15 +160,21 @@ export default async function (pi: ExtensionAPI) { const runtime = createCommandCodeRuntime(pi, { endpoint: modelsUrl, cachePath: modelsCachePath, - loadModels: () => + loadModels: (signal) => loadCommandCodeModels({ url: modelsUrl, cachePath: modelsCachePath, timeoutMs: modelsTimeoutMs, + signal, }), + loadCachedModels: () => loadCachedCommandCodeModels(modelsCachePath), createProviderConfig: (models) => createProviderConfig(models, apiBase, transport.stream), getTransport: transport.getTransport, }) + pi.on("session_shutdown", () => { + runtime.dispose() + }) + await runtime.initialize() } diff --git a/src/models.ts b/src/models.ts index be7c674..15cdadb 100644 --- a/src/models.ts +++ b/src/models.ts @@ -332,6 +332,17 @@ async function readCommandCodeModelsCache(cachePath: string): Promise { + try { + return await readCommandCodeModelsCache(cachePath) + } catch { + return [] + } +} + async function writeCommandCodeModelsCache( cachePath: string, models: readonly CommandCodeModel[], diff --git a/src/runtime.ts b/src/runtime.ts index e103f96..eb5e842 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -26,7 +26,9 @@ export interface CommandCodeRuntimeApi< export interface CommandCodeRuntimeOptions { endpoint: string cachePath: string - loadModels: () => Promise + loadModels: (signal: AbortSignal) => Promise + /** Cached catalog only; resolves to an empty list when no valid cache exists. */ + loadCachedModels: () => Promise createProviderConfig: (models: readonly CommandCodeModel[]) => TProviderConfig getTransport?: () => "unknown" | "provider" | "generate" now?: () => number @@ -108,6 +110,7 @@ export class CommandCodeRuntime | undefined + private readonly shutdown = new AbortController() constructor( private readonly pi: CommandCodeRuntimeApi, @@ -133,9 +136,34 @@ export class CommandCodeRuntime { this.registerCommands() - await this.refresh() + + const cached = await this.options.loadCachedModels() + if (cached.length === 0) { + await this.refresh() + return + } + + this.pi.registerProvider("commandcode", this.options.createProviderConfig(cached)) + this.providerRegistered = true + this.status = { + ...this.status, + source: "cache", + modelCount: cached.length, + lastSuccess: this.now(), + } + void this.refresh() + } + + /** Aborts any background refresh so a stopping host does not wait for the network. */ + dispose(): void { + this.shutdown.abort(new Error("Command Code provider shut down")) } refresh(): Promise { @@ -156,7 +184,7 @@ export class CommandCodeRuntime accessSync(modelsCachePath, constants.R_OK)) + modelsDelayMs = 5_000 + requestCount = 0 + const cachedStartedAt = Date.now() + const cachedPrint = await runPi( + [ + "--no-extensions", + "-e", + EXT_PATH, + "-p", + "say mock token", + "--provider", + "commandcode", + "--model", + TEST_MODEL, + ], + 30_000, + ) + const cachedElapsedMs = Date.now() - cachedStartedAt + assert.equal(cachedPrint.code, 0, cachedPrint.stderr) + assert.match(cachedPrint.stdout, /mock-pi-ok/) + assert.equal(requestCount, 1) + assert.ok(cachedElapsedMs < 5_000, `cached start took ${cachedElapsedMs}ms`) + modelsDelayMs = 0 + console.log("[pi-local] print mode with reasoning and tool schemas") requestCount = 0 const print = await runPi( diff --git a/tests/test-runtime.ts b/tests/test-runtime.ts index 2a89f12..28f9a42 100644 --- a/tests/test-runtime.ts +++ b/tests/test-runtime.ts @@ -97,6 +97,7 @@ describe("Command Code runtime", () => { endpoint: "https://api.commandcode.ai/provider/v1/models?token=user_secret_value", cachePath: "/tmp/commandcode-models.json", loadModels: () => firstLoad.promise, + loadCachedModels: async () => [], createProviderConfig: (models) => ({ models }), getTransport: () => "provider", now: () => now, @@ -105,6 +106,7 @@ describe("Command Code runtime", () => { const initialization = runtime.initialize() assert.deepEqual([...pi.commands.keys()], ["commandcode-refresh", "commandcode-status"]) + await Promise.resolve() assert.equal(runtime.getStatus().refreshing, true) assert.equal(runtime.getStatus().lastAttempt, now) @@ -142,6 +144,7 @@ describe("Command Code runtime", () => { if (!next) throw new Error("unexpected refresh") return next instanceof Promise ? next : next.promise }, + loadCachedModels: async () => [], createProviderConfig: (models) => ({ models }), logWarning: (warning) => warnings.push(warning), }) @@ -189,6 +192,7 @@ describe("Command Code runtime", () => { if (!result) throw new Error("unexpected refresh") return result }, + loadCachedModels: async () => [], createProviderConfig: (models) => ({ models }), logWarning: () => {}, }) @@ -204,6 +208,123 @@ describe("Command Code runtime", () => { assert.deepEqual(pi.providers.at(-1)?.models, [FIRST_MODEL, SECOND_MODEL]) }) + it("registers the cached catalog immediately and refreshes it in the background", async () => { + const pi = new ExtensionAPITestDouble() + const liveLoad = deferred() + let now = 1_700_000_000_000 + + const runtime = createCommandCodeRuntime(pi, { + endpoint: "https://api.commandcode.ai/provider/v1/models", + cachePath: "/tmp/commandcode-models.json", + loadModels: () => liveLoad.promise, + loadCachedModels: async () => [FIRST_MODEL], + createProviderConfig: (models) => ({ models }), + now: () => now, + logWarning: () => {}, + }) + + await runtime.initialize() + assert.equal(pi.providers.length, 1) + assert.deepEqual(pi.providers[0]?.models, [FIRST_MODEL]) + assert.equal(runtime.getStatus().source, "cache") + assert.equal(runtime.getStatus().modelCount, 1) + assert.equal(runtime.getStatus().refreshing, true) + + now += 1_000 + liveLoad.resolve(loaded([FIRST_MODEL, SECOND_MODEL])) + await runtime.refresh() + assert.equal(pi.providers.length, 2) + assert.deepEqual(pi.providers[1]?.models, [FIRST_MODEL, SECOND_MODEL]) + assert.equal(runtime.getStatus().source, "live") + assert.equal(runtime.getStatus().modelCount, 2) + assert.equal(runtime.getStatus().refreshing, false) + }) + + it("keeps the cached catalog when the background refresh fails", async () => { + const pi = new ExtensionAPITestDouble() + const warnings: string[] = [] + + const runtime = createCommandCodeRuntime(pi, { + endpoint: "https://api.commandcode.ai/provider/v1/models", + cachePath: "/tmp/commandcode-models.json", + loadModels: async () => { + throw new Error("offline") + }, + loadCachedModels: async () => [FIRST_MODEL], + createProviderConfig: (models) => ({ models }), + logWarning: (message) => warnings.push(message), + }) + + await runtime.initialize() + await runtime.refresh() + assert.equal(pi.providers.length, 1) + assert.deepEqual(pi.providers[0]?.models, [FIRST_MODEL]) + assert.equal(runtime.getStatus().source, "cache") + assert.equal(runtime.getStatus().modelCount, 1) + assert.match(runtime.getStatus().warning ?? "", /offline/) + assert.equal(warnings.length, 1) + }) + + it("aborts the background refresh on dispose without reporting a warning", async () => { + const pi = new ExtensionAPITestDouble() + const warnings: string[] = [] + let refreshSignal: AbortSignal | undefined + + const runtime = createCommandCodeRuntime(pi, { + endpoint: "https://api.commandcode.ai/provider/v1/models", + cachePath: "/tmp/commandcode-models.json", + loadModels: (signal) => + new Promise((_resolve, reject) => { + refreshSignal = signal + signal.addEventListener("abort", () => reject(signal.reason), { once: true }) + }), + loadCachedModels: async () => [FIRST_MODEL], + createProviderConfig: (models) => ({ models }), + logWarning: (message) => warnings.push(message), + }) + + await runtime.initialize() + const pending = runtime.refresh() + assert.equal(refreshSignal?.aborted, false) + + runtime.dispose() + const result = await pending + assert.equal(refreshSignal?.aborted, true) + assert.equal(result.refreshed, false) + assert.equal(runtime.getStatus().refreshing, false) + assert.equal(runtime.getStatus().warning, undefined) + assert.deepEqual(warnings, []) + assert.deepEqual(pi.providers[0]?.models, [FIRST_MODEL]) + }) + + it("awaits the live catalog when no cache exists", async () => { + const pi = new ExtensionAPITestDouble() + const liveLoad = deferred() + + const runtime = createCommandCodeRuntime(pi, { + endpoint: "https://api.commandcode.ai/provider/v1/models", + cachePath: "/tmp/commandcode-models.json", + loadModels: () => liveLoad.promise, + loadCachedModels: async () => [], + createProviderConfig: (models) => ({ models }), + logWarning: () => {}, + }) + + let initialized = false + const initialization = runtime.initialize().then(() => { + initialized = true + }) + await Promise.resolve() + assert.equal(pi.providers.length, 0) + assert.equal(initialized, false) + + liveLoad.resolve(loaded([FIRST_MODEL])) + await initialization + assert.equal(initialized, true) + assert.deepEqual(pi.providers[0]?.models, [FIRST_MODEL]) + assert.equal(runtime.getStatus().source, "live") + }) + it("installs a cached catalog after an initially empty start", async () => { const pi = new ExtensionAPITestDouble() const results = [ @@ -221,6 +342,7 @@ describe("Command Code runtime", () => { if (!result) throw new Error("unexpected refresh") return result }, + loadCachedModels: async () => [], createProviderConfig: (models) => ({ models }), logWarning: () => {}, }) @@ -254,6 +376,7 @@ describe("Command Code runtime", () => { if (!result) throw new Error("unexpected refresh") return result }, + loadCachedModels: async () => [], createProviderConfig: (models) => ({ models }), logWarning: () => {}, }) @@ -280,6 +403,7 @@ describe("Command Code runtime", () => { loadModels: async () => { throw new Error("offline; api_key=user_initial_secret") }, + loadCachedModels: async () => [], createProviderConfig: (models) => ({ models }), logWarning: () => {}, })