perf(models): start from the cached catalog and refresh in the background
Every host start awaited a full catalog request before the provider registered, costing one HTTPS round-trip (up to the discovery timeout on a hanging connection) even when a cache written seconds earlier was on disk. Register the cached catalog immediately and run the live refresh in the background; the live result re-registers the provider when it arrives. A first start without a cache still awaits the live catalog. The background refresh is aborted on session_shutdown so print mode does not wait for it. Closes #63 (cherry picked from commit 142191f9420fe90460cdc3cbae70e26d0c000860)
This commit is contained in:
@@ -29,6 +29,7 @@ import {
|
||||
DEFAULT_PROVIDER_API_BASE,
|
||||
getModelsTimeoutMs,
|
||||
inputModalitiesForModel,
|
||||
loadCachedCommandCodeModels,
|
||||
loadCommandCodeModels,
|
||||
MODEL_EFFORTS,
|
||||
thinkingMetadataForModel,
|
||||
@@ -159,15 +160,21 @@ export default async function (pi: ExtensionAPI) {
|
||||
const runtime = createCommandCodeRuntime<ProviderConfig, ExtensionCommandContext>(pi, {
|
||||
endpoint: modelsUrl,
|
||||
cachePath: modelsCachePath,
|
||||
loadModels: () =>
|
||||
loadModels: (signal) =>
|
||||
loadCommandCodeModels({
|
||||
url: modelsUrl,
|
||||
cachePath: modelsCachePath,
|
||||
timeoutMs: modelsTimeoutMs,
|
||||
signal,
|
||||
}),
|
||||
loadCachedModels: () => loadCachedCommandCodeModels(modelsCachePath),
|
||||
createProviderConfig: (models) => createProviderConfig(models, apiBase, transport.stream),
|
||||
getTransport: transport.getTransport,
|
||||
})
|
||||
|
||||
pi.on("session_shutdown", () => {
|
||||
runtime.dispose()
|
||||
})
|
||||
|
||||
await runtime.initialize()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user