feat(models): add model-aware runtime metadata

tests/test-models.ts

@@ -7,7 +7,12 @@ import { describe, it } from "node:test"
 import {
   commandCodeModelsFromApiResponse,
   commandCodeModelsFromCache,
+  DEFAULT_MODELS_TIMEOUT_MS,
+  getModelsTimeoutMs,
   loadCommandCodeModels,
+  MODEL_EFFORTS,
+  thinkingLevelMapForEfforts,
+  thinkingMetadataForModel,
   type CommandCodeModel,
 } from "../src/models.ts"
 
@@ -29,7 +34,7 @@ const EXPECTED_MODELS: readonly CommandCodeModel[] = [
   {
     id: "Qwen/Qwen3.7-Max",
     name: "Qwen 3.7 Max (CC)",
-    reasoning: true,
+    reasoning: false,
     contextWindow: 1_000_000,
     maxTokens: 65_536,
   },
@@ -49,6 +54,17 @@ function failingFetch(message = "offline"): typeof fetch {
   return () => Promise.reject(new TypeError(message))
 }
 
+function hangingFetch(): typeof fetch {
+  return (_input, init) =>
+    new Promise<Response>((_resolve, reject) => {
+      init?.signal?.addEventListener(
+        "abort",
+        () => reject(init.signal?.reason ?? new DOMException("Aborted", "AbortError")),
+        { once: true },
+      )
+    })
+}
+
 async function withTemporaryCache(
   run: (paths: { directory: string; cachePath: string }) => Promise<void>,
 ): Promise<void> {
@@ -65,6 +81,44 @@ describe("commandCodeModelsFromApiResponse()", () => {
     assert.deepEqual(commandCodeModelsFromApiResponse(API_RESPONSE), EXPECTED_MODELS)
   })
 
+  it("marks only known reasoning models as reasoning-capable", () => {
+    const models = commandCodeModelsFromApiResponse({
+      object: "list",
+      data: [
+        { ...API_RESPONSE.data[0], id: "deepseek/deepseek-v4-flash" },
+        { ...API_RESPONSE.data[0], id: "new-model-without-metadata" },
+      ],
+    })
+
+    assert.equal(models[0]?.reasoning, true)
+    assert.equal(models[1]?.reasoning, false)
+  })
+
+  it("builds explicit maps for every known effort set", () => {
+    for (const [modelId, efforts] of Object.entries(MODEL_EFFORTS)) {
+      const metadata = thinkingMetadataForModel(modelId)
+      assert.ok(metadata, `${modelId} should have reasoning metadata`)
+      for (const level of ["minimal", "low", "medium", "high", "xhigh", "max"] as const) {
+        const expected = efforts.includes(level)
+        assert.equal(
+          metadata.thinkingLevelMap[level],
+          expected ? level : null,
+          `${modelId} should map ${level} according to its catalog entry`,
+        )
+      }
+    }
+
+    assert.deepEqual(thinkingLevelMapForEfforts(MODEL_EFFORTS["deepseek/deepseek-v4-flash"]), {
+      minimal: null,
+      low: null,
+      medium: null,
+      high: "high",
+      xhigh: null,
+      max: "max",
+    })
+    assert.deepEqual(thinkingMetadataForModel("new-model-without-metadata"), undefined)
+  })
+
   it("rejects unexpected API shapes", () => {
     assert.throws(() => commandCodeModelsFromApiResponse({ object: "list", data: [{}] }))
   })
@@ -78,6 +132,20 @@ describe("commandCodeModelsFromCache()", () => {
     )
   })
 
+  it("normalizes cached reasoning metadata from the model id", () => {
+    const cached = commandCodeModelsFromCache({
+      version: 1,
+      models: [
+        {
+          ...EXPECTED_MODELS[0],
+          id: "deepseek/deepseek-v4-flash",
+          reasoning: false,
+        },
+      ],
+    })
+    assert.equal(cached[0]?.reasoning, true)
+  })
+
   it("rejects empty, invalid, and unsupported caches", () => {
     assert.throws(() => commandCodeModelsFromCache({ version: 1, models: [] }))
     assert.throws(() => commandCodeModelsFromCache({ version: 2, models: EXPECTED_MODELS }))
@@ -90,7 +158,58 @@ describe("commandCodeModelsFromCache()", () => {
   })
 })
 
+describe("model discovery configuration", () => {
+  it("uses a safe default timeout and ignores invalid environment values", () => {
+    assert.equal(getModelsTimeoutMs({}), DEFAULT_MODELS_TIMEOUT_MS)
+    assert.equal(
+      getModelsTimeoutMs({ COMMANDCODE_MODELS_TIMEOUT_MS: "0" }),
+      DEFAULT_MODELS_TIMEOUT_MS,
+    )
+    assert.equal(
+      getModelsTimeoutMs({ COMMANDCODE_MODELS_TIMEOUT_MS: "invalid" }),
+      DEFAULT_MODELS_TIMEOUT_MS,
+    )
+    assert.equal(getModelsTimeoutMs({ COMMANDCODE_MODELS_TIMEOUT_MS: "25" }), 25)
+  })
+})
+
 describe("loadCommandCodeModels()", () => {
+  it("falls back to cache when live discovery times out", async () => {
+    await withTemporaryCache(async ({ cachePath }) => {
+      await loadCommandCodeModels({ cachePath, fetchImpl: successfulFetch() })
+
+      const startedAt = Date.now()
+      const result = await loadCommandCodeModels({
+        cachePath,
+        fetchImpl: hangingFetch(),
+        timeoutMs: 25,
+      })
+
+      assert.ok(Date.now() - startedAt < 500)
+      assert.deepEqual(result.models, EXPECTED_MODELS)
+      assert.equal(result.source, "cache")
+      assert.match(result.warning ?? "", /timed out after 25ms/)
+      assert.match(result.warning ?? "", /Using the cached catalog/)
+    })
+  })
+
+  it("preserves an external abort instead of falling back to cache", async () => {
+    await withTemporaryCache(async ({ cachePath }) => {
+      await loadCommandCodeModels({ cachePath, fetchImpl: successfulFetch() })
+      const controller = new AbortController()
+      const promise = loadCommandCodeModels({
+        cachePath,
+        fetchImpl: hangingFetch(),
+        timeoutMs: 1_000,
+        signal: controller.signal,
+      })
+
+      controller.abort(new Error("caller cancelled discovery"))
+
+      await assert.rejects(promise, /caller cancelled discovery/)
+    })
+  })
+
   it("returns live models and writes a validated cache", async () => {
     await withTemporaryCache(async ({ cachePath }) => {
       const result = await loadCommandCodeModels({
@@ -132,7 +251,7 @@ describe("loadCommandCodeModels()", () => {
       assert.deepEqual(result.models, [])
       assert.equal(result.source, "empty")
       assert.match(result.warning ?? "", /no valid cached catalog/)
-      assert.match(result.warning ?? "", /until \/reload succeeds/)
+      assert.match(result.warning ?? "", /until \/commandcode-refresh succeeds/)
     })
   })
 

tests/test-pi-local.mjs

@@ -277,9 +277,12 @@ try {
   )
   assert.equal(firstOfflineList.code, 0, firstOfflineList.stderr)
   assert.doesNotMatch(firstOfflineList.stderr, /Failed to load extension/)
-  assert.match(firstOfflineList.stdout || firstOfflineList.stderr, /No models matching/)
+  assert.match(
+    firstOfflineList.stdout || firstOfflineList.stderr,
+    /No models matching|No models available/,
+  )
   assert.match(firstOfflineList.stderr, /no valid cached catalog/)
-  assert.match(firstOfflineList.stderr, /until \/reload succeeds/)
+  assert.match(firstOfflineList.stderr, /until \/commandcode-refresh succeeds/)
   assert.throws(() => accessSync(modelsCachePath, constants.R_OK), /ENOENT|no such file/i)
 
   // A fresh process re-runs the extension entrypoint, which is the same path /reload uses.

tests/test-runtime.ts

@@ -0,0 +1,293 @@
+import assert from "node:assert/strict"
+import { describe, it } from "node:test"
+
+import {
+  createCommandCodeRuntime,
+  type CommandCodeCommandContext,
+  type CommandCodeRuntimeApi,
+} from "../src/runtime.ts"
+import type { CommandCodeModel, LoadCommandCodeModelsResult } from "../src/models.ts"
+
+type ProviderConfig = {
+  models: readonly CommandCodeModel[]
+}
+
+class ExtensionAPITestDouble implements CommandCodeRuntimeApi<ProviderConfig, CommandContext> {
+  readonly providers: ProviderConfig[] = []
+  readonly commands = new Map<string, (args: string, ctx: CommandContext) => Promise<void> | void>()
+
+  registerProvider(_name: string, config: ProviderConfig): void {
+    this.providers.push(config)
+  }
+
+  registerCommand(
+    name: string,
+    options: {
+      description: string
+      handler: (args: string, ctx: CommandContext) => Promise<void> | void
+    },
+  ): void {
+    this.commands.set(name, options.handler)
+  }
+}
+
+class CommandContext implements CommandCodeCommandContext {
+  readonly notifications: Array<{ message: string; type?: "info" | "warning" | "error" }> = []
+  waitForIdleCalls = 0
+
+  readonly ui = {
+    notify: (message: string, type?: "info" | "warning" | "error") => {
+      this.notifications.push({ message, type })
+    },
+  }
+
+  async waitForIdle(): Promise<void> {
+    this.waitForIdleCalls += 1
+  }
+}
+
+const FIRST_MODEL: CommandCodeModel = {
+  id: "first-model",
+  name: "First Model",
+  reasoning: true,
+  contextWindow: 128_000,
+  maxTokens: 16_384,
+}
+
+const SECOND_MODEL: CommandCodeModel = {
+  id: "second-model",
+  name: "Second Model",
+  reasoning: true,
+  contextWindow: 256_000,
+  maxTokens: 32_768,
+}
+
+function loaded(
+  models: readonly CommandCodeModel[],
+  source: LoadCommandCodeModelsResult["source"] = "live",
+  warning?: string,
+): LoadCommandCodeModelsResult {
+  return warning ? { models, source, warning } : { models, source }
+}
+
+function deferred<T>(): {
+  promise: Promise<T>
+  resolve(value: T): void
+  reject(error: unknown): void
+} {
+  let resolvePromise: (value: T) => void = () => {}
+  let rejectPromise: (error: unknown) => void = () => {}
+  const promise = new Promise<T>((resolve, reject) => {
+    resolvePromise = resolve
+    rejectPromise = reject
+  })
+  return { promise, resolve: resolvePromise, reject: rejectPromise }
+}
+
+describe("Command Code runtime", () => {
+  it("registers refresh and status commands and exposes redacted state", async () => {
+    const pi = new ExtensionAPITestDouble()
+    const context = new CommandContext()
+    let now = 1_700_000_000_000
+    const firstLoad = deferred<LoadCommandCodeModelsResult>()
+
+    const runtime = createCommandCodeRuntime(pi, {
+      endpoint: "https://api.commandcode.ai/provider/v1/models?token=user_secret_value",
+      cachePath: "/tmp/commandcode-models.json",
+      loadModels: () => firstLoad.promise,
+      createProviderConfig: (models) => ({ models }),
+      now: () => now,
+      logWarning: () => {},
+    })
+
+    const initialization = runtime.initialize()
+    assert.deepEqual([...pi.commands.keys()], ["commandcode-refresh", "commandcode-status"])
+    assert.equal(runtime.getStatus().refreshing, true)
+    assert.equal(runtime.getStatus().lastAttempt, now)
+
+    firstLoad.resolve(loaded([FIRST_MODEL]))
+    await initialization
+    now += 1_000
+
+    const statusCommand = pi.commands.get("commandcode-status")
+    assert.ok(statusCommand)
+    await statusCommand("", context)
+    const statusMessage = context.notifications.at(-1)?.message ?? ""
+    assert.match(statusMessage, /source: live/)
+    assert.match(statusMessage, /model count: 1/)
+    assert.match(statusMessage, /last success:/)
+    assert.match(statusMessage, /last attempt:/)
+    assert.match(statusMessage, /cache path: \/tmp\/commandcode-models\.json/)
+    assert.match(statusMessage, /endpoint: https:\/\/api\.commandcode\.ai\/provider\/v1\/models/)
+    assert.doesNotMatch(statusMessage, /token=user_secret_value/)
+    assert.doesNotMatch(statusMessage, /user_secret_value/)
+  })
+
+  it("coalesces overlapping refreshes and preserves the current catalog on failure", async () => {
+    const pi = new ExtensionAPITestDouble()
+    const warnings: string[] = []
+    const loads = [Promise.resolve(loaded([FIRST_MODEL])), deferred<LoadCommandCodeModelsResult>()]
+    let loadCount = 0
+
+    const runtime = createCommandCodeRuntime(pi, {
+      endpoint: "https://api.commandcode.ai/provider/v1/models",
+      cachePath: "/tmp/commandcode-models.json",
+      loadModels: () => {
+        const next = loads[loadCount]
+        loadCount += 1
+        if (!next) throw new Error("unexpected refresh")
+        return next instanceof Promise ? next : next.promise
+      },
+      createProviderConfig: (models) => ({ models }),
+      logWarning: (warning) => warnings.push(warning),
+    })
+
+    await runtime.initialize()
+    assert.equal(pi.providers.length, 1)
+    assert.deepEqual(pi.providers[0]?.models, [FIRST_MODEL])
+
+    const pending = loads[1]
+    assert.ok(!(pending instanceof Promise))
+    const firstRefresh = runtime.refresh()
+    const secondRefresh = runtime.refresh()
+    assert.strictEqual(firstRefresh, secondRefresh)
+    assert.equal(runtime.getStatus().refreshing, true)
+
+    pending.reject(new Error("request failed with apiKey=user_secret_value"))
+    const result = await firstRefresh
+
+    assert.equal(result.refreshed, false)
+    assert.equal(result.modelCount, 1)
+    assert.equal(runtime.getStatus().modelCount, 1)
+    assert.equal(runtime.getStatus().source, "live")
+    assert.equal(pi.providers.length, 1)
+    assert.equal(runtime.getStatus().refreshing, false)
+    assert.match(runtime.getStatus().warning ?? "", /Could not refresh/)
+    assert.doesNotMatch(runtime.getStatus().warning ?? "", /user_secret_value/)
+    assert.doesNotMatch(warnings.join("\n"), /user_secret_value/)
+  })
+
+  it("runs the refresh command and reports the updated catalog", async () => {
+    const pi = new ExtensionAPITestDouble()
+    const context = new CommandContext()
+    const results = [
+      Promise.resolve(loaded([FIRST_MODEL])),
+      Promise.resolve(loaded([FIRST_MODEL, SECOND_MODEL])),
+    ]
+    let index = 0
+
+    const runtime = createCommandCodeRuntime(pi, {
+      endpoint: "https://api.commandcode.ai/provider/v1/models",
+      cachePath: "/tmp/commandcode-models.json",
+      loadModels: () => {
+        const result = results[index]
+        index += 1
+        if (!result) throw new Error("unexpected refresh")
+        return result
+      },
+      createProviderConfig: (models) => ({ models }),
+      logWarning: () => {},
+    })
+
+    await runtime.initialize()
+    const refreshCommand = pi.commands.get("commandcode-refresh")
+    assert.ok(refreshCommand)
+    await refreshCommand("", context)
+
+    assert.equal(context.waitForIdleCalls, 1)
+    assert.equal(context.notifications.at(-1)?.type, "info")
+    assert.match(context.notifications.at(-1)?.message ?? "", /2 models from live/)
+    assert.deepEqual(pi.providers.at(-1)?.models, [FIRST_MODEL, SECOND_MODEL])
+  })
+
+  it("installs a cached catalog after an initially empty start", async () => {
+    const pi = new ExtensionAPITestDouble()
+    const results = [
+      Promise.resolve(loaded([], "empty", "offline")),
+      Promise.resolve(loaded([SECOND_MODEL], "cache")),
+    ]
+    let index = 0
+
+    const runtime = createCommandCodeRuntime(pi, {
+      endpoint: "https://api.commandcode.ai/provider/v1/models",
+      cachePath: "/tmp/commandcode-models.json",
+      loadModels: () => {
+        const result = results[index]
+        index += 1
+        if (!result) throw new Error("unexpected refresh")
+        return result
+      },
+      createProviderConfig: (models) => ({ models }),
+      logWarning: () => {},
+    })
+
+    await runtime.initialize()
+    assert.equal(pi.providers.length, 1)
+    assert.deepEqual(pi.providers[0]?.models, [])
+
+    const result = await runtime.refresh()
+    assert.equal(result.refreshed, true)
+    assert.equal(result.source, "cache")
+    assert.deepEqual(pi.providers.at(-1)?.models, [SECOND_MODEL])
+    assert.equal(runtime.getStatus().modelCount, 1)
+  })
+
+  it("does not replace an existing provider with an empty failed catalog", async () => {
+    const pi = new ExtensionAPITestDouble()
+    const results = [
+      Promise.resolve(loaded([FIRST_MODEL])),
+      Promise.resolve(loaded([], "cache", "No valid catalog is available at /private/cache")),
+      Promise.resolve(loaded([SECOND_MODEL])),
+    ]
+    let index = 0
+
+    const runtime = createCommandCodeRuntime(pi, {
+      endpoint: "http://127.0.0.1:1234/provider/v1/models",
+      cachePath: "/private/cache",
+      loadModels: () => {
+        const result = results[index]
+        index += 1
+        if (!result) throw new Error("unexpected refresh")
+        return result
+      },
+      createProviderConfig: (models) => ({ models }),
+      logWarning: () => {},
+    })
+
+    await runtime.initialize()
+    const refreshResult = await runtime.refresh()
+    assert.equal(refreshResult.refreshed, false)
+    assert.equal(pi.providers.length, 1)
+    assert.deepEqual(pi.providers[0]?.models, [FIRST_MODEL])
+    assert.equal(runtime.getStatus().modelCount, 1)
+    assert.equal(runtime.getStatus().source, "live")
+
+    await runtime.refresh()
+    assert.equal(pi.providers.length, 2)
+    assert.deepEqual(pi.providers[1]?.models, [SECOND_MODEL])
+  })
+
+  it("reports a failed initial refresh without leaking diagnostics", async () => {
+    const pi = new ExtensionAPITestDouble()
+    const context = new CommandContext()
+    const runtime = createCommandCodeRuntime(pi, {
+      endpoint: "https://api.commandcode.ai/provider/v1/models?api_key=user_initial_secret",
+      cachePath: "/tmp/commandcode-models.json",
+      loadModels: async () => {
+        throw new Error("offline; api_key=user_initial_secret")
+      },
+      createProviderConfig: (models) => ({ models }),
+      logWarning: () => {},
+    })
+
+    await runtime.initialize()
+    const statusCommand = pi.commands.get("commandcode-status")
+    assert.ok(statusCommand)
+    await statusCommand("", context)
+    const message = context.notifications.at(-1)?.message ?? ""
+    assert.match(message, /source: empty/)
+    assert.match(message, /model count: 0/)
+    assert.match(message, /warning:/)
+    assert.doesNotMatch(message, /user_initial_secret/)
+  })
+})

tests/test-stream.ts

@@ -7,6 +7,7 @@ import assert from "node:assert/strict"
 import { after, before, beforeEach, describe, it } from "node:test"
 
 import type { AssistantMessageEvent } from "../src/core.ts"
+import { MODEL_EFFORTS, thinkingLevelMapForEfforts } from "../src/models.ts"
 import {
   collectEvents,
   createTestDeps,
@@ -415,6 +416,84 @@ describe("streamCommandCode — request serialization", () => {
     assert.equal(headers["x-session-id"], undefined)
   })
 
+  it("accepts the legacy OMP nested reasoning map", async () => {
+    server.mockResponse({
+      type: "success",
+      events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
+    })
+    const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
+    const model = makeModel({
+      id: "omp-compat-reasoning-model",
+      reasoning: true,
+      thinking: { effortMap: { high: "legacy-high" } },
+    })
+
+    await collectEvents(
+      streamCommandCode(model, makeContext(), { apiKey: "mock-key", reasoning: "high" }),
+    )
+
+    assert.equal(objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]), "legacy-high")
+  })
+
+  it("forwards a supported Pi reasoning level as reasoning_effort", async () => {
+    server.mockResponse({
+      type: "success",
+      events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
+    })
+    const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
+    const model = makeModel({
+      id: "deepseek/deepseek-v4-flash",
+      reasoning: true,
+      thinkingLevelMap: thinkingLevelMapForEfforts(MODEL_EFFORTS["deepseek/deepseek-v4-flash"]),
+    })
+
+    await collectEvents(
+      streamCommandCode(model, makeContext(), { apiKey: "mock-key", reasoning: "max" }),
+    )
+
+    assert.equal(objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]), "max")
+  })
+
+  it("omits reasoning_effort for off, unsupported, and unknown reasoning levels", async () => {
+    const model = makeModel({
+      id: "deepseek/deepseek-v4-flash",
+      reasoning: true,
+      thinkingLevelMap: thinkingLevelMapForEfforts(MODEL_EFFORTS["deepseek/deepseek-v4-flash"]),
+    })
+
+    for (const reasoning of ["off", "low"] as const) {
+      server.mockResponse({
+        type: "success",
+        events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
+      })
+      const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
+
+      await collectEvents(
+        streamCommandCode(model, makeContext(), { apiKey: "mock-key", reasoning }),
+      )
+      assert.equal(
+        objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]),
+        undefined,
+        `${reasoning} should not be sent when it has no supported Command Code field`,
+      )
+      server.reset()
+    }
+
+    server.mockResponse({
+      type: "success",
+      events: [JSON.stringify({ type: "finish", finishReason: "stop" })],
+    })
+    const { streamCommandCode } = createTestDeps({ apiBase: server.baseUrl() })
+    await collectEvents(
+      streamCommandCode(
+        makeModel({ id: "new-model-without-metadata", reasoning: false }),
+        makeContext(),
+        { apiKey: "mock-key", reasoning: "high" },
+      ),
+    )
+    assert.equal(objectAt(server.lastRequestBody(), ["params", "reasoning_effort"]), undefined)
+  })
+
   it("caps maxTokens and passes custom headers", async () => {
     server.mockResponse({
       type: "success",