From 8b5e4d78ba8f921d5195a1a51a84dbb0ee70d2fd Mon Sep 17 00:00:00 2001 From: Thomas Byr Date: Fri, 28 Aug 2026 17:00:39 +0200 Subject: [PATCH] feat(models): add refresh-model-catalog skill Add an agent skill that walks through a full model catalog refresh: drift detection, catalog sync, manually reviewed pricing updates, fixture refresh, and test updates. Its helper scripts run on Windows and Linux: one snapshots the live model-id list into the test fixture, the other regenerates the pricing fixture from MODEL_COSTS through prettier so format:check stays green. Typecheck now covers the skill scripts. --- .agents/skills/refresh-model-catalog/SKILL.md | 76 +++++++++++++++++++ .../scripts/refresh-model-ids.mjs | 40 ++++++++++ .../scripts/sync-pricing-fixture.ts | 43 +++++++++++ CHANGELOG.md | 1 + tsconfig.json | 2 +- 5 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 .agents/skills/refresh-model-catalog/SKILL.md create mode 100644 .agents/skills/refresh-model-catalog/scripts/refresh-model-ids.mjs create mode 100644 .agents/skills/refresh-model-catalog/scripts/sync-pricing-fixture.ts diff --git a/.agents/skills/refresh-model-catalog/SKILL.md b/.agents/skills/refresh-model-catalog/SKILL.md new file mode 100644 index 0000000..8a598f5 --- /dev/null +++ b/.agents/skills/refresh-model-catalog/SKILL.md @@ -0,0 +1,76 @@ +--- +name: refresh-model-catalog +description: Use when adding or removing Command Code models, refreshing the model catalog snapshot (image, reasoning, effort, output-limit metadata), updating display pricing, or refreshing test fixtures in pi-commandcode-provider. +--- + +# Refresh Model Catalog + +Use this skill whenever the Command Code model catalog changes: new or retired models, changed reasoning efforts, output limits, or pricing. All commands run from the repository root and work on Windows and Linux. + +## Core rules + +- Do not commit, tag, push, or publish unless the user explicitly asks in the current conversation. +- Pricing is manually reviewed: temporary promotions and long-context tiers require explicit review of the official pricing page. Never copy prices blindly from the API. +- Keep the change focused: one refresh per PR, no unrelated refactors. +- Follow [CONTRIBUTING.md](../../../CONTRIBUTING.md) for commit message rules. + +## Workflow + +### 1. Detect drift + +```sh +npm run check:commandcode-catalog +``` + +This compares the repository snapshot against the latest published `command-code` npm package and reports added/removed models, changed efforts, and version drift. Use the report to scope the work. + +### 2. Sync static model metadata + +```sh +npm run sync:commandcode-catalog +``` + +Regenerates `src/commandcode-catalog.ts` and bumps the documented CLI version in `README.md`. Review the diff; the catalog also lists reasoning models without selectable efforts. + +### 3. Update display pricing (manual review) + +Fetch and compare against `src/pricing.ts`: + +- Add entries for new models and remove entries for retired models. Missing models silently display zero cost, so `MODEL_COSTS` must cover the full catalog. +- The pricing page's "Cache Read"/"Cache Write" columns map to `cacheRead`/`cacheWrite`; a "—" column means `0`. +- Update `PRICING_LAST_VERIFIED` to today's date. +- Add or update `TEMPORARY_PRICING` entries for promotions with an end date, so tests fail when they expire. + +### 4. Refresh the test fixtures + +```sh +node .agents/skills/refresh-model-catalog/scripts/refresh-model-ids.mjs +npx tsx .agents/skills/refresh-model-catalog/scripts/sync-pricing-fixture.ts +``` + +The first script snapshots the live model-id list into `tests/fixtures/commandcode-model-ids.json`; the second regenerates `tests/fixtures/commandcode-pricing.json` from `MODEL_COSTS`. The pricing test fails until `MODEL_COSTS` matches the catalog snapshot exactly. + +### 5. Update test expectations + +Adjust the model-specific assertions that the refresh invalidated, typically in: + +- `tests/test-pricing.ts`: fixture date assertions, the `freeModels` set, and per-model rate assertions. +- `tests/test-models.ts`: image/reasoning/effort/output-limit assertions and catalog entry counts. + +Do not weaken assertions to make them pass; update them to the verified upstream values. + +### 6. Validate + +```sh +npm run test:models +npm run test:pricing +npm run typecheck +npm run format:check +git diff --check +``` + +Run the full `npm test` before reporting the work as done when the environment allows it. + +### 7. Document + +Add entries to the `Unreleased` section of `CHANGELOG.md` covering new/retired models, effort changes, and pricing refreshes. diff --git a/.agents/skills/refresh-model-catalog/scripts/refresh-model-ids.mjs b/.agents/skills/refresh-model-catalog/scripts/refresh-model-ids.mjs new file mode 100644 index 0000000..f9be878 --- /dev/null +++ b/.agents/skills/refresh-model-catalog/scripts/refresh-model-ids.mjs @@ -0,0 +1,40 @@ +#!/usr/bin/env node +// Refreshes tests/fixtures/commandcode-model-ids.json from the live Command Code +// models API. Run from the repository root: +// node .agents/skills/refresh-model-catalog/scripts/refresh-model-ids.mjs +import { writeFile } from "node:fs/promises" + +import { format, resolveConfig } from "prettier" + +const MODELS_URL = "https://api.commandcode.ai/provider/v1/models" +const FIXTURE_PATH = new URL( + "../../../../tests/fixtures/commandcode-model-ids.json", + import.meta.url, +) + +const response = await fetch(MODELS_URL) +if (!response.ok) { + throw new Error(`Failed to fetch Command Code models: ${response.status} ${response.statusText}`) +} + +const body = await response.json() +if (body?.object !== "list" || !Array.isArray(body.data)) { + throw new Error("Expected a Command Code models list response") +} + +const modelIds = body.data.map((model) => { + if (typeof model?.id !== "string" || model.id.length === 0) { + throw new Error("Expected each model entry to have a non-empty id") + } + return model.id +}) +if (modelIds.length === 0) throw new Error("Command Code returned an empty model catalog") + +const fixture = { fetchedAt: new Date().toISOString(), source: MODELS_URL, modelIds } +const options = await resolveConfig(new URL("../../../../.prettierrc.json", import.meta.url)) +const contents = await format(JSON.stringify(fixture), { + ...options, + filepath: "commandcode-model-ids.json", +}) +await writeFile(FIXTURE_PATH, contents, "utf-8") +console.log(`Wrote ${modelIds.length} model ids to tests/fixtures/commandcode-model-ids.json`) diff --git a/.agents/skills/refresh-model-catalog/scripts/sync-pricing-fixture.ts b/.agents/skills/refresh-model-catalog/scripts/sync-pricing-fixture.ts new file mode 100644 index 0000000..21d8773 --- /dev/null +++ b/.agents/skills/refresh-model-catalog/scripts/sync-pricing-fixture.ts @@ -0,0 +1,43 @@ +// Regenerates tests/fixtures/commandcode-pricing.json from src/pricing.ts so the +// snapshot always matches MODEL_COSTS. Run from the repository root: +// npx tsx .agents/skills/refresh-model-catalog/scripts/sync-pricing-fixture.ts +import { writeFile } from "node:fs/promises" + +import { format, resolveConfig } from "prettier" + +import { MODEL_COSTS, PRICING_LAST_VERIFIED, PRICING_SOURCE_URL } from "../../../../src/pricing.ts" + +const FIXTURE_PATH = new URL("../../../../tests/fixtures/commandcode-pricing.json", import.meta.url) + +const costs: Record = {} +const tiers: Record = {} +for (const [modelId, cost] of Object.entries(MODEL_COSTS)) { + costs[modelId] = [cost.input, cost.output, cost.cacheRead, cost.cacheWrite] + if (cost.tiers) { + tiers[modelId] = cost.tiers.map((tier) => [ + tier.inputTokensAbove, + tier.input, + tier.output, + tier.cacheRead, + tier.cacheWrite, + ]) + } +} + +const fixture = { + verifiedAt: PRICING_LAST_VERIFIED, + source: PRICING_SOURCE_URL, + tierPolicy: + "Use request-wide input tiers; the highest threshold exceeded by input plus cache tokens applies to the full request.", + tiers, + costs, +} +const options = await resolveConfig(new URL("../../../../.prettierrc.json", import.meta.url)) +const contents = await format(JSON.stringify(fixture), { + ...options, + filepath: "commandcode-pricing.json", +}) +await writeFile(FIXTURE_PATH, contents, "utf-8") +console.log( + `Wrote ${Object.keys(costs).length} model prices to tests/fixtures/commandcode-pricing.json`, +) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53a1a73..0cc7cea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - Fix `npm run sync:commandcode-catalog` and `npm run check:commandcode-catalog` on Windows by spawning npm through the shell. +- Add a `refresh-model-catalog` agent skill with cross-platform helper scripts that snapshot the live model catalog and regenerate the pricing fixture from `MODEL_COSTS`. ## 0.6.0 - 2026-08-25 diff --git a/tsconfig.json b/tsconfig.json index 2cf50c2..aac5ebe 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,5 +9,5 @@ "strict": true, "types": ["node"] }, - "include": [".github/scripts/**/*.ts", "src/**/*.ts", "tests/**/*.ts"] + "include": [".github/scripts/**/*.ts", ".agents/skills/**/*.ts", "src/**/*.ts", "tests/**/*.ts"] }