ci(models): check Command Code metadata daily

.github/scripts/check-commandcode-model-metadata.ts

@@ -0,0 +1,327 @@
+import { execFile } from "node:child_process"
+import { appendFile, mkdtemp, readFile, rm } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { join, resolve } from "node:path"
+import { pathToFileURL } from "node:url"
+import { promisify } from "node:util"
+
+import { COMMAND_CODE_CLI_VERSION } from "../../src/core.ts"
+import { MODEL_EFFORTS, MODEL_INPUT_MODALITIES } from "../../src/models.ts"
+
+const execFileAsync = promisify(execFile)
+const MODELS_REFERENCE_PATH = "dist/bundled/command-code-knowledge/reference/models.md"
+const CLI_BUNDLE_PATH = "dist/cli.mjs"
+const TEXT_ONLY_MARKER = ',__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")'
+const VALID_EFFORTS = new Set(["low", "medium", "high", "xhigh", "max"])
+
+export interface CommandCodeModelMetadata {
+  imageModelIds: readonly string[]
+  reasoningEfforts: Readonly<Record<string, readonly string[]>>
+}
+
+export interface ModelMetadataDiff {
+  addedImageModelIds: readonly string[]
+  removedImageModelIds: readonly string[]
+  addedReasoningModelIds: readonly string[]
+  removedReasoningModelIds: readonly string[]
+  changedReasoningModelIds: readonly string[]
+}
+
+interface PackedPackage {
+  filename: string
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+}
+
+function isStringArray(value: unknown): value is string[] {
+  return Array.isArray(value) && value.every((entry) => typeof entry === "string")
+}
+
+function sorted(values: Iterable<string>): string[] {
+  return [...values].sort((left, right) => left.localeCompare(right))
+}
+
+function parsePackedPackage(value: unknown): PackedPackage {
+  if (!Array.isArray(value) || value.length !== 1 || !isRecord(value[0])) {
+    throw new Error("Expected npm pack to return one package")
+  }
+
+  const filename = value[0].filename
+  if (typeof filename !== "string" || filename.length === 0) {
+    throw new Error("Expected npm pack to return a tarball filename")
+  }
+
+  return { filename }
+}
+
+export function parsePackageVersion(value: unknown): string {
+  if (typeof value !== "string" || !/^\d+\.\d+\.\d+(?:[-+].+)?$/.test(value)) {
+    throw new Error("Expected npm view to return one semantic version")
+  }
+  return value
+}
+
+export function parseModelsReference(markdown: string): {
+  modelIds: readonly string[]
+  reasoningEfforts: Readonly<Record<string, readonly string[]>>
+} {
+  const modelIds = new Set<string>()
+  const reasoningEfforts: Record<string, readonly string[]> = {}
+
+  for (const line of markdown.split("\n")) {
+    const match = /^\| `([^`]+)` \| [^|]* \| [^|]* \| ([^|]*) \|/.exec(line)
+    if (!match) continue
+
+    const modelId = match[1]
+    const effortsColumn = match[2]?.trim()
+    if (!modelId || !effortsColumn) throw new Error(`Could not parse model row: ${line}`)
+    if (modelIds.has(modelId)) throw new Error(`Duplicate model id in reference: ${modelId}`)
+    modelIds.add(modelId)
+
+    if (effortsColumn === "—") continue
+
+    const efforts = effortsColumn.split(",").map((effort) => effort.trim())
+    if (efforts.length === 0 || efforts.some((effort) => !VALID_EFFORTS.has(effort))) {
+      throw new Error(`Unexpected reasoning efforts for ${modelId}: ${effortsColumn}`)
+    }
+    reasoningEfforts[modelId] = efforts
+  }
+
+  if (modelIds.size === 0) throw new Error("No model rows found in Command Code reference")
+
+  return {
+    modelIds: sorted(modelIds),
+    reasoningEfforts: Object.fromEntries(
+      Object.entries(reasoningEfforts).sort(([left], [right]) => left.localeCompare(right)),
+    ),
+  }
+}
+
+export function parseKnownTextOnlyModelIds(bundle: string): readonly string[] {
+  const markerIndex = bundle.indexOf(TEXT_ONLY_MARKER)
+  if (markerIndex < 0) {
+    throw new Error("Could not find Command Code's isKnownTextOnlyModel catalog")
+  }
+
+  const setStart = bundle.lastIndexOf("new Set([", markerIndex)
+  if (setStart < 0) throw new Error("Could not find the text-only model set")
+
+  const arrayStart = setStart + "new Set(".length
+  const arrayEnd = markerIndex - 1
+  const literal = bundle.slice(arrayStart, arrayEnd)
+  const parsed: unknown = JSON.parse(literal)
+  if (!isStringArray(parsed)) throw new Error("Expected the text-only model catalog to be strings")
+
+  return sorted(new Set(parsed))
+}
+
+export function commandCodeModelMetadataFromContents(
+  modelsReference: string,
+  cliBundle: string,
+): CommandCodeModelMetadata {
+  const reference = parseModelsReference(modelsReference)
+  const textOnlyModelIds = new Set(parseKnownTextOnlyModelIds(cliBundle))
+
+  return {
+    imageModelIds: reference.modelIds.filter((modelId) => !textOnlyModelIds.has(modelId)),
+    reasoningEfforts: reference.reasoningEfforts,
+  }
+}
+
+export function currentModelMetadata(): CommandCodeModelMetadata {
+  return {
+    imageModelIds: sorted(Object.keys(MODEL_INPUT_MODALITIES)),
+    reasoningEfforts: Object.fromEntries(
+      Object.entries(MODEL_EFFORTS)
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([modelId, efforts]) => [modelId, [...efforts]]),
+    ),
+  }
+}
+
+export function diffModelMetadata(
+  current: CommandCodeModelMetadata,
+  upstream: CommandCodeModelMetadata,
+): ModelMetadataDiff {
+  const currentImages = new Set(current.imageModelIds)
+  const upstreamImages = new Set(upstream.imageModelIds)
+  const currentReasoningIds = Object.keys(current.reasoningEfforts)
+  const upstreamReasoningIds = Object.keys(upstream.reasoningEfforts)
+  const currentReasoningSet = new Set(currentReasoningIds)
+  const upstreamReasoningSet = new Set(upstreamReasoningIds)
+
+  return {
+    addedImageModelIds: sorted(
+      upstream.imageModelIds.filter((modelId) => !currentImages.has(modelId)),
+    ),
+    removedImageModelIds: sorted(
+      current.imageModelIds.filter((modelId) => !upstreamImages.has(modelId)),
+    ),
+    addedReasoningModelIds: sorted(
+      upstreamReasoningIds.filter((modelId) => !currentReasoningSet.has(modelId)),
+    ),
+    removedReasoningModelIds: sorted(
+      currentReasoningIds.filter((modelId) => !upstreamReasoningSet.has(modelId)),
+    ),
+    changedReasoningModelIds: sorted(
+      upstreamReasoningIds.filter(
+        (modelId) =>
+          currentReasoningSet.has(modelId) &&
+          JSON.stringify(current.reasoningEfforts[modelId]) !==
+            JSON.stringify(upstream.reasoningEfforts[modelId]),
+      ),
+    ),
+  }
+}
+
+export function hasModelMetadataDiff(diff: ModelMetadataDiff): boolean {
+  return Object.values(diff).some((modelIds) => modelIds.length > 0)
+}
+
+function formatList(modelIds: readonly string[]): string {
+  return modelIds.length > 0 ? modelIds.map((modelId) => `\`${modelId}\``).join(", ") : "None"
+}
+
+function formatReasoningChanges(
+  modelIds: readonly string[],
+  current: CommandCodeModelMetadata,
+  upstream: CommandCodeModelMetadata,
+): string {
+  if (modelIds.length === 0) return "None"
+  return modelIds
+    .map(
+      (modelId) =>
+        `\`${modelId}\`: \`${(current.reasoningEfforts[modelId] ?? []).join(", ")}\` → \`${(
+          upstream.reasoningEfforts[modelId] ?? []
+        ).join(", ")}\``,
+    )
+    .join("<br>")
+}
+
+function metadataReport(
+  packageVersion: string,
+  current: CommandCodeModelMetadata,
+  upstream: CommandCodeModelMetadata,
+  diff: ModelMetadataDiff,
+): string {
+  const status = hasModelMetadataDiff(diff) ? "❌ Drift detected" : "✅ Metadata is current"
+  return [
+    "## Command Code static model metadata",
+    "",
+    `**${status}**`,
+    "",
+    `- Repository snapshot: \`command-code@${COMMAND_CODE_CLI_VERSION}\``,
+    `- Inspected package: \`command-code@${packageVersion}\``,
+    `- Image-capable models: ${current.imageModelIds.length} repository / ${upstream.imageModelIds.length} upstream`,
+    `- Reasoning models: ${Object.keys(current.reasoningEfforts).length} repository / ${Object.keys(upstream.reasoningEfforts).length} upstream`,
+    "",
+    "| Change | Models |",
+    "| --- | --- |",
+    `| New image support | ${formatList(diff.addedImageModelIds)} |`,
+    `| Removed image support | ${formatList(diff.removedImageModelIds)} |`,
+    `| New reasoning metadata | ${formatList(diff.addedReasoningModelIds)} |`,
+    `| Removed reasoning metadata | ${formatList(diff.removedReasoningModelIds)} |`,
+    `| Changed reasoning efforts | ${formatReasoningChanges(diff.changedReasoningModelIds, current, upstream)} |`,
+    "",
+  ].join("\n")
+}
+
+async function resolvePackageSpec(
+  packageSpec: string,
+  directory: string,
+  npmCacheDirectory: string,
+): Promise<string> {
+  if (packageSpec !== "command-code@latest") return packageSpec
+
+  const { stdout } = await execFileAsync(
+    "npm",
+    ["view", packageSpec, "version", "--json", "--prefer-online", "--cache", npmCacheDirectory],
+    {
+      cwd: directory,
+      encoding: "utf-8",
+    },
+  )
+  return `command-code@${parsePackageVersion(JSON.parse(stdout) as unknown)}`
+}
+
+async function inspectPackedPackage(packageSpec: string): Promise<{
+  packageVersion: string
+  metadata: CommandCodeModelMetadata
+}> {
+  const directory = await mkdtemp(join(tmpdir(), "pi-commandcode-model-check-"))
+  const npmCacheDirectory = join(directory, "npm-cache")
+
+  try {
+    const resolvedPackageSpec = await resolvePackageSpec(packageSpec, directory, npmCacheDirectory)
+    const { stdout } = await execFileAsync(
+      "npm",
+      ["pack", resolvedPackageSpec, "--json", "--prefer-online", "--cache", npmCacheDirectory],
+      {
+        cwd: directory,
+        encoding: "utf-8",
+        maxBuffer: 10 * 1024 * 1024,
+      },
+    )
+    const packed = parsePackedPackage(JSON.parse(stdout) as unknown)
+    await execFileAsync("tar", ["-xzf", packed.filename], { cwd: directory })
+
+    const packageDirectory = join(directory, "package")
+    const packageJsonContents = await readFile(join(packageDirectory, "package.json"), "utf-8")
+    const packageJson: unknown = JSON.parse(packageJsonContents)
+    if (!isRecord(packageJson) || typeof packageJson.version !== "string") {
+      throw new Error("Expected command-code package.json to contain a version")
+    }
+
+    const [modelsReference, cliBundle] = await Promise.all([
+      readFile(join(packageDirectory, MODELS_REFERENCE_PATH), "utf-8"),
+      readFile(join(packageDirectory, CLI_BUNDLE_PATH), "utf-8"),
+    ])
+
+    return {
+      packageVersion: packageJson.version,
+      metadata: commandCodeModelMetadataFromContents(modelsReference, cliBundle),
+    }
+  } finally {
+    await rm(directory, { recursive: true, force: true })
+  }
+}
+
+async function main(): Promise<void> {
+  const packageSpec = process.argv[2] ?? "command-code@latest"
+  const current = currentModelMetadata()
+  const upstreamPackage = await inspectPackedPackage(packageSpec)
+  const diff = diffModelMetadata(current, upstreamPackage.metadata)
+  const report = metadataReport(
+    upstreamPackage.packageVersion,
+    current,
+    upstreamPackage.metadata,
+    diff,
+  )
+
+  console.log(report)
+
+  const summaryPath = process.env.GITHUB_STEP_SUMMARY
+  if (summaryPath) await appendFile(summaryPath, report, "utf-8")
+
+  if (hasModelMetadataDiff(diff)) {
+    throw new Error(
+      `Static model metadata differs from command-code@${upstreamPackage.packageVersion}. Update src/models.ts and the snapshot version.`,
+    )
+  }
+}
+
+function isMainModule(): boolean {
+  const entrypoint = process.argv[1]
+  return entrypoint !== undefined && pathToFileURL(resolve(entrypoint)).href === import.meta.url
+}
+
+if (isMainModule()) {
+  try {
+    await main()
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : String(error))
+    process.exitCode = 1
+  }
+}

.github/workflows/model-metadata.yml

@@ -0,0 +1,28 @@
+name: Command Code model metadata
+
+on:
+  schedule:
+    - cron: "17 6 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: commandcode-model-metadata
+  cancel-in-progress: true
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          registry-url: https://registry.npmjs.org
+      - run: npm ci
+      - name: Compare static metadata with the latest Command Code CLI
+        run: npm run check:model-metadata

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Add a daily GitHub Actions check that compares static image and reasoning metadata with the latest published Command Code CLI catalog.
 - Refresh static model capabilities from `command-code@1.32.2`, including new image and reasoning metadata.
 - Add `/commandcode-quota` with live credits, plan, usage totals, and rolling-limit diagnostics from Command Code's alpha usage endpoints.
 - Add `zai-org/GLM-5.3` with its verified reasoning efforts and display pricing.

README.md

@@ -144,7 +144,7 @@ The following environment variables are intended for tests, local mocks, and com
 
 ## Image input
 
-The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed.
+The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed. A daily GitHub Actions check compares the static image and reasoning metadata with the latest published CLI package and reports any drift.
 
 For vision-capable models, Pi's native provider adapters forward image blocks from user messages and tool results using the documented OpenAI or Anthropic message schema. Unknown and text-only models remain marked text-only in Pi.
 

package.json

@@ -29,16 +29,17 @@
     "LICENSE"
   ],
   "scripts": {
-    "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs",
+    "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs",
     "typecheck": "tsc --noEmit",
     "format:check": "prettier --check '**/*.{ts,mjs,json,md}'",
     "format": "prettier --write '**/*.{ts,mjs,json,md}'",
     "pi:isolated": "node scripts/pi-isolated.mjs",
     "pi:authenticated": "node scripts/pi-authenticated.mjs",
+    "check:model-metadata": "tsx .github/scripts/check-commandcode-model-metadata.ts command-code@latest",
     "test:quota": "tsx tests/test-quota.ts && tsx tests/test-quota-command.ts",
-    "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts",
+    "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts",
     "test:api-key": "tsx tests/test-api-key.ts",
-    "test:models": "tsx tests/test-models.ts",
+    "test:models": "tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts",
     "test:runtime": "tsx tests/test-runtime.ts",
     "test:pricing": "tsx tests/test-pricing.ts",
     "test:oauth": "tsx tests/test-oauth.ts",

tests/test-model-metadata-check.ts

@@ -0,0 +1,88 @@
+import assert from "node:assert/strict"
+import { describe, it } from "node:test"
+
+import {
+  commandCodeModelMetadataFromContents,
+  diffModelMetadata,
+  hasModelMetadataDiff,
+  parseKnownTextOnlyModelIds,
+  parseModelsReference,
+  parsePackageVersion,
+  type CommandCodeModelMetadata,
+} from "../.github/scripts/check-commandcode-model-metadata.ts"
+
+const MODELS_REFERENCE = `
+| Id (use EXACTLY this) | Name | Context | Efforts | $/1M in/out · cache read | Min plan | Best for |
+|---|---|---|---|---|---|---|
+| \`vision-model\` | Vision | 1M | low, high | $1/$2 | Go | images |
+| \`text-model\` | Text | 200K | — | $1/$2 | Go | text |
+`
+
+const CLI_BUNDLE =
+  'const catalog=new Set(["text-model"]),__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")'
+
+describe("Command Code model metadata checker", () => {
+  it("parses model ids and reasoning efforts from the generated reference", () => {
+    assert.deepEqual(parseModelsReference(MODELS_REFERENCE), {
+      modelIds: ["text-model", "vision-model"],
+      reasoningEfforts: { "vision-model": ["low", "high"] },
+    })
+  })
+
+  it("extracts the text-only set from the bundled CLI catalog", () => {
+    assert.deepEqual(parseKnownTextOnlyModelIds(CLI_BUNDLE), ["text-model"])
+  })
+
+  it("accepts one exact npm registry version and rejects stale-looking output shapes", () => {
+    assert.equal(parsePackageVersion("1.32.2"), "1.32.2")
+    assert.equal(parsePackageVersion("2.0.0-beta.1"), "2.0.0-beta.1")
+    assert.throws(() => parsePackageVersion(["1.32.1", "1.32.2"]), /one semantic version/)
+    assert.throws(() => parsePackageVersion("latest"), /one semantic version/)
+  })
+
+  it("derives image support by excluding known text-only models", () => {
+    assert.deepEqual(commandCodeModelMetadataFromContents(MODELS_REFERENCE, CLI_BUNDLE), {
+      imageModelIds: ["vision-model"],
+      reasoningEfforts: { "vision-model": ["low", "high"] },
+    })
+  })
+
+  it("reports additions, removals, and changed reasoning efforts", () => {
+    const current: CommandCodeModelMetadata = {
+      imageModelIds: ["removed-image", "stable-image"],
+      reasoningEfforts: {
+        "changed-reasoning": ["low"],
+        "removed-reasoning": ["high"],
+        "stable-reasoning": ["low", "high"],
+      },
+    }
+    const upstream: CommandCodeModelMetadata = {
+      imageModelIds: ["added-image", "stable-image"],
+      reasoningEfforts: {
+        "added-reasoning": ["max"],
+        "changed-reasoning": ["low", "high"],
+        "stable-reasoning": ["low", "high"],
+      },
+    }
+
+    const diff = diffModelMetadata(current, upstream)
+
+    assert.deepEqual(diff, {
+      addedImageModelIds: ["added-image"],
+      removedImageModelIds: ["removed-image"],
+      addedReasoningModelIds: ["added-reasoning"],
+      removedReasoningModelIds: ["removed-reasoning"],
+      changedReasoningModelIds: ["changed-reasoning"],
+    })
+    assert.equal(hasModelMetadataDiff(diff), true)
+  })
+
+  it("rejects unexpected upstream structures instead of silently passing", () => {
+    assert.throws(() => parseModelsReference("# no catalog"), /No model rows/)
+    assert.throws(
+      () => parseModelsReference(MODELS_REFERENCE.replace("low, high", "low, turbo")),
+      /Unexpected reasoning efforts/,
+    )
+    assert.throws(() => parseKnownTextOnlyModelIds("const unrelated = true"), /Could not find/)
+  })
+})

tsconfig.json

@@ -9,5 +9,5 @@
     "strict": true,
     "types": ["node"]
   },
-  "include": ["src/**/*.ts", "tests/**/*.ts"]
+  "include": [".github/scripts/**/*.ts", "src/**/*.ts", "tests/**/*.ts"]
 }