diff --git a/.github/scripts/check-commandcode-model-metadata.ts b/.github/scripts/check-commandcode-model-metadata.ts new file mode 100644 index 0000000..7182cc5 --- /dev/null +++ b/.github/scripts/check-commandcode-model-metadata.ts @@ -0,0 +1,327 @@ +import { execFile } from "node:child_process" +import { appendFile, mkdtemp, readFile, rm } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join, resolve } from "node:path" +import { pathToFileURL } from "node:url" +import { promisify } from "node:util" + +import { COMMAND_CODE_CLI_VERSION } from "../../src/core.ts" +import { MODEL_EFFORTS, MODEL_INPUT_MODALITIES } from "../../src/models.ts" + +const execFileAsync = promisify(execFile) +const MODELS_REFERENCE_PATH = "dist/bundled/command-code-knowledge/reference/models.md" +const CLI_BUNDLE_PATH = "dist/cli.mjs" +const TEXT_ONLY_MARKER = ',__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' +const VALID_EFFORTS = new Set(["low", "medium", "high", "xhigh", "max"]) + +export interface CommandCodeModelMetadata { + imageModelIds: readonly string[] + reasoningEfforts: Readonly> +} + +export interface ModelMetadataDiff { + addedImageModelIds: readonly string[] + removedImageModelIds: readonly string[] + addedReasoningModelIds: readonly string[] + removedReasoningModelIds: readonly string[] + changedReasoningModelIds: readonly string[] +} + +interface PackedPackage { + filename: string +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function isStringArray(value: unknown): value is string[] { + return Array.isArray(value) && value.every((entry) => typeof entry === "string") +} + +function sorted(values: Iterable): string[] { + return [...values].sort((left, right) => left.localeCompare(right)) +} + +function parsePackedPackage(value: unknown): PackedPackage { + if (!Array.isArray(value) || value.length !== 1 || !isRecord(value[0])) { + throw new Error("Expected npm pack to return one package") + } + + const filename = value[0].filename + if (typeof filename !== "string" || filename.length === 0) { + throw new Error("Expected npm pack to return a tarball filename") + } + + return { filename } +} + +export function parsePackageVersion(value: unknown): string { + if (typeof value !== "string" || !/^\d+\.\d+\.\d+(?:[-+].+)?$/.test(value)) { + throw new Error("Expected npm view to return one semantic version") + } + return value +} + +export function parseModelsReference(markdown: string): { + modelIds: readonly string[] + reasoningEfforts: Readonly> +} { + const modelIds = new Set() + const reasoningEfforts: Record = {} + + for (const line of markdown.split("\n")) { + const match = /^\| `([^`]+)` \| [^|]* \| [^|]* \| ([^|]*) \|/.exec(line) + if (!match) continue + + const modelId = match[1] + const effortsColumn = match[2]?.trim() + if (!modelId || !effortsColumn) throw new Error(`Could not parse model row: ${line}`) + if (modelIds.has(modelId)) throw new Error(`Duplicate model id in reference: ${modelId}`) + modelIds.add(modelId) + + if (effortsColumn === "—") continue + + const efforts = effortsColumn.split(",").map((effort) => effort.trim()) + if (efforts.length === 0 || efforts.some((effort) => !VALID_EFFORTS.has(effort))) { + throw new Error(`Unexpected reasoning efforts for ${modelId}: ${effortsColumn}`) + } + reasoningEfforts[modelId] = efforts + } + + if (modelIds.size === 0) throw new Error("No model rows found in Command Code reference") + + return { + modelIds: sorted(modelIds), + reasoningEfforts: Object.fromEntries( + Object.entries(reasoningEfforts).sort(([left], [right]) => left.localeCompare(right)), + ), + } +} + +export function parseKnownTextOnlyModelIds(bundle: string): readonly string[] { + const markerIndex = bundle.indexOf(TEXT_ONLY_MARKER) + if (markerIndex < 0) { + throw new Error("Could not find Command Code's isKnownTextOnlyModel catalog") + } + + const setStart = bundle.lastIndexOf("new Set([", markerIndex) + if (setStart < 0) throw new Error("Could not find the text-only model set") + + const arrayStart = setStart + "new Set(".length + const arrayEnd = markerIndex - 1 + const literal = bundle.slice(arrayStart, arrayEnd) + const parsed: unknown = JSON.parse(literal) + if (!isStringArray(parsed)) throw new Error("Expected the text-only model catalog to be strings") + + return sorted(new Set(parsed)) +} + +export function commandCodeModelMetadataFromContents( + modelsReference: string, + cliBundle: string, +): CommandCodeModelMetadata { + const reference = parseModelsReference(modelsReference) + const textOnlyModelIds = new Set(parseKnownTextOnlyModelIds(cliBundle)) + + return { + imageModelIds: reference.modelIds.filter((modelId) => !textOnlyModelIds.has(modelId)), + reasoningEfforts: reference.reasoningEfforts, + } +} + +export function currentModelMetadata(): CommandCodeModelMetadata { + return { + imageModelIds: sorted(Object.keys(MODEL_INPUT_MODALITIES)), + reasoningEfforts: Object.fromEntries( + Object.entries(MODEL_EFFORTS) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([modelId, efforts]) => [modelId, [...efforts]]), + ), + } +} + +export function diffModelMetadata( + current: CommandCodeModelMetadata, + upstream: CommandCodeModelMetadata, +): ModelMetadataDiff { + const currentImages = new Set(current.imageModelIds) + const upstreamImages = new Set(upstream.imageModelIds) + const currentReasoningIds = Object.keys(current.reasoningEfforts) + const upstreamReasoningIds = Object.keys(upstream.reasoningEfforts) + const currentReasoningSet = new Set(currentReasoningIds) + const upstreamReasoningSet = new Set(upstreamReasoningIds) + + return { + addedImageModelIds: sorted( + upstream.imageModelIds.filter((modelId) => !currentImages.has(modelId)), + ), + removedImageModelIds: sorted( + current.imageModelIds.filter((modelId) => !upstreamImages.has(modelId)), + ), + addedReasoningModelIds: sorted( + upstreamReasoningIds.filter((modelId) => !currentReasoningSet.has(modelId)), + ), + removedReasoningModelIds: sorted( + currentReasoningIds.filter((modelId) => !upstreamReasoningSet.has(modelId)), + ), + changedReasoningModelIds: sorted( + upstreamReasoningIds.filter( + (modelId) => + currentReasoningSet.has(modelId) && + JSON.stringify(current.reasoningEfforts[modelId]) !== + JSON.stringify(upstream.reasoningEfforts[modelId]), + ), + ), + } +} + +export function hasModelMetadataDiff(diff: ModelMetadataDiff): boolean { + return Object.values(diff).some((modelIds) => modelIds.length > 0) +} + +function formatList(modelIds: readonly string[]): string { + return modelIds.length > 0 ? modelIds.map((modelId) => `\`${modelId}\``).join(", ") : "None" +} + +function formatReasoningChanges( + modelIds: readonly string[], + current: CommandCodeModelMetadata, + upstream: CommandCodeModelMetadata, +): string { + if (modelIds.length === 0) return "None" + return modelIds + .map( + (modelId) => + `\`${modelId}\`: \`${(current.reasoningEfforts[modelId] ?? []).join(", ")}\` → \`${( + upstream.reasoningEfforts[modelId] ?? [] + ).join(", ")}\``, + ) + .join("
") +} + +function metadataReport( + packageVersion: string, + current: CommandCodeModelMetadata, + upstream: CommandCodeModelMetadata, + diff: ModelMetadataDiff, +): string { + const status = hasModelMetadataDiff(diff) ? "❌ Drift detected" : "✅ Metadata is current" + return [ + "## Command Code static model metadata", + "", + `**${status}**`, + "", + `- Repository snapshot: \`command-code@${COMMAND_CODE_CLI_VERSION}\``, + `- Inspected package: \`command-code@${packageVersion}\``, + `- Image-capable models: ${current.imageModelIds.length} repository / ${upstream.imageModelIds.length} upstream`, + `- Reasoning models: ${Object.keys(current.reasoningEfforts).length} repository / ${Object.keys(upstream.reasoningEfforts).length} upstream`, + "", + "| Change | Models |", + "| --- | --- |", + `| New image support | ${formatList(diff.addedImageModelIds)} |`, + `| Removed image support | ${formatList(diff.removedImageModelIds)} |`, + `| New reasoning metadata | ${formatList(diff.addedReasoningModelIds)} |`, + `| Removed reasoning metadata | ${formatList(diff.removedReasoningModelIds)} |`, + `| Changed reasoning efforts | ${formatReasoningChanges(diff.changedReasoningModelIds, current, upstream)} |`, + "", + ].join("\n") +} + +async function resolvePackageSpec( + packageSpec: string, + directory: string, + npmCacheDirectory: string, +): Promise { + if (packageSpec !== "command-code@latest") return packageSpec + + const { stdout } = await execFileAsync( + "npm", + ["view", packageSpec, "version", "--json", "--prefer-online", "--cache", npmCacheDirectory], + { + cwd: directory, + encoding: "utf-8", + }, + ) + return `command-code@${parsePackageVersion(JSON.parse(stdout) as unknown)}` +} + +async function inspectPackedPackage(packageSpec: string): Promise<{ + packageVersion: string + metadata: CommandCodeModelMetadata +}> { + const directory = await mkdtemp(join(tmpdir(), "pi-commandcode-model-check-")) + const npmCacheDirectory = join(directory, "npm-cache") + + try { + const resolvedPackageSpec = await resolvePackageSpec(packageSpec, directory, npmCacheDirectory) + const { stdout } = await execFileAsync( + "npm", + ["pack", resolvedPackageSpec, "--json", "--prefer-online", "--cache", npmCacheDirectory], + { + cwd: directory, + encoding: "utf-8", + maxBuffer: 10 * 1024 * 1024, + }, + ) + const packed = parsePackedPackage(JSON.parse(stdout) as unknown) + await execFileAsync("tar", ["-xzf", packed.filename], { cwd: directory }) + + const packageDirectory = join(directory, "package") + const packageJsonContents = await readFile(join(packageDirectory, "package.json"), "utf-8") + const packageJson: unknown = JSON.parse(packageJsonContents) + if (!isRecord(packageJson) || typeof packageJson.version !== "string") { + throw new Error("Expected command-code package.json to contain a version") + } + + const [modelsReference, cliBundle] = await Promise.all([ + readFile(join(packageDirectory, MODELS_REFERENCE_PATH), "utf-8"), + readFile(join(packageDirectory, CLI_BUNDLE_PATH), "utf-8"), + ]) + + return { + packageVersion: packageJson.version, + metadata: commandCodeModelMetadataFromContents(modelsReference, cliBundle), + } + } finally { + await rm(directory, { recursive: true, force: true }) + } +} + +async function main(): Promise { + const packageSpec = process.argv[2] ?? "command-code@latest" + const current = currentModelMetadata() + const upstreamPackage = await inspectPackedPackage(packageSpec) + const diff = diffModelMetadata(current, upstreamPackage.metadata) + const report = metadataReport( + upstreamPackage.packageVersion, + current, + upstreamPackage.metadata, + diff, + ) + + console.log(report) + + const summaryPath = process.env.GITHUB_STEP_SUMMARY + if (summaryPath) await appendFile(summaryPath, report, "utf-8") + + if (hasModelMetadataDiff(diff)) { + throw new Error( + `Static model metadata differs from command-code@${upstreamPackage.packageVersion}. Update src/models.ts and the snapshot version.`, + ) + } +} + +function isMainModule(): boolean { + const entrypoint = process.argv[1] + return entrypoint !== undefined && pathToFileURL(resolve(entrypoint)).href === import.meta.url +} + +if (isMainModule()) { + try { + await main() + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)) + process.exitCode = 1 + } +} diff --git a/.github/workflows/model-metadata.yml b/.github/workflows/model-metadata.yml new file mode 100644 index 0000000..c933905 --- /dev/null +++ b/.github/workflows/model-metadata.yml @@ -0,0 +1,28 @@ +name: Command Code model metadata + +on: + schedule: + - cron: "17 6 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: commandcode-model-metadata + cancel-in-progress: true + +jobs: + check: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + registry-url: https://registry.npmjs.org + - run: npm ci + - name: Compare static metadata with the latest Command Code CLI + run: npm run check:model-metadata diff --git a/CHANGELOG.md b/CHANGELOG.md index 99640d1..c4e2930 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Add a daily GitHub Actions check that compares static image and reasoning metadata with the latest published Command Code CLI catalog. - Refresh static model capabilities from `command-code@1.32.2`, including new image and reasoning metadata. - Add `/commandcode-quota` with live credits, plan, usage totals, and rolling-limit diagnostics from Command Code's alpha usage endpoints. - Add `zai-org/GLM-5.3` with its verified reasoning efforts and display pricing. diff --git a/README.md b/README.md index 6caf1d2..7657981 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ The following environment variables are intended for tests, local mocks, and com ## Image input -The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed. +The provider advertises image input only for models marked with the `image` input modality in the official Command Code CLI model catalog. The capability snapshot currently follows `command-code@1.32.2`; unknown models default to text-only until their upstream metadata is reviewed. A daily GitHub Actions check compares the static image and reasoning metadata with the latest published CLI package and reports any drift. For vision-capable models, Pi's native provider adapters forward image blocks from user messages and tool results using the documented OpenAI or Anthropic message schema. Unknown and text-only models remain marked text-only in Pi. diff --git a/package.json b/package.json index 6e07011..8d14353 100644 --- a/package.json +++ b/package.json @@ -29,16 +29,17 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", "pi:isolated": "node scripts/pi-isolated.mjs", "pi:authenticated": "node scripts/pi-authenticated.mjs", + "check:model-metadata": "tsx .github/scripts/check-commandcode-model-metadata.ts command-code@latest", "test:quota": "tsx tests/test-quota.ts && tsx tests/test-quota-command.ts", - "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", + "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", "test:api-key": "tsx tests/test-api-key.ts", - "test:models": "tsx tests/test-models.ts", + "test:models": "tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts", "test:runtime": "tsx tests/test-runtime.ts", "test:pricing": "tsx tests/test-pricing.ts", "test:oauth": "tsx tests/test-oauth.ts", diff --git a/tests/test-model-metadata-check.ts b/tests/test-model-metadata-check.ts new file mode 100644 index 0000000..145b298 --- /dev/null +++ b/tests/test-model-metadata-check.ts @@ -0,0 +1,88 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +import { + commandCodeModelMetadataFromContents, + diffModelMetadata, + hasModelMetadataDiff, + parseKnownTextOnlyModelIds, + parseModelsReference, + parsePackageVersion, + type CommandCodeModelMetadata, +} from "../.github/scripts/check-commandcode-model-metadata.ts" + +const MODELS_REFERENCE = ` +| Id (use EXACTLY this) | Name | Context | Efforts | $/1M in/out · cache read | Min plan | Best for | +|---|---|---|---|---|---|---| +| \`vision-model\` | Vision | 1M | low, high | $1/$2 | Go | images | +| \`text-model\` | Text | 200K | — | $1/$2 | Go | text | +` + +const CLI_BUNDLE = + 'const catalog=new Set(["text-model"]),__name(isKnownTextOnlyModel,"isKnownTextOnlyModel")' + +describe("Command Code model metadata checker", () => { + it("parses model ids and reasoning efforts from the generated reference", () => { + assert.deepEqual(parseModelsReference(MODELS_REFERENCE), { + modelIds: ["text-model", "vision-model"], + reasoningEfforts: { "vision-model": ["low", "high"] }, + }) + }) + + it("extracts the text-only set from the bundled CLI catalog", () => { + assert.deepEqual(parseKnownTextOnlyModelIds(CLI_BUNDLE), ["text-model"]) + }) + + it("accepts one exact npm registry version and rejects stale-looking output shapes", () => { + assert.equal(parsePackageVersion("1.32.2"), "1.32.2") + assert.equal(parsePackageVersion("2.0.0-beta.1"), "2.0.0-beta.1") + assert.throws(() => parsePackageVersion(["1.32.1", "1.32.2"]), /one semantic version/) + assert.throws(() => parsePackageVersion("latest"), /one semantic version/) + }) + + it("derives image support by excluding known text-only models", () => { + assert.deepEqual(commandCodeModelMetadataFromContents(MODELS_REFERENCE, CLI_BUNDLE), { + imageModelIds: ["vision-model"], + reasoningEfforts: { "vision-model": ["low", "high"] }, + }) + }) + + it("reports additions, removals, and changed reasoning efforts", () => { + const current: CommandCodeModelMetadata = { + imageModelIds: ["removed-image", "stable-image"], + reasoningEfforts: { + "changed-reasoning": ["low"], + "removed-reasoning": ["high"], + "stable-reasoning": ["low", "high"], + }, + } + const upstream: CommandCodeModelMetadata = { + imageModelIds: ["added-image", "stable-image"], + reasoningEfforts: { + "added-reasoning": ["max"], + "changed-reasoning": ["low", "high"], + "stable-reasoning": ["low", "high"], + }, + } + + const diff = diffModelMetadata(current, upstream) + + assert.deepEqual(diff, { + addedImageModelIds: ["added-image"], + removedImageModelIds: ["removed-image"], + addedReasoningModelIds: ["added-reasoning"], + removedReasoningModelIds: ["removed-reasoning"], + changedReasoningModelIds: ["changed-reasoning"], + }) + assert.equal(hasModelMetadataDiff(diff), true) + }) + + it("rejects unexpected upstream structures instead of silently passing", () => { + assert.throws(() => parseModelsReference("# no catalog"), /No model rows/) + assert.throws( + () => parseModelsReference(MODELS_REFERENCE.replace("low, high", "low, turbo")), + /Unexpected reasoning efforts/, + ) + assert.throws(() => parseKnownTextOnlyModelIds("const unrelated = true"), /Could not find/) + }) +}) diff --git a/tsconfig.json b/tsconfig.json index c691796..2cf50c2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,5 +9,5 @@ "strict": true, "types": ["node"] }, - "include": ["src/**/*.ts", "tests/**/*.ts"] + "include": [".github/scripts/**/*.ts", "src/**/*.ts", "tests/**/*.ts"] }