pi-commandcode-provider/tests/test-omp-compat.mjs

#!/usr/bin/env node
/**
 * OMP compatibility smoke test.
 *
 * Uses an isolated HOME/PI_CODING_AGENT_DIR so the test does not depend on or
 * mutate the user's real ~/.omp state. The Command Code API base is pointed at
 * a deterministic local mock server so print mode can exercise the provider
 * without touching the real API.
 */

import assert from "node:assert/strict"
import { spawn } from "node:child_process"
import { accessSync, constants, mkdtempSync, rmSync } from "node:fs"
import { createServer } from "node:http"
import { tmpdir } from "node:os"
import { delimiter, dirname, join, resolve } from "node:path"
import { fileURLToPath } from "node:url"

const __dirname = dirname(fileURLToPath(import.meta.url))
const PROJECT_DIR = resolve(__dirname, "..")
const EXT_PATH = resolve(PROJECT_DIR, "index.ts")
const ADVISORY_EXT_PATH = resolve(PROJECT_DIR, "tests/fixtures/advisory-injector-extension.ts")
const TEST_MODEL = "deepseek/deepseek-v4-flash"
const ADVISORY_XML =
  '<advisory severity="blocker" guidance="weigh, don\'t blindly obey">\nStop and correct the benchmark.\n</advisory>'

function findOmpBinary() {
  if (process.env.OMP_BIN) return process.env.OMP_BIN
  const candidates = (process.env.PATH ?? "").split(delimiter).map((entry) => resolve(entry, "omp"))
  for (const candidate of candidates) {
    try {
      accessSync(candidate, constants.X_OK)
      return candidate
    } catch {
      // Try next PATH entry.
    }
  }
  return undefined
}

const OMP_BIN = findOmpBinary()
if (!OMP_BIN) {
  console.log("[omp-compat] SKIP - omp is not on PATH")
  process.exit(0)
}

const tempHome = mkdtempSync(join(tmpdir(), "omp-cc-home-"))
let requestCount = 0
let modelListRequestCount = 0
let lastRequestBody
let requestBodies = []
let lastRequestHeaders = {}

const server = createServer((req, res) => {
  if (req.method === "GET" && req.url === "/provider/v1/models") {
    modelListRequestCount += 1
    res.writeHead(200, { "Content-Type": "application/json; charset=utf-8" })
    res.end(
      JSON.stringify({
        object: "list",
        data: [
          {
            id: TEST_MODEL,
            object: "model",
            created: 1779824324,
            owned_by: "command-code",
            name: "DeepSeek V4 Flash",
            context_length: 1_000_000,
          },
          {
            id: "Qwen/Qwen3.7-Max",
            object: "model",
            created: 1779824324,
            owned_by: "command-code",
            name: "Qwen 3.7 Max",
            context_length: 1_000_000,
          },
        ],
      }),
    )
    return
  }

  if (req.method !== "POST" || req.url !== "/alpha/generate") {
    res.writeHead(404)
    res.end("Not found")
    return
  }

  requestCount += 1
  lastRequestHeaders = Object.fromEntries(
    Object.entries(req.headers).map(([key, value]) => [
      key,
      Array.isArray(value) ? value.join(", ") : (value ?? ""),
    ]),
  )

  let body = ""
  req.on("data", (chunk) => {
    body += chunk.toString("utf-8")
  })
  req.on("end", () => {
    try {
      lastRequestBody = JSON.parse(body)
      requestBodies.push(lastRequestBody)
    } catch {
      lastRequestBody = undefined
    }

    res.writeHead(200, {
      "Content-Type": "text/plain; charset=utf-8",
      "Transfer-Encoding": "chunked",
    })
    res.write(`${JSON.stringify({ type: "text-delta", text: "mock-omp-ok" })}\n`)
    res.write(
      `${JSON.stringify({ type: "finish", finishReason: "stop", totalUsage: { inputTokens: 1, outputTokens: 1 } })}\n`,
    )
    res.end()
  })
})

await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve))
const address = server.address()
const port = typeof address === "object" && address ? address.port : 0
const apiBase = `http://127.0.0.1:${port}`

function runOmp(args, timeoutMs = 30_000) {
  return new Promise((resolve) => {
    const child = spawn(OMP_BIN, args, {
      cwd: PROJECT_DIR,
      env: {
        ...process.env,
        HOME: tempHome,
        USERPROFILE: tempHome,
        PI_CODING_AGENT_DIR: join(tempHome, ".omp", "agent"),
        COMMANDCODE_API_KEY: "mock-key",
        COMMANDCODE_API_BASE: apiBase,
        COMMANDCODE_MODELS_URL: `${apiBase}/provider/v1/models`,
      },
      stdio: ["ignore", "pipe", "pipe"],
    })
    let stdout = ""
    let stderr = ""
    const timer = setTimeout(() => {
      child.kill()
      resolve({
        code: -1,
        stdout,
        stderr: `${stderr}\nTIMEOUT after ${timeoutMs}ms`,
      })
    }, timeoutMs)
    child.stdout.on("data", (chunk) => {
      stdout += chunk.toString("utf-8")
    })
    child.stderr.on("data", (chunk) => {
      stderr += chunk.toString("utf-8")
    })
    child.on("close", (code) => {
      clearTimeout(timer)
      resolve({ code, stdout, stderr })
    })
  })
}

try {
  console.log("[omp-compat] list models through real extension")
  modelListRequestCount = 0
  const list = await runOmp(["models", "--json", "-e", EXT_PATH, "--no-extensions"])
  if (list.code !== 0 && /unknown|unrecognized/i.test(list.stderr + list.stdout)) {
    console.log("[omp-compat] SKIP models phase - omp models subcommand unavailable")
  } else {
    assert.equal(list.code, 0, list.stderr)
    let listed = null
    try {
      listed = JSON.parse(list.stdout)
    } catch {
      listed = null
    }
    const models = Array.isArray(listed?.models) ? listed.models : []
    assert.ok(
      models.some((model) => model.provider === "commandcode"),
      "commandcode provider should be listed",
    )
    assert.ok(
      models.some((model) => model.id === TEST_MODEL),
      "mock catalog model should be listed",
    )
    assert.ok(modelListRequestCount >= 1)
    assert.doesNotMatch(list.stdout + list.stderr, /Failed to load extension/)
  }

  console.log("[omp-compat] print mode through real extension and mock API")
  requestCount = 0
  requestBodies = []
  const print = await runOmp(
    ["-e", EXT_PATH, "-p", "say mock token", "--model", `commandcode/${TEST_MODEL}`],
    30_000,
  )
  assert.equal(print.code, 0, print.stderr)
  assert.match(print.stdout, /mock-omp-ok/)
  assert.equal(requestCount, 1)
  assert.equal(
    lastRequestHeaders.authorization,
    "Bearer mock-key",
    "should send the resolved env-var value, not the literal var name",
  )
  assert.equal(lastRequestBody?.params?.model, TEST_MODEL)
  assert.equal(typeof lastRequestBody?.params?.system, "string")
  assert.doesNotThrow(() =>
    accessSync(join(tempHome, ".omp", "agent", "commandcode-models.json"), constants.R_OK),
  )

  console.log("[omp-compat] developer advisory reaches the provider request body")
  requestCount = 0
  requestBodies = []
  const advisoryRun = await runOmp(
    [
      "-e",
      EXT_PATH,
      "-e",
      ADVISORY_EXT_PATH,
      "-p",
      "say mock token",
      "--model",
      `commandcode/${TEST_MODEL}`,
      "--no-tools",
      "--no-title",
    ],
    30_000,
  )
  assert.equal(advisoryRun.code, 0, advisoryRun.stderr)
  assert.match(advisoryRun.stdout, /mock-omp-ok/)

  const promptBodies = requestBodies.filter((body) =>
    JSON.stringify(body?.params?.messages ?? []).includes("say mock token"),
  )
  assert.ok(promptBodies.length >= 1, "expected at least one generate request with the prompt")

  for (const body of promptBodies) {
    const messages = body?.params?.messages ?? []
    const advisoryMessages = messages.filter((message) =>
      JSON.stringify(message).includes("Stop and correct the benchmark."),
    )
    assert.equal(advisoryMessages.length, 1, "the advisory should survive conversion exactly once")
    const advisoryMessage = advisoryMessages[0]
    assert.equal(advisoryMessage.role, "user")
    const advisoryText =
      typeof advisoryMessage.content === "string"
        ? advisoryMessage.content
        : (advisoryMessage.content ?? [])
            .map((part) => (part?.type === "text" ? part.text : ""))
            .join("\n")
    assert.equal(advisoryText, ADVISORY_XML, "advisory content must arrive verbatim")

    const advisoryIndex = messages.indexOf(advisoryMessage)
    const promptIndex = messages.findIndex((message) =>
      JSON.stringify(message).includes("say mock token"),
    )
    assert.ok(
      advisoryIndex < promptIndex,
      "advisory must keep its chronological position relative to the prompt",
    )
    assert.doesNotMatch(
      String(body?.params?.system ?? ""),
      /Stop and correct the benchmark|<advisory/,
      "advisory must not be hoisted into the system prompt",
    )
  }

  console.log("[omp-compat] PASS")
} finally {
  await new Promise((resolve) => server.close(resolve))
  rmSync(tempHome, { recursive: true, force: true })
}