From 1f312f8d61514270066263446efeeaaa369b8261 Mon Sep 17 00:00:00 2001 From: warc0s Date: Mon, 17 Aug 2026 09:04:32 +0200 Subject: [PATCH] test(omp): cover developer advisory delivery Add a fixture extension that injects an advisor-style custom message on session start, and assert the advisory XML reaches params.messages verbatim, in chronological position, and is not hoisted into params.system. Port the models phase to `omp models --json`. `--list-models` no longer exists in current OMP (verified on 17.3.5), so the phase failed before reaching print mode. --- tests/fixtures/advisory-injector-extension.ts | 41 ++++++++ tests/test-omp-compat.mjs | 99 +++++++++++++++++-- 2 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 tests/fixtures/advisory-injector-extension.ts diff --git a/tests/fixtures/advisory-injector-extension.ts b/tests/fixtures/advisory-injector-extension.ts new file mode 100644 index 0000000..f88d5ed --- /dev/null +++ b/tests/fixtures/advisory-injector-extension.ts @@ -0,0 +1,41 @@ +/** + * Minimal OMP extension used by tests/test-omp-compat.mjs. + * + * Appends a custom advisor message to the session on session start, mirroring + * how the OMP Advisor runtime injects steering notes. OMP converts custom + * messages to `role: "developer"` LLM messages before handing them to the + * provider. Types are declared inline so the fixture has no runtime or + * typecheck dependency on OMP packages. + */ + +interface AdvisorInjectorSendMessage { + ( + message: { + customType: string + content: string + display: boolean + attribution: string + }, + options?: { triggerTurn?: boolean }, + ): void +} + +interface AdvisorInjectorApi { + on: (event: "session_start", handler: () => void | Promise) => void + sendMessage: AdvisorInjectorSendMessage +} + +export default function advisoryInjectorExtension(pi: AdvisorInjectorApi): void { + pi.on("session_start", async () => { + pi.sendMessage( + { + customType: "advisor", + content: + '\nStop and correct the benchmark.\n', + display: true, + attribution: "agent", + }, + { triggerTurn: false }, + ) + }) +} diff --git a/tests/test-omp-compat.mjs b/tests/test-omp-compat.mjs index af7c2bd..04f8c78 100644 --- a/tests/test-omp-compat.mjs +++ b/tests/test-omp-compat.mjs @@ -19,7 +19,10 @@ import { fileURLToPath } from "node:url" const __dirname = dirname(fileURLToPath(import.meta.url)) const PROJECT_DIR = resolve(__dirname, "..") const EXT_PATH = resolve(PROJECT_DIR, "index.ts") +const ADVISORY_EXT_PATH = resolve(PROJECT_DIR, "tests/fixtures/advisory-injector-extension.ts") const TEST_MODEL = "deepseek/deepseek-v4-flash" +const ADVISORY_XML = + '\nStop and correct the benchmark.\n' function findOmpBinary() { if (process.env.OMP_BIN) return process.env.OMP_BIN @@ -45,6 +48,7 @@ const tempHome = mkdtempSync(join(tmpdir(), "omp-cc-home-")) let requestCount = 0 let modelListRequestCount = 0 let lastRequestBody +let requestBodies = [] let lastRequestHeaders = {} const server = createServer((req, res) => { @@ -98,6 +102,7 @@ const server = createServer((req, res) => { req.on("end", () => { try { lastRequestBody = JSON.parse(body) + requestBodies.push(lastRequestBody) } catch { lastRequestBody = undefined } @@ -160,19 +165,33 @@ function runOmp(args, timeoutMs = 30_000) { try { console.log("[omp-compat] list models through real extension") modelListRequestCount = 0 - const result = await runOmp(["-e", EXT_PATH, "--list-models"]) - assert.equal(result.code, 0, result.stderr) - const listOutput = result.stdout || result.stderr - assert.match(listOutput, /commandcode/) - assert.match(listOutput, /deepseek\/deepseek-v4-flash/) - assert.equal(modelListRequestCount, 1) - assert.doesNotThrow(() => - accessSync(join(tempHome, ".omp", "agent", "commandcode-models.json"), constants.R_OK), - ) - assert.doesNotMatch(result.stdout + result.stderr, /Failed to load extension/) + const list = await runOmp(["models", "--json", "-e", EXT_PATH, "--no-extensions"]) + if (list.code !== 0 && /unknown|unrecognized/i.test(list.stderr + list.stdout)) { + console.log("[omp-compat] SKIP models phase - omp models subcommand unavailable") + } else { + assert.equal(list.code, 0, list.stderr) + let listed = null + try { + listed = JSON.parse(list.stdout) + } catch { + listed = null + } + const models = Array.isArray(listed?.models) ? listed.models : [] + assert.ok( + models.some((model) => model.provider === "commandcode"), + "commandcode provider should be listed", + ) + assert.ok( + models.some((model) => model.id === TEST_MODEL), + "mock catalog model should be listed", + ) + assert.ok(modelListRequestCount >= 1) + assert.doesNotMatch(list.stdout + list.stderr, /Failed to load extension/) + } console.log("[omp-compat] print mode through real extension and mock API") requestCount = 0 + requestBodies = [] const print = await runOmp( ["-e", EXT_PATH, "-p", "say mock token", "--model", `commandcode/${TEST_MODEL}`], 30_000, @@ -187,6 +206,66 @@ try { ) assert.equal(lastRequestBody?.params?.model, TEST_MODEL) assert.equal(typeof lastRequestBody?.params?.system, "string") + assert.doesNotThrow(() => + accessSync(join(tempHome, ".omp", "agent", "commandcode-models.json"), constants.R_OK), + ) + + console.log("[omp-compat] developer advisory reaches the provider request body") + requestCount = 0 + requestBodies = [] + const advisoryRun = await runOmp( + [ + "-e", + EXT_PATH, + "-e", + ADVISORY_EXT_PATH, + "-p", + "say mock token", + "--model", + `commandcode/${TEST_MODEL}`, + "--no-tools", + "--no-title", + ], + 30_000, + ) + assert.equal(advisoryRun.code, 0, advisoryRun.stderr) + assert.match(advisoryRun.stdout, /mock-omp-ok/) + + const promptBodies = requestBodies.filter((body) => + JSON.stringify(body?.params?.messages ?? []).includes("say mock token"), + ) + assert.ok(promptBodies.length >= 1, "expected at least one generate request with the prompt") + + for (const body of promptBodies) { + const messages = body?.params?.messages ?? [] + const advisoryMessages = messages.filter((message) => + JSON.stringify(message).includes("Stop and correct the benchmark."), + ) + assert.equal(advisoryMessages.length, 1, "the advisory should survive conversion exactly once") + const advisoryMessage = advisoryMessages[0] + assert.equal(advisoryMessage.role, "user") + const advisoryText = + typeof advisoryMessage.content === "string" + ? advisoryMessage.content + : (advisoryMessage.content ?? []) + .map((part) => (part?.type === "text" ? part.text : "")) + .join("\n") + assert.equal(advisoryText, ADVISORY_XML, "advisory content must arrive verbatim") + + const advisoryIndex = messages.indexOf(advisoryMessage) + const promptIndex = messages.findIndex((message) => + JSON.stringify(message).includes("say mock token"), + ) + assert.ok( + advisoryIndex < promptIndex, + "advisory must keep its chronological position relative to the prompt", + ) + assert.doesNotMatch( + String(body?.params?.system ?? ""), + /Stop and correct the benchmark|