cat-shark/pi-commandcode-provider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(e2e): cover live Go and GOAT accounts

Browse Source
This commit is contained in:
Patrick Wozniak
2026-08-25 16:37:24 +02:00
parent 804ba5862e
commit cee0d7cc96
6 changed files with 157 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test-live-e2e.mjs
+128 -14
View File
@@ -27,7 +27,20 @@ const extensionPath = join(projectDir, "index.ts")
const testModel = process.env.COMMANDCODE_E2E_MODEL ?? "deepseek/deepseek-v4-flash"
const testProfile = process.env.COMMANDCODE_E2E_PROFILE
const expectedTransport =
testProfile === "go" ? "generate" : testProfile === "provider" ? "provider" : undefined
testProfile === "go"
? "generate"
: testProfile === "goat" || testProfile === "provider"
? "provider"
: undefined
const expectedPlan =
testProfile === "go"
? "go"
: testProfile === "goat"
? "goat"
: testProfile === "provider"
? "provider"
: undefined
const goatVisionModel = process.env.COMMANDCODE_E2E_GOAT_VISION_MODEL ?? "google/gemini-3.7-flash"
const marker = "commandcode-live-e2e-ok"
function findPiBinary() {
@@ -104,7 +117,7 @@ function run(command, args, options = {}) {
})
}
async function runRpc(extension, action, timeoutMs = 120_000) {
async function runRpc(extension, action, timeoutMs = 120_000, model = testModel) {
const child = spawn(
piBin,
[
@@ -116,7 +129,9 @@ async function runRpc(extension, action, timeoutMs = 120_000) {
"--provider",
"commandcode",
"--model",
testModel,
model,
"--thinking",
"high",
],
{ cwd: projectDir, env: safeEnv(), stdio: ["pipe", "pipe", "pipe"] },
)
@@ -224,8 +239,19 @@ try {
await waitFor(
(event) => event.type === "response" && event.id === "reasoning-turn-1" && event.success,
)
await waitFor((event) => event.type === "agent_settled")
const firstThinkingDeltas = countThinkingDeltas(firstStart)
const firstSettled = await waitFor(
(event) => event.type === "agent_settled" && events.indexOf(event) >= firstStart,
)
const firstSettledIndex = events.indexOf(firstSettled)
const firstThinkingDeltas = events
.slice(firstStart, firstSettledIndex + 1)
.filter(
(event) =>
event.type === "message_update" &&
event.assistantMessageEvent?.type === "thinking_delta" &&
typeof event.assistantMessageEvent.delta === "string" &&
event.assistantMessageEvent.delta.length > 0,
).length
const secondStart = events.length
send({
@@ -237,15 +263,16 @@ try {
await waitFor(
(event) => event.type === "response" && event.id === "reasoning-turn-2" && event.success,
)
await waitFor((event) => event.type === "agent_settled" && events.indexOf(event) >= secondStart)
const secondSettled = await waitFor(
(event) => event.type === "agent_settled" && events.indexOf(event) >= secondStart,
)
const secondThinkingDeltas = countThinkingDeltas(secondStart)
assert.ok(events.indexOf(secondSettled) >= secondStart)
return { firstThinkingDeltas, secondThinkingDeltas, stderr: getStderr() }
})
if (testProfile !== "provider") {
assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning")
assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning")
}
assert.ok(multiTurn.firstThinkingDeltas > 0, "first turn should stream reasoning")
assert.ok(multiTurn.secondThinkingDeltas > 0, "follow-up turn should stream fresh reasoning")
assert.doesNotMatch(multiTurn.stderr, /Bearer\s+\S+/i)
console.log("[live-e2e] live runtime refresh/status commands")
@@ -283,15 +310,66 @@ try {
typeof event.message === "string" &&
event.message.includes("source:"),
)
return { names, refresh: refresh.message, status: status.message, stderr: getStderr() }
send({ id: "quota", type: "prompt", message: "/commandcode-quota" })
await waitFor((event) => event.type === "response" && event.id === "quota" && event.success)
const quota = await waitFor(
(event) =>
event.type === "extension_ui_request" &&
event.method === "notify" &&
typeof event.message === "string" &&
event.message.includes("Plan:"),
)
return {
names,
refresh: refresh.message,
status: status.message,
quota: quota.message,
stderr: getStderr(),
}
})
assert.ok(runtime.names.includes("commandcode-refresh"))
assert.ok(runtime.names.includes("commandcode-status"))
assert.ok(runtime.names.includes("commandcode-quota"))
assert.match(runtime.refresh, /model catalog (?:refreshed|unchanged)/)
if (expectedTransport) assert.match(runtime.status, new RegExp(`transport: ${expectedTransport}`))
assert.match(runtime.status, /source: (?:live|cache)/)
assert.match(runtime.status, /model count: [1-9][0-9]*/)
assert.doesNotMatch(`${runtime.refresh}\n${runtime.status}\n${runtime.stderr}`, /Bearer\s+\S+/i)
if (expectedPlan) assert.match(runtime.quota, new RegExp(`Plan:.*\\b${expectedPlan}\\b`, "i"))
assert.doesNotMatch(
`${runtime.refresh}\n${runtime.status}\n${runtime.quota}\n${runtime.stderr}`,
/Bearer\s+\S+/i,
)
console.log("[live-e2e] live abort through real RPC host")
const abortResult = await runRpc(extensionPath, async ({ send, waitFor, events, getStderr }) => {
const startIndex = events.length
send({
id: "abort-turn",
type: "prompt",
message: "Write a very long detailed explanation of every integer from 1 to 10000.",
})
await waitFor(
(event) => event.type === "response" && event.id === "abort-turn" && event.success,
)
await waitFor((event) => event.type === "message_update" && events.indexOf(event) >= startIndex)
send({ id: "abort", type: "abort" })
await waitFor((event) => event.type === "response" && event.id === "abort" && event.success)
await waitFor((event) => event.type === "agent_settled" && events.indexOf(event) >= startIndex)
return {
aborted: events
.slice(startIndex)
.some(
(event) =>
event.type === "message_end" &&
event.message?.role === "assistant" &&
event.message?.stopReason === "aborted",
),
stderr: getStderr(),
}
})
assert.equal(abortResult.aborted, true)
assert.doesNotMatch(abortResult.stderr, /Bearer\s+\S+/i)
console.log("[live-e2e] live tool-call round trip")
const toolRoot = join(tempRoot, "tool-roundtrip")
@@ -319,9 +397,45 @@ try {
)
assert.equal(toolResult.code, 0, toolResult.stderr)
assert.match(toolResult.stdout, new RegExp(marker))
assert.equal(readFileSync(targetPath, "utf-8"), marker)
assert.equal(readFileSync(targetPath, "utf-8").trimEnd(), marker)
if (testProfile !== "provider") {
if (testProfile === "goat") {
console.log("[live-e2e] live vision request through Provider API")
const vision = await runRpc(
extensionPath,
async ({ send, waitFor, events, getStderr }) => {
const startIndex = events.length
send({
id: "vision",
type: "prompt",
message: "Describe the attached image briefly.",
images: [
{
type: "image",
data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=",
mimeType: "image/png",
},
],
})
await waitFor(
(event) => event.type === "response" && event.id === "vision" && event.success,
)
await waitFor(
(event) => event.type === "agent_settled" && events.indexOf(event) >= startIndex,
)
const messageEnd = events
.slice(startIndex)
.find((event) => event.type === "message_end" && event.message?.role === "assistant")
return { messageEnd, stderr: getStderr() }
},
180_000,
goatVisionModel,
)
assert.notEqual(vision.messageEnd?.message?.stopReason, "error")
assert.doesNotMatch(vision.stderr, /Bearer\s+\S+/i)
}
if (testProfile === "go") {
console.log("[live-e2e] image rejection through real RPC host")
const image = await runRpc(extensionPath, async ({ send, waitFor, events }) => {
send({