ci(memory): report pull request memory changes

.github/scripts/memory-benchmark.mjs

@@ -0,0 +1,408 @@
+#!/usr/bin/env node
+
+import { mkdir, readFile, writeFile } from "node:fs/promises"
+import { tmpdir } from "node:os"
+import { join, resolve } from "node:path"
+import { execFile, spawn } from "node:child_process"
+import process from "node:process"
+import { promisify } from "node:util"
+
+const execFileAsync = promisify(execFile)
+
+const KIB_PER_MIB = 1024
+const RUNS = positiveInteger(process.env.MEMORY_BENCHMARK_RUNS, 6)
+const WARMUP_MS = positiveInteger(process.env.MEMORY_BENCHMARK_WARMUP_MS, 2500)
+const SAMPLE_COUNT = positiveInteger(process.env.MEMORY_BENCHMARK_SAMPLES, 12)
+const SAMPLE_INTERVAL_MS = positiveInteger(process.env.MEMORY_BENCHMARK_INTERVAL_MS, 100)
+
+const basePath = requiredPath("MEMORY_BENCHMARK_BASE_PATH")
+const headPath = requiredPath("MEMORY_BENCHMARK_HEAD_PATH")
+const piCli = requiredPath("MEMORY_BENCHMARK_PI_CLI")
+const outputPath = resolve(process.env.MEMORY_BENCHMARK_OUTPUT ?? "memory-benchmark.md")
+const jsonOutputPath = resolve(process.env.MEMORY_BENCHMARK_JSON_OUTPUT ?? "memory-benchmark.json")
+const baseSha = process.env.MEMORY_BENCHMARK_BASE_SHA ?? "base"
+const headSha = process.env.MEMORY_BENCHMARK_HEAD_SHA ?? "head"
+const piVersion = process.env.MEMORY_BENCHMARK_PI_VERSION ?? "unknown"
+const runtimeName = process.versions.bun ? "Bun" : "Node"
+const runtimeVersion = process.versions.bun ?? process.versions.node
+
+const benchmarkDir = await mkdir(join(tmpdir(), `pi-memory-benchmark-${process.pid}`), {
+  recursive: true,
+}).then(() => join(tmpdir(), `pi-memory-benchmark-${process.pid}`))
+const cachePath = join(benchmarkDir, "commandcode-models.json")
+await writeFile(
+  cachePath,
+  `${JSON.stringify(
+    {
+      version: 1,
+      models: [
+        {
+          id: "memory-benchmark-model",
+          name: "Memory Benchmark Model (CC)",
+          reasoning: true,
+          contextWindow: 128_000,
+          maxTokens: 65_536,
+        },
+      ],
+    },
+    null,
+    2,
+  )}\n`,
+)
+
+const variants = {
+  baseline: { label: "pi without extension", extensionPath: undefined },
+  base: { label: `Base (${shortSha(baseSha)})`, extensionPath: basePath },
+  head: { label: `PR (${shortSha(headSha)})`, extensionPath: headPath },
+}
+
+const results = Object.fromEntries(Object.keys(variants).map((key) => [key, []]))
+
+console.log(
+  `Benchmarking ${RUNS} alternating rounds with pi ${piVersion}, ${runtimeName} ${runtimeVersion}, ` +
+    `${SAMPLE_COUNT} samples after ${WARMUP_MS} ms warm-up`,
+)
+
+// Discard one cold run per variant before collecting measurements.
+for (const key of ["baseline", "base", "head"]) {
+  console.log(`Cold warm-up: ${variants[key].label}`)
+  await measureProcess(variants[key].extensionPath)
+}
+
+for (let round = 0; round < RUNS; round += 1) {
+  const order = round % 2 === 0 ? ["baseline", "base", "head"] : ["baseline", "head", "base"]
+  console.log(`Round ${round + 1}/${RUNS}: ${order.map((key) => variants[key].label).join(" → ")}`)
+
+  for (const key of order) {
+    const measurement = await measureProcess(variants[key].extensionPath)
+    results[key].push(measurement)
+    console.log(`  ${variants[key].label}: ${formatProgress(measurement)}`)
+  }
+}
+
+const summary = Object.fromEntries(
+  Object.entries(results).map(([key, measurements]) => [key, summarize(measurements)]),
+)
+const comparisons = summarizeComparisons(results)
+const report = renderReport(summary, comparisons)
+
+await writeFile(outputPath, report)
+await writeFile(
+  jsonOutputPath,
+  `${JSON.stringify(
+    {
+      metadata: {
+        baseSha,
+        headSha,
+        piVersion,
+        runtimeName,
+        runtimeVersion,
+        runs: RUNS,
+        warmupMs: WARMUP_MS,
+        sampleCount: SAMPLE_COUNT,
+        sampleIntervalMs: SAMPLE_INTERVAL_MS,
+      },
+      runs: results,
+      summary,
+      comparisons,
+    },
+    null,
+    2,
+  )}\n`,
+)
+
+console.log(`Wrote ${outputPath}`)
+console.log(`Wrote ${jsonOutputPath}`)
+
+async function measureProcess(extensionPath) {
+  const args = [
+    piCli,
+    "--mode",
+    "rpc",
+    "--no-session",
+    "--no-extensions",
+    "--no-skills",
+    "--no-prompt-templates",
+    "--no-themes",
+    "--no-context-files",
+  ]
+  if (extensionPath) args.push("-e", extensionPath)
+
+  const child = spawn(process.execPath, args, {
+    detached: true,
+    env: {
+      ...process.env,
+      PI_CODING_AGENT_DIR: join(benchmarkDir, "pi-agent"),
+      PI_OFFLINE: "1",
+      COMMANDCODE_MODELS_CACHE: cachePath,
+      COMMANDCODE_MODELS_URL: "http://127.0.0.1:9/provider/v1/models",
+    },
+    stdio: ["pipe", "pipe", "pipe"],
+  })
+
+  let stderr = ""
+  child.stdout.resume()
+  child.stderr.on("data", (chunk) => {
+    stderr += chunk.toString("utf8")
+    if (stderr.length > 16_384) stderr = stderr.slice(-16_384)
+  })
+
+  try {
+    await wait(WARMUP_MS)
+    ensureRunning(child, stderr)
+
+    const samples = []
+    for (let index = 0; index < SAMPLE_COUNT; index += 1) {
+      samples.push(await readSampledMemory(child.pid))
+      await wait(SAMPLE_INTERVAL_MS)
+      ensureRunning(child, stderr)
+    }
+
+    const sampled = Object.fromEntries(
+      Object.keys(samples[0]).map((metric) => [
+        metric,
+        metric === "peakRss"
+          ? Math.max(...samples.map((sample) => sample[metric]))
+          : median(samples.map((sample) => sample[metric])),
+      ]),
+    )
+    const snapshot = process.platform === "darwin" ? await readDarwinFootprint(child.pid) : {}
+    return { ...sampled, ...snapshot }
+  } finally {
+    stopProcessGroup(child)
+    await Promise.race([onceExit(child), wait(3000)])
+    stopProcessGroup(child, "SIGKILL")
+  }
+}
+
+async function readSampledMemory(pid) {
+  if (process.platform === "darwin") {
+    const { stdout } = await execFileAsync("/bin/ps", ["-o", "rss=", "-p", String(pid)])
+    const rssKiB = Number.parseInt(stdout.trim(), 10)
+    if (!Number.isFinite(rssKiB)) throw new Error(`Could not parse RSS from ps output: ${stdout}`)
+    return { rss: rssKiB / KIB_PER_MIB }
+  }
+
+  if (process.platform === "linux") return readLinuxMemory(pid)
+  throw new Error(`Unsupported memory benchmark platform: ${process.platform}`)
+}
+
+async function readDarwinFootprint(pid) {
+  const { stdout } = await execFileAsync("/usr/bin/footprint", [
+    "-p",
+    String(pid),
+    "-f",
+    "bytes",
+    "--noCategories",
+  ])
+  const footprint = /Footprint:\s*(\d+) B/.exec(stdout)
+  const peak = /phys_footprint_peak:\s*(\d+) B/.exec(stdout)
+  if (!footprint || !peak) throw new Error(`Could not parse macOS footprint output:\n${stdout}`)
+
+  return {
+    physicalFootprint: Number(footprint[1]) / 1024 / 1024,
+    physicalPeak: Number(peak[1]) / 1024 / 1024,
+  }
+}
+
+async function readLinuxMemory(pid) {
+  const [status, smaps] = await Promise.all([
+    readFile(`/proc/${pid}/status`, "utf8"),
+    readFile(`/proc/${pid}/smaps_rollup`, "utf8"),
+  ])
+  const statusValues = parseKiBFields(status)
+  const smapsValues = parseKiBFields(smaps)
+  const privateMemory =
+    (smapsValues.Private_Clean ?? 0) +
+    (smapsValues.Private_Dirty ?? 0) +
+    (smapsValues.Private_Hugetlb ?? 0)
+
+  return {
+    rss: requireMetric(statusValues, "VmRSS"),
+    anonymousRss: requireMetric(statusValues, "RssAnon"),
+    pss: requireMetric(smapsValues, "Pss"),
+    uss: privateMemory,
+    peakRss: requireMetric(statusValues, "VmHWM"),
+  }
+}
+
+function parseKiBFields(contents) {
+  const result = {}
+  for (const line of contents.split("\n")) {
+    const match = /^([A-Za-z_]+):\s+(\d+) kB$/.exec(line.trim())
+    if (match) result[match[1]] = Number(match[2]) / KIB_PER_MIB
+  }
+  return result
+}
+
+function summarize(measurements) {
+  return Object.fromEntries(
+    Object.keys(measurements[0]).map((metric) => {
+      const values = measurements.map((measurement) => measurement[metric])
+      const center = median(values)
+      return [
+        metric,
+        { median: center, mad: median(values.map((value) => Math.abs(value - center))) },
+      ]
+    }),
+  )
+}
+
+function summarizeComparisons(measurements) {
+  const metrics = Object.keys(measurements.baseline[0])
+  const paired = (left, right, metric) =>
+    left.map((measurement, index) => measurement[metric] - right[index][metric])
+  const estimate = (values) => {
+    const center = median(values)
+    return { median: center, mad: median(values.map((value) => Math.abs(value - center))) }
+  }
+
+  return Object.fromEntries(
+    metrics.map((metric) => [
+      metric,
+      {
+        headMinusBase: estimate(paired(measurements.head, measurements.base, metric)),
+        baseOverhead: estimate(paired(measurements.base, measurements.baseline, metric)),
+        headOverhead: estimate(paired(measurements.head, measurements.baseline, metric)),
+      },
+    ]),
+  )
+}
+
+function renderReport(summary, comparisons) {
+  const metrics =
+    process.platform === "darwin"
+      ? [
+          ["rss", "Stable RSS"],
+          ["physicalFootprint", "Physical footprint"],
+          ["physicalPeak", "Physical peak"],
+        ]
+      : [
+          ["rss", "Stable RSS"],
+          ["anonymousRss", "Anonymous RSS"],
+          ["pss", "PSS"],
+          ["uss", "USS (private memory)"],
+          ["peakRss", "Peak RSS"],
+        ]
+
+  const comparisonRows = metrics
+    .map(([key, label]) => {
+      const base = summary.base[key]
+      const head = summary.head[key]
+      const difference = comparisons[key].headMinusBase
+      const percentage = base.median === 0 ? 0 : (difference.median / base.median) * 100
+      return `| ${label} | ${formatEstimate(base)} | ${formatEstimate(head)} | ${formatSignedEstimate(difference)} | ${formatSigned(percentage, "%")} |`
+    })
+    .join("\n")
+
+  const overheadRows = metrics
+    .map(([key, label]) => {
+      const comparison = comparisons[key]
+      return `| ${label} | ${formatSignedEstimate(comparison.baseOverhead)} | ${formatSignedEstimate(comparison.headOverhead)} | ${formatSignedEstimate(comparison.headMinusBase)} |`
+    })
+    .join("\n")
+
+  return (
+    `## Memory benchmark\n\n` +
+    `Compared base \`${shortSha(baseSha)}\` with PR head \`${shortSha(headSha)}\` on the same GitHub-hosted ${process.platform} runner. Lower values are better.\n\n` +
+    `| Metric | Base | PR | PR − Base | Change |\n` +
+    `|---|---:|---:|---:|---:|\n` +
+    `${comparisonRows}\n\n` +
+    `### Extension overhead above pi baseline\n\n` +
+    `| Metric | Base overhead | PR overhead | Difference |\n` +
+    `|---|---:|---:|---:|\n` +
+    `${overheadRows}\n\n` +
+    `Values are medians of ${RUNS} alternating, paired runs. The value after \`±\` is the median absolute deviation (MAD). ` +
+    `Each process was sampled ${SAMPLE_COUNT} times after a ${WARMUP_MS} ms warm-up.\n\n` +
+    `Environment: pi \`${piVersion}\`, ${runtimeName} \`${runtimeVersion}\`, ${process.platform} \`${process.arch}\`. ` +
+    measurementSource() +
+    `\n\n> This is a comparative signal, not a pass/fail threshold. GitHub-hosted runner noise can affect absolute values.\n`
+  )
+}
+
+function measurementSource() {
+  if (process.platform === "darwin") {
+    return "RSS comes from `ps`; physical footprint and peak come from macOS `footprint`."
+  }
+  return "PSS and USS come from `/proc/<pid>/smaps_rollup`; RSS metrics come from `/proc/<pid>/status`."
+}
+
+function median(values) {
+  const sorted = [...values].sort((left, right) => left - right)
+  const middle = Math.floor(sorted.length / 2)
+  return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]
+}
+
+function formatProgress(measurement) {
+  if (process.platform === "darwin") {
+    return `${formatMiB(measurement.rss)} RSS, ${formatMiB(measurement.physicalFootprint)} physical`
+  }
+  return `${formatMiB(measurement.rss)} RSS, ${formatMiB(measurement.pss)} PSS`
+}
+
+function formatEstimate(value) {
+  return `${formatMiB(value.median)} ± ${value.mad.toFixed(1)} MiB`
+}
+
+function formatMiB(value) {
+  return `${value.toFixed(1)} MiB`
+}
+
+function formatSignedEstimate(value) {
+  return `${formatSigned(value.median)} ± ${value.mad.toFixed(1)} MiB`
+}
+
+function formatSigned(value, suffix = " MiB") {
+  const sign = value > 0 ? "+" : ""
+  return `${sign}${value.toFixed(1)}${suffix}`
+}
+
+function requiredPath(name) {
+  const value = process.env[name]
+  if (!value) throw new Error(`${name} is required`)
+  return resolve(value)
+}
+
+function positiveInteger(value, fallback) {
+  if (value === undefined) return fallback
+  const parsed = Number.parseInt(value, 10)
+  if (!Number.isInteger(parsed) || parsed <= 0)
+    throw new Error(`Expected a positive integer, got ${value}`)
+  return parsed
+}
+
+function requireMetric(values, name) {
+  const value = values[name]
+  if (value === undefined) throw new Error(`Missing ${name} in Linux process memory data`)
+  return value
+}
+
+function shortSha(sha) {
+  return sha.slice(0, 7)
+}
+
+function ensureRunning(child, stderr) {
+  if (child.exitCode !== null || child.signalCode !== null) {
+    throw new Error(
+      `pi exited before memory sampling completed (code ${child.exitCode}, signal ${child.signalCode})\n${stderr}`,
+    )
+  }
+}
+
+function stopProcessGroup(child, signal = "SIGTERM") {
+  if (!child.pid) return
+  try {
+    process.kill(-child.pid, signal)
+  } catch (error) {
+    if (error?.code !== "ESRCH") throw error
+  }
+}
+
+function onceExit(child) {
+  if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve()
+  return new Promise((resolveExit) => child.once("exit", resolveExit))
+}
+
+function wait(milliseconds) {
+  return new Promise((resolveWait) => setTimeout(resolveWait, milliseconds))
+}

.github/workflows/memory-benchmark.yml

@@ -0,0 +1,154 @@
+name: Memory benchmark
+
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened, ready_for_review]
+
+concurrency:
+  group: memory-benchmark-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  compare:
+    name: Compare base and PR memory
+    runs-on: macos-14
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    env:
+      PI_VERSION: 0.82.1
+      BUN_VERSION: 1.3.11
+      NODE_VERSION: 22.19.0
+      MEMORY_BENCHMARK_RUNS: 6
+      MEMORY_BENCHMARK_WARMUP_MS: 2500
+      MEMORY_BENCHMARK_SAMPLES: 12
+      MEMORY_BENCHMARK_INTERVAL_MS: 100
+
+    steps:
+      - name: Check out benchmark implementation
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: benchmark
+          persist-credentials: false
+
+      - name: Check out base revision
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          persist-credentials: false
+
+      - name: Check out PR revision
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: head
+          persist-credentials: false
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: npm
+          cache-dependency-path: |
+            base/package-lock.json
+            head/package-lock.json
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: ${{ env.BUN_VERSION }}
+
+      - name: Install base dependencies
+        working-directory: base
+        run: npm ci --ignore-scripts
+
+      - name: Install PR dependencies
+        working-directory: head
+        run: npm ci --ignore-scripts
+
+      - name: Install pinned pi host
+        run: |
+          npm install \
+            --prefix pi-host \
+            --ignore-scripts \
+            --no-save \
+            "@earendil-works/pi-coding-agent@$PI_VERSION"
+
+      - name: Compare memory usage
+        env:
+          MEMORY_BENCHMARK_BASE_PATH: ${{ github.workspace }}/base
+          MEMORY_BENCHMARK_HEAD_PATH: ${{ github.workspace }}/head
+          MEMORY_BENCHMARK_PI_CLI: ${{ github.workspace }}/pi-host/node_modules/@earendil-works/pi-coding-agent/dist/cli.js
+          MEMORY_BENCHMARK_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          MEMORY_BENCHMARK_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          MEMORY_BENCHMARK_PI_VERSION: ${{ env.PI_VERSION }}
+          MEMORY_BENCHMARK_OUTPUT: ${{ github.workspace }}/memory-benchmark.md
+          MEMORY_BENCHMARK_JSON_OUTPUT: ${{ github.workspace }}/memory-benchmark.json
+        run: bun benchmark/.github/scripts/memory-benchmark.mjs
+
+      - name: Add benchmark to job summary
+        if: always() && hashFiles('memory-benchmark.md') != ''
+        run: cat memory-benchmark.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload benchmark report
+        if: always() && hashFiles('memory-benchmark.md') != ''
+        uses: actions/upload-artifact@v4
+        with:
+          name: memory-benchmark-report
+          path: |
+            memory-benchmark.md
+            memory-benchmark.json
+          retention-days: 30
+
+  comment:
+    name: Update PR comment
+    needs: compare
+    if: >-
+      always() &&
+      needs.compare.result == 'success' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+      contents: read
+      pull-requests: write
+
+    steps:
+      - name: Download benchmark report
+        uses: actions/download-artifact@v4
+        with:
+          name: memory-benchmark-report
+
+      - name: Update sticky PR comment
+        uses: actions/github-script@v7
+        env:
+          REPORT_PATH: memory-benchmark.md
+        with:
+          script: |
+            const fs = require("node:fs")
+            const marker = "<!-- pi-commandcode-memory-benchmark -->"
+            const report = fs.readFileSync(process.env.REPORT_PATH, "utf8")
+            const body = `${marker}\n${report}`
+            const { owner, repo } = context.repo
+            const issue_number = context.issue.number
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner,
+              repo,
+              issue_number,
+              per_page: 100,
+            })
+            const previous = comments.find(
+              (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker),
+            )
+
+            if (previous) {
+              await github.rest.issues.updateComment({
+                owner,
+                repo,
+                comment_id: previous.id,
+                body,
+              })
+            } else {
+              await github.rest.issues.createComment({ owner, repo, issue_number, body })
+            }