From cfcf114497b532becf934e31d2785ca9d658f669 Mon Sep 17 00:00:00 2001
From: Patrick Wozniak <email@patwoz.de>
Date: Tue, 4 Aug 2026 09:35:14 +0200
Subject: [PATCH] ci(memory): harden benchmark reporting

---
 .github/scripts/memory-benchmark.mjs   | 17 +++++++++++++----
 .github/workflows/memory-benchmark.yml | 16 ++++++++--------
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/.github/scripts/memory-benchmark.mjs b/.github/scripts/memory-benchmark.mjs
index c693f57..59d15be 100644
--- a/.github/scripts/memory-benchmark.mjs
+++ b/.github/scripts/memory-benchmark.mjs
@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-import { mkdir, readFile, writeFile } from "node:fs/promises"
+import { mkdtemp, readFile, writeFile } from "node:fs/promises"
 import { tmpdir } from "node:os"
 import { join, resolve } from "node:path"
 import { execFile, spawn } from "node:child_process"
@@ -26,9 +26,7 @@ const piVersion = process.env.MEMORY_BENCHMARK_PI_VERSION ?? "unknown"
 const runtimeName = process.versions.bun ? "Bun" : "Node"
 const runtimeVersion = process.versions.bun ?? process.versions.node
 
-const benchmarkDir = await mkdir(join(tmpdir(), `pi-memory-benchmark-${process.pid}`), {
-  recursive: true,
-}).then(() => join(tmpdir(), `pi-memory-benchmark-${process.pid}`))
+const benchmarkDir = await mkdtemp(join(tmpdir(), "pi-memory-benchmark-"))
 const cachePath = join(benchmarkDir, "commandcode-models.json")
 await writeFile(
   cachePath,
@@ -295,6 +293,16 @@ function renderReport(summary, comparisons) {
     })
     .join("\n")
 
+  const clearChanges = metrics
+    .map(([key, label]) => [label, comparisons[key].headMinusBase])
+    .filter(([, estimate]) => Math.abs(estimate.median) > estimate.mad)
+  const interpretation =
+    clearChanges.length === 0
+      ? "No metric shows a clear Base-to-PR difference beyond its measured run-to-run variation."
+      : `Differences larger than their measured MAD: ${clearChanges
+          .map(([label, estimate]) => `${label} ${formatSignedEstimate(estimate)}`)
+          .join(", ")}.`
+
   const overheadRows = metrics
     .map(([key, label]) => {
       const comparison = comparisons[key]
@@ -308,6 +316,7 @@ function renderReport(summary, comparisons) {
     `| Metric | Base | PR | PR − Base | Change |\n` +
     `|---|---:|---:|---:|---:|\n` +
     `${comparisonRows}\n\n` +
+    `**Interpretation:** ${interpretation}\n\n` +
     `### Extension overhead above pi baseline\n\n` +
     `| Metric | Base overhead | PR overhead | Difference |\n` +
     `|---|---:|---:|---:|\n` +
diff --git a/.github/workflows/memory-benchmark.yml b/.github/workflows/memory-benchmark.yml
index 112b4f1..3924c83 100644
--- a/.github/workflows/memory-benchmark.yml
+++ b/.github/workflows/memory-benchmark.yml
@@ -27,27 +27,27 @@ jobs:
 
     steps:
       - name: Check out benchmark implementation
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           path: benchmark
           persist-credentials: false
 
       - name: Check out base revision
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           ref: ${{ github.event.pull_request.base.sha }}
           path: base
           persist-credentials: false
 
       - name: Check out PR revision
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           path: head
           persist-credentials: false
 
-      - uses: actions/setup-node@v4
+      - uses: actions/setup-node@v7
         with:
           node-version: ${{ env.NODE_VERSION }}
           cache: npm
@@ -55,7 +55,7 @@ jobs:
             base/package-lock.json
             head/package-lock.json
 
-      - uses: oven-sh/setup-bun@v2
+      - uses: oven-sh/setup-bun@v2.2.0
         with:
           bun-version: ${{ env.BUN_VERSION }}
 
@@ -93,7 +93,7 @@ jobs:
 
       - name: Upload benchmark report
         if: always() && hashFiles('memory-benchmark.md') != ''
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v7
         with:
           name: memory-benchmark-report
           path: |
@@ -116,12 +116,12 @@ jobs:
 
     steps:
       - name: Download benchmark report
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v8
         with:
           name: memory-benchmark-report
 
       - name: Update sticky PR comment
-        uses: actions/github-script@v7
+        uses: actions/github-script@v9
         env:
           REPORT_PATH: memory-benchmark.md
         with: