ci(memory): report pull request memory changes

2026-08-04 09:29:59 +02:00
parent 0e91395d7f
commit eb6c18bba2
2 changed files with 562 additions and 0 deletions
@@ -0,0 +1,154 @@
+name: Memory benchmark
+
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened, ready_for_review]
+
+concurrency:
+  group: memory-benchmark-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  compare:
+    name: Compare base and PR memory
+    runs-on: macos-14
+    timeout-minutes: 15
+    permissions:
+      contents: read
+    env:
+      PI_VERSION: 0.82.1
+      BUN_VERSION: 1.3.11
+      NODE_VERSION: 22.19.0
+      MEMORY_BENCHMARK_RUNS: 6
+      MEMORY_BENCHMARK_WARMUP_MS: 2500
+      MEMORY_BENCHMARK_SAMPLES: 12
+      MEMORY_BENCHMARK_INTERVAL_MS: 100
+
+    steps:
+      - name: Check out benchmark implementation
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: benchmark
+          persist-credentials: false
+
+      - name: Check out base revision
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha }}
+          path: base
+          persist-credentials: false
+
+      - name: Check out PR revision
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: head
+          persist-credentials: false
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: npm
+          cache-dependency-path: |
+            base/package-lock.json
+            head/package-lock.json
+
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: ${{ env.BUN_VERSION }}
+
+      - name: Install base dependencies
+        working-directory: base
+        run: npm ci --ignore-scripts
+
+      - name: Install PR dependencies
+        working-directory: head
+        run: npm ci --ignore-scripts
+
+      - name: Install pinned pi host
+        run: |
+          npm install \
+            --prefix pi-host \
+            --ignore-scripts \
+            --no-save \
+            "@earendil-works/pi-coding-agent@$PI_VERSION"
+
+      - name: Compare memory usage
+        env:
+          MEMORY_BENCHMARK_BASE_PATH: ${{ github.workspace }}/base
+          MEMORY_BENCHMARK_HEAD_PATH: ${{ github.workspace }}/head
+          MEMORY_BENCHMARK_PI_CLI: ${{ github.workspace }}/pi-host/node_modules/@earendil-works/pi-coding-agent/dist/cli.js
+          MEMORY_BENCHMARK_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          MEMORY_BENCHMARK_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+          MEMORY_BENCHMARK_PI_VERSION: ${{ env.PI_VERSION }}
+          MEMORY_BENCHMARK_OUTPUT: ${{ github.workspace }}/memory-benchmark.md
+          MEMORY_BENCHMARK_JSON_OUTPUT: ${{ github.workspace }}/memory-benchmark.json
+        run: bun benchmark/.github/scripts/memory-benchmark.mjs
+
+      - name: Add benchmark to job summary
+        if: always() && hashFiles('memory-benchmark.md') != ''
+        run: cat memory-benchmark.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload benchmark report
+        if: always() && hashFiles('memory-benchmark.md') != ''
+        uses: actions/upload-artifact@v4
+        with:
+          name: memory-benchmark-report
+          path: |
+            memory-benchmark.md
+            memory-benchmark.json
+          retention-days: 30
+
+  comment:
+    name: Update PR comment
+    needs: compare
+    if: >-
+      always() &&
+      needs.compare.result == 'success' &&
+      github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+      contents: read
+      pull-requests: write
+
+    steps:
+      - name: Download benchmark report
+        uses: actions/download-artifact@v4
+        with:
+          name: memory-benchmark-report
+
+      - name: Update sticky PR comment
+        uses: actions/github-script@v7
+        env:
+          REPORT_PATH: memory-benchmark.md
+        with:
+          script: |
+            const fs = require("node:fs")
+            const marker = "<!-- pi-commandcode-memory-benchmark -->"
+            const report = fs.readFileSync(process.env.REPORT_PATH, "utf8")
+            const body = `${marker}\n${report}`
+            const { owner, repo } = context.repo
+            const issue_number = context.issue.number
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner,
+              repo,
+              issue_number,
+              per_page: 100,
+            })
+            const previous = comments.find(
+              (comment) => comment.user?.type === "Bot" && comment.body?.includes(marker),
+            )
+
+            if (previous) {
+              await github.rest.issues.updateComment({
+                owner,
+                repo,
+                comment_id: previous.id,
+                body,
+              })
+            } else {
+              await github.rest.issues.createComment({ owner, repo, issue_number, body })
+            }