ci: track benchmark results with github-action-benchmark

Each bench job writes metrics to a JSONL file, collects into JSON,
and feeds to benchmark-action/github-action-benchmark. On push to
main, results are stored on gh-pages with trend charts. On PRs,
alerts fire if any metric regresses >30%. Linux and Windows use
separate data directories to avoid push races.

Signed-off-by: danbugs <danilochiarlone@gmail.com>

Author: danbugs

.github/workflows/benchmarks.yml

@@ -109,8 +109,9 @@ jobs:
     runs-on: ubuntu-latest
     needs: build-image
     permissions:
-      contents: read
+      contents: write
       packages: read
+      pull-requests: write
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
@@ -186,6 +187,7 @@ jobs:
           echo "| Runs   | $n |"
           echo ""
           echo "::notice::hello_world: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms"
+          echo "{\"name\":\"hello_world (median)\",\"unit\":\"ms\",\"value\":$median}" >> /tmp/bench-results.jsonl
 
       - name: "Perf: pandas (10 runs)"
         if: steps.kvm_check.outputs.available == 'true'
@@ -221,6 +223,7 @@ jobs:
           echo "| Runs   | $n |"
           echo ""
           echo "::notice::pandas: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms"
+          echo "{\"name\":\"pandas (median)\",\"unit\":\"ms\",\"value\":$median}" >> /tmp/bench-results.jsonl
 
       - name: "Density: concurrent VMs (5 VMs)"
         if: steps.kvm_check.outputs.available == 'true'
@@ -253,6 +256,7 @@ jobs:
             echo "| Total Private_Dirty | $((total_private / 1024)) MB |"
             echo ""
             echo "::notice::density: ${count} VMs, per_vm=${per_vm}MB private_dirty"
+            echo "{\"name\":\"density (per VM)\",\"unit\":\"MB\",\"value\":$per_vm}" >> /tmp/bench-results.jsonl
           else
             echo "::warning::no VMs were alive long enough to measure"
           fi
@@ -270,16 +274,41 @@ jobs:
           echo "| Disk usage | ${disk} MiB |"
           echo ""
           echo "::notice::snapshot: apparent=${apparent}MiB disk=${disk}MiB"
+          echo "{\"name\":\"snapshot (disk)\",\"unit\":\"MiB\",\"value\":$disk}" >> /tmp/bench-results.jsonl
+
+      - name: Collect benchmark results
+        if: steps.kvm_check.outputs.available == 'true'
+        run: |
+          if [ -f /tmp/bench-results.jsonl ]; then
+            jq -s '.' /tmp/bench-results.jsonl > /tmp/bench-results.json
+          else
+            echo "[]" > /tmp/bench-results.json
+          fi
+          cat /tmp/bench-results.json
+
+      - name: Store benchmark results
+        if: steps.kvm_check.outputs.available == 'true'
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Linux Benchmarks
+          tool: customSmallerIsBetter
+          output-file-path: /tmp/bench-results.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          benchmark-data-dir-path: dev/bench/linux
+          auto-push: ${{ github.event_name == 'push' }}
+          comment-on-alert: true
+          alert-threshold: '130%'
 
   # ------------------------------------------------------------------
   # Performance + density on Windows
   # ------------------------------------------------------------------
   bench-windows:
     runs-on: windows-latest
-    needs: build-image
+    needs: [build-image, bench-linux]
     permissions:
-      contents: read
+      contents: write
       packages: read
+      pull-requests: write
     timeout-minutes: 30
     steps:
       - uses: actions/checkout@v4
@@ -339,6 +368,7 @@ jobs:
           Write-Host "| Runs   | $n |"
           Write-Host ""
           Write-Host "::notice::hello_world: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms"
+          @{name="hello_world (median)"; unit="ms"; value=$median} | ConvertTo-Json -Compress | Out-File -Append bench-results.jsonl -Encoding ascii
 
       - name: "Perf: pandas (10 runs)"
         shell: pwsh
@@ -374,6 +404,7 @@ jobs:
           Write-Host "| Runs   | $n |"
           Write-Host ""
           Write-Host "::notice::pandas: median=${median}ms avg=${avg}ms min=${min}ms max=${max}ms"
+          @{name="pandas (median)"; unit="ms"; value=$median} | ConvertTo-Json -Compress | Out-File -Append bench-results.jsonl -Encoding ascii
 
       - name: "Density: concurrent VMs (5 VMs)"
         shell: pwsh
@@ -407,6 +438,7 @@ jobs:
             Write-Host "| Total Private | ${totalPrivate} MB |"
             Write-Host ""
             Write-Host "::notice::density: ${count} VMs, per_vm=${perVM}MB private"
+            @{name="density (per VM)"; unit="MB"; value=$perVM} | ConvertTo-Json -Compress | Out-File -Append bench-results.jsonl -Encoding ascii
           } else {
             Write-Host "::warning::no VMs were alive long enough to measure"
           }
@@ -444,6 +476,31 @@ jobs:
           Write-Host "| Disk usage | ${diskMiB} MiB |"
           Write-Host ""
           Write-Host "::notice::snapshot: apparent=${apparentMiB}MiB disk=${diskMiB}MiB"
+          @{name="snapshot (disk)"; unit="MiB"; value=$diskMiB} | ConvertTo-Json -Compress | Out-File -Append bench-results.jsonl -Encoding ascii
+
+      - name: Collect benchmark results
+        shell: pwsh
+        run: |
+          if (Test-Path bench-results.jsonl) {
+            $lines = Get-Content bench-results.jsonl
+            $json = "[" + ($lines -join ",") + "]"
+            $json | Out-File bench-results.json -Encoding ascii
+          } else {
+            "[]" | Out-File bench-results.json -Encoding ascii
+          }
+          Get-Content bench-results.json
+
+      - name: Store benchmark results
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Windows Benchmarks
+          tool: customSmallerIsBetter
+          output-file-path: bench-results.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          benchmark-data-dir-path: dev/bench/windows
+          auto-push: ${{ github.event_name == 'push' }}
+          comment-on-alert: true
+          alert-threshold: '130%'
 
   all-checks-passed:
     if: always()