|
| 1 | +name: Benchmark ⏱️ |
| 2 | + |
| 3 | +env: |
| 4 | + NODE_OPTIONS: --max-old-space-size=6144 |
| 5 | + |
| 6 | +permissions: |
| 7 | + contents: read |
| 8 | + pull-requests: write |
| 9 | + |
| 10 | +on: |
| 11 | + push: |
| 12 | + branches: [main] |
| 13 | + paths: |
| 14 | + - "packages/react-server/**" |
| 15 | + - "packages/rsc/**" |
| 16 | + - "examples/benchmark/**" |
| 17 | + - ".github/workflows/benchmark.yml" |
| 18 | + pull_request: |
| 19 | + paths: |
| 20 | + - "packages/react-server/**" |
| 21 | + - "packages/rsc/**" |
| 22 | + - "examples/benchmark/**" |
| 23 | + - ".github/workflows/benchmark.yml" |
| 24 | + |
| 25 | +concurrency: |
| 26 | + group: benchmark-${{ github.event.number || github.sha }} |
| 27 | + cancel-in-progress: true |
| 28 | + |
| 29 | +jobs: |
| 30 | + benchmark: |
| 31 | + name: Run benchmarks ⏱️ |
| 32 | + runs-on: ubuntu-latest |
| 33 | + timeout-minutes: 30 |
| 34 | + steps: |
| 35 | + - name: Checkout |
| 36 | + uses: actions/checkout@v4 |
| 37 | + |
| 38 | + - uses: ./.github/workflows/actions/common-setup |
| 39 | + |
| 40 | + - name: Build benchmark example |
| 41 | + run: pnpm --filter @lazarv/react-server-example-benchmark build |
| 42 | + |
| 43 | + - name: Run benchmark |
| 44 | + working-directory: examples/benchmark |
| 45 | + run: node bench.mjs --save current |
| 46 | + |
| 47 | + - name: Upload benchmark results |
| 48 | + uses: actions/upload-artifact@v4 |
| 49 | + with: |
| 50 | + name: benchmark-results-${{ github.sha }} |
| 51 | + path: examples/benchmark/results-current.json |
| 52 | + retention-days: 90 |
| 53 | + |
| 54 | + # ── Main branch: save baseline to cache ────────────────────────────── |
| 55 | + - name: Copy results to baseline path |
| 56 | + if: github.ref == 'refs/heads/main' |
| 57 | + run: cp examples/benchmark/results-current.json examples/benchmark/benchmark-baseline.json |
| 58 | + |
| 59 | + - name: Save baseline to cache |
| 60 | + if: github.ref == 'refs/heads/main' |
| 61 | + uses: actions/cache/save@v4 |
| 62 | + with: |
| 63 | + path: examples/benchmark/benchmark-baseline.json |
| 64 | + key: benchmark-baseline-${{ github.sha }} |
| 65 | + |
| 66 | + # ── Pull request: compare against main baseline ────────────────────── |
| 67 | + - name: Restore baseline from main |
| 68 | + if: github.event_name == 'pull_request' |
| 69 | + id: baseline |
| 70 | + uses: actions/cache/restore@v4 |
| 71 | + with: |
| 72 | + path: examples/benchmark/benchmark-baseline.json |
| 73 | + key: benchmark-baseline-${{ github.event.pull_request.base.sha }} |
| 74 | + restore-keys: benchmark-baseline- |
| 75 | + |
| 76 | + - name: Generate comparison comment |
| 77 | + if: github.event_name == 'pull_request' |
| 78 | + working-directory: examples/benchmark |
| 79 | + env: |
| 80 | + HAS_BASELINE: ${{ steps.baseline.outputs.cache-matched-key != '' }} |
| 81 | + PR_SHA: ${{ github.event.pull_request.head.sha }} |
| 82 | + BASELINE_KEY: ${{ steps.baseline.outputs.cache-matched-key }} |
| 83 | + run: | |
| 84 | + node --input-type=module << 'SCRIPT' |
| 85 | + import { readFileSync, writeFileSync } from "node:fs"; |
| 86 | +
|
| 87 | + const prSha = process.env.PR_SHA?.slice(0, 7) || "unknown"; |
| 88 | + const hasBaseline = process.env.HAS_BASELINE === "true"; |
| 89 | + const baselineKey = process.env.BASELINE_KEY || ""; |
| 90 | + const baselineSha = baselineKey.replace("benchmark-baseline-", "").slice(0, 7); |
| 91 | +
|
| 92 | + const current = JSON.parse(readFileSync("results-current.json", "utf8")); |
| 93 | +
|
| 94 | + let baseline = null; |
| 95 | + if (hasBaseline) { |
| 96 | + try { |
| 97 | + baseline = JSON.parse(readFileSync("benchmark-baseline.json", "utf8")); |
| 98 | + } catch { |
| 99 | + // baseline file not found or invalid |
| 100 | + } |
| 101 | + } |
| 102 | +
|
| 103 | + const baselineMap = baseline |
| 104 | + ? new Map(baseline.results.map((r) => [r.name, r])) |
| 105 | + : null; |
| 106 | +
|
| 107 | + function fmtDelta(current, base, lowerIsBetter = false) { |
| 108 | + if (base == null || base === 0) return ""; |
| 109 | + const pct = ((current - base) / base) * 100; |
| 110 | + const sign = pct > 0 ? "+" : ""; |
| 111 | + const good = lowerIsBetter ? pct < -1 : pct > 1; |
| 112 | + const bad = lowerIsBetter ? pct > 1 : pct < -1; |
| 113 | + const icon = good ? "\u{1f7e2}" : bad ? "\u{1f534}" : "\u{26aa}"; |
| 114 | + return ` ${icon} ${sign}${pct.toFixed(1)}%`; |
| 115 | + } |
| 116 | +
|
| 117 | + const lines = []; |
| 118 | + lines.push("<!-- benchmark-results -->"); |
| 119 | + lines.push("## \u26a1 Benchmark Results"); |
| 120 | + lines.push(""); |
| 121 | +
|
| 122 | + if (baselineMap) { |
| 123 | + lines.push(`| | **PR** \`${prSha}\` | **main** \`${baselineSha}\` |`); |
| 124 | + lines.push(`|---|---|---|`); |
| 125 | + lines.push(`| **Config** | ${current.config.connections} connections, ${current.config.duration}s/test | ${baseline.config.connections} connections, ${baseline.config.duration}s/test |`); |
| 126 | + } else { |
| 127 | + lines.push(`**Commit:** \`${prSha}\` | **No baseline available** (will be created when this PR's target is next pushed to main)`); |
| 128 | + } |
| 129 | +
|
| 130 | + lines.push(""); |
| 131 | +
|
| 132 | + if (baselineMap) { |
| 133 | + lines.push("| Benchmark | Req/s | vs main | Avg Latency | vs main | P99 Latency | Throughput |"); |
| 134 | + lines.push("|:----------|------:|--------:|------------:|--------:|------------:|-----------:|"); |
| 135 | +
|
| 136 | + for (const r of current.results) { |
| 137 | + const b = baselineMap.get(r.name); |
| 138 | + const reqDelta = fmtDelta(r.reqSec, b?.reqSec); |
| 139 | + const latDelta = fmtDelta(r.latencyAvg, b?.latencyAvg, true); |
| 140 | + lines.push( |
| 141 | + `| **${r.name}** | ${r.reqSec.toFixed(0)} | ${reqDelta} | ${r.latencyAvg} ms | ${latDelta} | ${r.latencyP99} ms | ${r.throughputMB} MB/s |` |
| 142 | + ); |
| 143 | + } |
| 144 | + } else { |
| 145 | + lines.push("| Benchmark | Req/s | Avg Latency | P50 | P99 | Throughput |"); |
| 146 | + lines.push("|:----------|------:|------------:|----:|----:|-----------:|"); |
| 147 | +
|
| 148 | + for (const r of current.results) { |
| 149 | + lines.push( |
| 150 | + `| **${r.name}** | ${r.reqSec.toFixed(0)} | ${r.latencyAvg} ms | ${r.latencyP50} ms | ${r.latencyP99} ms | ${r.throughputMB} MB/s |` |
| 151 | + ); |
| 152 | + } |
| 153 | + } |
| 154 | +
|
| 155 | + lines.push(""); |
| 156 | + lines.push("<details><summary>Legend</summary>"); |
| 157 | + lines.push(""); |
| 158 | + lines.push("\u{1f7e2} > 1% improvement | \u{1f534} > 1% regression | \u{26aa} within noise margin"); |
| 159 | + lines.push(""); |
| 160 | + lines.push("Benchmarks run on GitHub Actions runners (shared infrastructure) — expect ~5% variance between runs. Consistent directional changes across multiple routes are more meaningful than any single number."); |
| 161 | + lines.push(""); |
| 162 | + lines.push("</details>"); |
| 163 | +
|
| 164 | + writeFileSync("comment.md", lines.join("\n") + "\n"); |
| 165 | + console.log("Generated comment.md"); |
| 166 | + SCRIPT |
| 167 | +
|
| 168 | + - name: Post or update PR comment |
| 169 | + if: github.event_name == 'pull_request' |
| 170 | + env: |
| 171 | + GH_TOKEN: ${{ github.token }} |
| 172 | + run: | |
| 173 | + PR_NUMBER=${{ github.event.pull_request.number }} |
| 174 | + REPO=${{ github.repository }} |
| 175 | +
|
| 176 | + # Find existing benchmark comment by HTML marker |
| 177 | + COMMENT_ID=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ |
| 178 | + --paginate --jq '.[] | select(.body | contains("<!-- benchmark-results -->")) | .id' \ |
| 179 | + | head -1) |
| 180 | +
|
| 181 | + if [ -n "$COMMENT_ID" ]; then |
| 182 | + echo "Updating existing comment ${COMMENT_ID}" |
| 183 | + gh api "repos/${REPO}/issues/comments/${COMMENT_ID}" \ |
| 184 | + -X PATCH \ |
| 185 | + -F "body=@examples/benchmark/comment.md" |
| 186 | + else |
| 187 | + echo "Creating new comment" |
| 188 | + gh pr comment "$PR_NUMBER" --body-file examples/benchmark/comment.md |
| 189 | + fi |
0 commit comments