Benchmark PR #688
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark PR | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pr: | |
| description: 'PR number' | |
| required: true | |
| framework: | |
| description: 'Framework to benchmark' | |
| required: true | |
| profile: | |
| description: 'Profile (e.g. baseline, baseline-h2, leave empty for all)' | |
| required: false | |
| default: '' | |
| save: | |
| description: 'Save results (true/false)' | |
| required: false | |
| default: '' | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| concurrency: | |
| group: benchmark | |
| cancel-in-progress: false | |
| jobs: | |
| runner-busy: | |
| if: vars.RUNNER_LOCAL == 'true' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Post runner-busy notice | |
| run: | | |
| gh pr comment "${{ inputs.pr }}" \ | |
| --repo "${{ github.repository }}" \ | |
| --body "⏸️ Runner is currently performing local benchmark runs and is disabled for GitHub Actions, please try later or check our Discord announcements on runner state for more info." | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| benchmark: | |
| if: vars.RUNNER_LOCAL != 'true' | |
| runs-on: self-hosted | |
| environment: runner | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| ref: refs/pull/${{ inputs.pr }}/head | |
| fetch-depth: 0 | |
| - name: Validate inputs | |
| run: | | |
| if [ -z "${{ inputs.framework }}" ]; then | |
| echo "Error: framework is required" | |
| exit 1 | |
| fi | |
| if [ ! -d "frameworks/${{ inputs.framework }}" ]; then | |
| echo "Error: framework '${{ inputs.framework }}' not found" | |
| exit 1 | |
| fi | |
| # When --save is requested, merge origin/main into the PR branch first. | |
| # The bot commits its results back to the PR head; on an old branch the | |
| # site/data/*.json files have drifted vs. main and the bot commit | |
| # immediately makes the PR CONFLICTING. Merging main here lands the | |
| # bench commit on a base that's already in sync, so the PR stays | |
| # MERGEABLE. Skipped for non-save runs since they don't push back. | |
| - name: Merge main into PR branch | |
| if: inputs.save == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git fetch origin main --depth=1 | |
| if ! git merge --no-edit origin/main; then | |
| git merge --abort 2>/dev/null || true | |
| gh pr comment "${{ inputs.pr }}" \ | |
| --repo "${{ github.repository }}" \ | |
| --body "⚠️ \`/benchmark --save\` aborted: \`main\` has diverged and cannot be auto-merged into this branch. Please merge or rebase \`main\` manually, push, and re-run \`/benchmark --save\`." | |
| exit 1 | |
| fi | |
| - name: Clean previous containers, results, and temp data | |
| run: | | |
| docker ps -aq --filter "name=httparena-" | xargs -r docker rm -f 2>/dev/null || true | |
| rm -rf results/ | |
| rm -rf /tmp/pr_site_data /tmp/main_site_data /tmp/bench_site_data /tmp/bench_comparison.md /tmp/bench_body.txt /tmp/bench_full.txt /tmp/bench_table.txt | |
| - name: Fetch main branch data for comparison | |
| run: | | |
| # Save PR's site/data aside, get main's for comparison | |
| cp -r site/data /tmp/pr_site_data | |
| git fetch origin main --depth=1 | |
| git checkout origin/main -- site/data/ 2>/dev/null || true | |
| # Keep main's site/data for comparison, restore PR code | |
| cp -r site/data /tmp/main_site_data | |
| cp -r /tmp/pr_site_data/* site/data/ | |
| - name: Run benchmarks | |
| id: bench | |
| run: | | |
| log=$(mktemp) | |
| # Always save results to disk for comparison; only commit if --save requested | |
| ./scripts/benchmark.sh "${{ inputs.framework }}" "${{ inputs.profile }}" --save 2>&1 | tee "$log" | |
| echo "log_file=$log" >> "$GITHUB_OUTPUT" | |
| - name: Compare with main | |
| id: compare | |
| run: | | |
| # Preserve benchmark results before overwriting site/data for comparison | |
| cp -r site/data /tmp/bench_site_data 2>/dev/null || true | |
| # Use main's site/data for comparison | |
| if [ -d /tmp/main_site_data ]; then | |
| cp -r /tmp/main_site_data/* site/data/ 2>/dev/null || true | |
| fi | |
| comparison=$(./scripts/compare.sh "${{ inputs.framework }}" "${{ inputs.profile }}" 2>/dev/null || echo "") | |
| echo "$comparison" > /tmp/bench_comparison.md | |
| # Restore benchmark site/data (not original PR data) | |
| if [ -d /tmp/bench_site_data ]; then | |
| cp -r /tmp/bench_site_data/* site/data/ 2>/dev/null || true | |
| fi | |
| - name: Commit saved results | |
| if: inputs.save == 'true' | |
| run: | | |
| PR_DATA=$(gh api "/repos/${{ github.repository }}/pulls/${{ inputs.pr }}" --jq '.head.ref + " " + .head.repo.full_name') | |
| PR_BRANCH=$(echo "$PR_DATA" | awk '{print $1}') | |
| PR_REPO=$(echo "$PR_DATA" | awk '{print $2}') | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| find site/static/logs -name "${{ inputs.framework }}.log" -exec git add -f {} + | |
| git add -f site/data/frameworks.json site/data/current.json 2>/dev/null || true | |
| # Only add leaderboard files for the profiles that were actually benchmarked | |
| for f in results/*/*/${{ inputs.framework }}.json; do | |
| [ -f "$f" ] || continue | |
| dir=$(dirname "$f") | |
| conns=$(basename "$dir") | |
| prof=$(basename "$(dirname "$dir")") | |
| git add -f "site/data/${prof}-${conns}.json" 2>/dev/null || true | |
| done | |
| if git diff --cached --quiet; then | |
| echo "No results to commit" | |
| else | |
| git commit -m "Benchmark results: ${{ inputs.framework }} ${{ inputs.profile }}" | |
| git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${PR_REPO}.git" | |
| git push origin HEAD:"${PR_BRANCH}" || echo "Warning: could not push to fork (permissions)" | |
| fi | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Post results to PR | |
| if: always() | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| log="${{ steps.bench.outputs.log_file }}" | |
| profile="${{ inputs.profile }}" | |
| profile_label="${profile:-all tests}" | |
| fw="${{ inputs.framework }}" | |
| # Build markdown table with results and deltas | |
| python3 -c ' | |
| import sys, re, os | |
| log_file, fw, comparison_file = sys.argv[1], sys.argv[2], sys.argv[3] | |
| # Parse deltas from compare.sh output | |
| deltas = {} | |
| if os.path.exists(comparison_file): | |
| with open(comparison_file) as f: | |
| comp = f.read() | |
| cur_profile = None | |
| conn_cols = [] | |
| for line in comp.splitlines(): | |
| m = re.match(r"^### (.+)", line) | |
| if m: | |
| cur_profile = m.group(1).strip() | |
| conn_cols = [] | |
| continue | |
| if cur_profile and line.startswith("| Metric"): | |
| conn_cols = re.findall(r"(\d+)c", line) | |
| continue | |
| if cur_profile and conn_cols and line.startswith("| **RPS**"): | |
| cells = [c.strip() for c in line.split("|")[1:] if c.strip()] | |
| for i, conns in enumerate(conn_cols): | |
| idx = i * 2 + 2 | |
| if idx < len(cells): | |
| deltas.setdefault(cur_profile + "/" + conns, {})["rps"] = cells[idx] | |
| if cur_profile and conn_cols and line.startswith("| **Memory**"): | |
| cells = [c.strip() for c in line.split("|")[1:] if c.strip()] | |
| for i, conns in enumerate(conn_cols): | |
| idx = i * 2 + 2 | |
| if idx < len(cells): | |
| deltas.setdefault(cur_profile + "/" + conns, {})["mem"] = cells[idx] | |
| # Parse log for results | |
| rows = [] | |
| cur_profile = None | |
| cur_conns = None | |
| with open(log_file) as f: | |
| for line in f: | |
| line = line.rstrip() | |
| m = re.match(r"^=== " + re.escape(fw) + r" / (.+) / (\d+)c .* ===$", line) | |
| if m: | |
| cur_profile = m.group(1) | |
| cur_conns = m.group(2) | |
| continue | |
| if cur_profile and re.match(r"^=== Best:", line): | |
| m2 = re.match(r"^=== Best: (\d+) req/s \(CPU: ([\d.]+)%, Mem: ([^\)]+)\)", line) | |
| if m2: | |
| rps = int(m2.group(1)) | |
| cpu = m2.group(2) | |
| mem = m2.group(3) | |
| key = cur_profile + "/" + cur_conns | |
| d = deltas.get(key, {}) | |
| rps_str = "{:,}".format(rps) | |
| d_rps = d.get("rps", "") | |
| d_mem = d.get("mem", "") | |
| rows.append((cur_profile, cur_conns, rps_str, cpu + "%", mem, d_rps, d_mem)) | |
| cur_profile = None | |
| # Build table | |
| with open("/tmp/bench_body.txt", "w") as out: | |
| out.write("| Test | Conn | RPS | CPU | Mem | \u0394 RPS | \u0394 Mem |\n") | |
| out.write("|------|------|-----|-----|-----|-------|-------|\n") | |
| for prof, conns, rps, cpu, mem, d_rps, d_mem in rows: | |
| out.write("| {} | {} | {} | {} | {} | {} | {} |\n".format( | |
| prof, conns, rps, cpu, mem, d_rps, d_mem)) | |
| ' "$log" "$fw" /tmp/bench_comparison.md | |
| body="## Benchmark Results"$'\n\n' | |
| body+="**Framework:** \`${fw}\` | **Test:** \`${profile_label}\`"$'\n\n' | |
| body+=$(cat /tmp/bench_body.txt 2>/dev/null || echo "No results captured") | |
| body+=$'\n\n' | |
| body+="<details><summary>Full log</summary>"$'\n\n```\n' | |
| tail -200 "$log" >> /tmp/bench_full.txt 2>/dev/null || true | |
| body+=$(cat /tmp/bench_full.txt 2>/dev/null || echo "No log") | |
| body+=$'\n```\n</details>' | |
| gh pr comment "${{ inputs.pr }}" --repo "${{ github.repository }}" --body "$body" | |
| rm -f "$log" /tmp/bench_table.txt /tmp/bench_full.txt /tmp/bench_comparison.md /tmp/pr_site_data /tmp/main_site_data 2>/dev/null || true |