feat: switch continuous benchmarks to gungraun #913
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| types: [opened, synchronize] | |
| paths-ignore: | |
| - "**/*.md" | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - "**/*.md" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| benchmark: | |
| name: Benchmark (Gungraun) | |
| runs-on: ubuntu-latest | |
| env: | |
| GUNGRAUN_HOME: ${{ github.workspace }}/target/gungraun | |
| GUNGRAUN_VALGRIND_BIN: /usr/bin/valgrind | |
| GUNGRAUN_HAS_BASELINE: "0" | |
| steps: | |
| - name: Checkout Branch | |
| uses: taiki-e/checkout-action@7d1e50e93dc4fb3bba58f85018fadf77898aee8b # v1.4.2 | |
| - uses: pnpm/action-setup@8912a9102ac27614460f54aedde9e1e7f9aec20d # v6.0.5 | |
| with: | |
| version: 10.33.2 | |
| - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 | |
| with: | |
| node-version-file: .node-version | |
| cache: pnpm | |
| - name: Setup benchmark data | |
| run: cd benches && pnpm install --ignore-workspace | |
| - uses: actions-rust-lang/setup-rust-toolchain@46268bd060767258de96ed93c1251119784f2ab6 # v1.16.1 | |
| with: | |
| cache-key: benchmark-gungraun | |
| cache-save-if: ${{ github.ref_name == 'main' }} | |
| - uses: ./.github/actions/pnpm | |
| - name: Install valgrind | |
| run: sudo apt-get update && sudo apt-get install -y valgrind | |
| - uses: taiki-e/install-action@055f5df8c3f65ea01cd41e9dc855becd88953486 # v2.75.18 | |
| with: | |
| tool: gungraun-runner@0.18.2 | |
| - name: Run gungraun baseline benchmark (PR base) | |
| if: github.event_name == 'pull_request' | |
| timeout-minutes: 45 | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| RUSTFLAGS: "-C debuginfo=1 -C strip=none -g" | |
| run: | | |
| set -euo pipefail | |
| WORKTREE_DIR=/tmp/gungraun-pr-base | |
| cleanup() { | |
| if [ -d "${WORKTREE_DIR}" ]; then | |
| git worktree remove "${WORKTREE_DIR}" --force || true | |
| fi | |
| } | |
| trap cleanup EXIT | |
| echo "GUNGRAUN_HAS_BASELINE=0" >> "$GITHUB_ENV" | |
| git fetch --no-tags --depth=1 origin "${BASE_SHA}" | |
| git worktree add "${WORKTREE_DIR}" "${BASE_SHA}" | |
| pushd "${WORKTREE_DIR}" | |
| pnpm install --dir benches --ignore-workspace | |
| if cargo bench --bench resolver_gungraun --no-run >/dev/null 2>&1; then | |
| cargo bench --bench resolver_gungraun -- \ | |
| --home="${GUNGRAUN_HOME}" \ | |
| --save-baseline=pr_base \ | |
| --save-summary=pretty-json | |
| echo "GUNGRAUN_HAS_BASELINE=1" >> "$GITHUB_ENV" | |
| else | |
| echo "Base commit ${BASE_SHA} has no resolver_gungraun bench target; skipping baseline compare." | |
| fi | |
| popd | |
| - name: Run gungraun benchmark with baseline compare (PR head) | |
| if: github.event_name == 'pull_request' | |
| timeout-minutes: 45 | |
| env: | |
| RUSTFLAGS: "-C debuginfo=1 -C strip=none -g" | |
| run: | | |
| set -euo pipefail | |
| mkdir -p target/gungraun | |
| if [ "${GUNGRAUN_HAS_BASELINE:-0}" = "1" ]; then | |
| cargo bench --bench resolver_gungraun -- \ | |
| --home="${GUNGRAUN_HOME}" \ | |
| --baseline=pr_base \ | |
| --save-summary=pretty-json \ | |
| --output-format=json \ | |
| > target/gungraun/benchmark-diff.ndjson | |
| else | |
| cargo bench --bench resolver_gungraun -- \ | |
| --home="${GUNGRAUN_HOME}" \ | |
| --save-summary=pretty-json | |
| fi | |
| - name: Run gungraun benchmark (push/manual) | |
| if: github.event_name != 'pull_request' | |
| timeout-minutes: 45 | |
| env: | |
| RUSTFLAGS: "-C debuginfo=1 -C strip=none -g" | |
| run: | | |
| set -euxo pipefail | |
| cargo bench --bench resolver_gungraun -- \ | |
| --home="${GUNGRAUN_HOME}" \ | |
| --save-summary=pretty-json | |
| - name: Publish benchmark and commit diff summary | |
| if: always() | |
| env: | |
| BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| HEAD_SHA: ${{ github.sha }} | |
| run: | | |
| set -euo pipefail | |
| echo "## Gungraun Benchmark" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| echo "- Base commit: \`${BASE_SHA}\`" >> "$GITHUB_STEP_SUMMARY" | |
| echo "- Head commit: \`${HEAD_SHA}\`" >> "$GITHUB_STEP_SUMMARY" | |
| echo "- Baseline compare: $([ "${GUNGRAUN_HAS_BASELINE:-0}" = "1" ] && echo "enabled" || echo "skipped (base has no resolver_gungraun target)")" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "### Benchmark Diff (Estimated Cycles)" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "| Benchmark | Base | Head | Diff |" >> "$GITHUB_STEP_SUMMARY" | |
| echo "|---|---:|---:|---:|" >> "$GITHUB_STEP_SUMMARY" | |
| shopt -s nullglob | |
| summary_files=(target/gungraun/rspack_resolver/resolver_gungraun/resolver_group/*/summary.json) | |
| if [ "${#summary_files[@]}" -eq 0 ]; then | |
| echo "| (no benchmark summary found) | - | - | - |" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| for f in "${summary_files[@]}"; do | |
| if [ "${GUNGRAUN_HAS_BASELINE:-0}" = "1" ]; then | |
| jq -r ' | |
| def n: (.Int // .Float // 0); | |
| def m($k): .profiles[0].summaries.total.summary.Callgrind[$k]; | |
| "| \(.id) | \((m("EstimatedCycles").metrics.Both[1] | n)) | \((m("EstimatedCycles").metrics.Both[0] | n)) | \((m("EstimatedCycles").diffs.diff_pct | tonumber) | if . > 0 then "+" + (.|tostring) else (.|tostring) end)% |" | |
| ' "$f" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| jq -r ' | |
| def n: (.Int // .Float // 0); | |
| def m($k): .profiles[0].summaries.total.summary.Callgrind[$k]; | |
| "| \(.id) | - | \((m("EstimatedCycles").metrics.Both[0] | n)) | - |" | |
| ' "$f" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| done | |
| fi | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "### Code Diff (Base..Head)" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo '```text' >> "$GITHUB_STEP_SUMMARY" | |
| git diff --stat --no-color "${BASE_SHA}..${HEAD_SHA}" >> "$GITHUB_STEP_SUMMARY" | |
| echo '```' >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "- Commit: \`${HEAD_SHA}\`" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| if [ -d target/gungraun ]; then | |
| flamegraph_count="$(find target/gungraun -name '*.flamegraph*.svg' | wc -l | tr -d ' ')" | |
| echo "Flamegraph SVG files: ${flamegraph_count}" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "Flamegraph SVG files: 0 (target/gungraun not generated)" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Upload gungraun result files (including flamegraphs) | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: gungraun-target | |
| path: target/gungraun | |
| if-no-files-found: ignore |