Benchmarks #15
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Instruction-count (Callgrind) perf gate vs a committed benchmarks/baseline.json; no CodSpeed account needed | |
| # (compare_baseline.py parses the raw callgrind dumps). Rationale in benchmarks/README.md and benchmarks/PLAN.md. | |
| # workflow_dispatch input `regen`: false compares (report only), true writes baseline.json and uploads it. | |
| name: Benchmarks | |
| on: | |
| schedule: | |
| - cron: "0 3 * * *" # nightly at 03:00 UTC | |
| workflow_dispatch: | |
| inputs: | |
| regen: | |
| description: "Regenerate benchmarks/baseline.json (upload as artifact) instead of comparing" | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: codspeed-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmarks: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 # ~25 min sweep (BENCH_SCALE=10) + ~10 min cold build, plus margin | |
| permissions: | |
| contents: read | |
| env: | |
| PYTHONHASHSEED: "0" # deterministic instruction counts | |
| CODSPEED_ENV: "1" # enable pytest-codspeed callgrind hooks | |
| # shrink O(rows) benches to fit the timeout; a baseline only compares to a run at the SAME scale (meta.bench_scale) | |
| BENCH_SCALE: "10" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 # setuptools_scm needs history | |
| - name: Resolve DuckDB submodule SHA | |
| id: duckdb_sha | |
| # sccache key + compare_baseline.py engine-bump guard | |
| run: echo "sha=$(git rev-parse HEAD:external/duckdb)" >> "$GITHUB_OUTPUT" | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.13" | |
| - name: Install valgrind | |
| run: sudo apt-get update && sudo apt-get install -y valgrind | |
| - name: Cache sccache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/sccache | |
| key: sccache-codspeed-${{ steps.duckdb_sha.outputs.sha }} | |
| restore-keys: sccache-codspeed- | |
| - name: Install sccache | |
| run: | | |
| curl -fsSL https://github.com/mozilla/sccache/releases/download/v0.8.2/sccache-v0.8.2-x86_64-unknown-linux-musl.tar.gz \ | |
| | tar -xz --strip-components=1 -C /usr/local/bin sccache-v0.8.2-x86_64-unknown-linux-musl/sccache | |
| - name: Build the extension (release) + pinned benchmark deps | |
| env: | |
| CMAKE_C_COMPILER_LAUNCHER: sccache | |
| CMAKE_CXX_COMPILER_LAUNCHER: sccache | |
| run: | | |
| uv sync --only-group build --no-install-project -p 3.13 | |
| # pins BEFORE the build: numpy>=2.0 is a build-system.requires (C API headers), absent under | |
| # --no-build-isolation and not in the `build` group, so find_package(NumPy) fails otherwise. | |
| uv pip install -r benchmarks/requirements-bench.txt | |
| # uv pip is additive; uv sync would prune numpy back out before the build. release, no dev group. | |
| uv pip install --no-build-isolation --no-deps --reinstall -C cmake.build-type=Release . | |
| - name: Collect gate node-ids | |
| # regen uses these to classify each benchmark. Block scalar so `\` is a real shell continuation; a plain | |
| # scalar folds it to a literal `\ `, mangling pytest args into an empty (all-informational) gate list. | |
| run: | | |
| uv run --no-sync pytest benchmarks/ -m gate --collect-only -q -o addopts= -p no:cacheprovider \ | |
| | grep '::' > gate_list.txt || true | |
| # an empty gate list yields a 0-gate baseline that never catches a regression | |
| test -s gate_list.txt || { echo "ERROR: gate_list.txt empty"; exit 1; } | |
| echo "collected $(wc -l < gate_list.txt) gate node-ids" | |
| - name: Run benchmarks under Callgrind (per-benchmark instruction counts) | |
| # one sweep; concurrency module excluded (Callgrind serializes threads). | |
| run: | | |
| mkdir -p profiles | |
| # run the venv python directly, NOT `uv run`: valgrind without --trace-children only instruments uv, | |
| # leaving pytest un-instrumented (native speed, zero dumps). | |
| CODSPEED_PROFILE_FOLDER="$PWD/profiles" valgrind --tool=callgrind --instr-atstart=no \ | |
| --callgrind-out-file="$PWD/profiles/cg.%p.%n" \ | |
| "$VIRTUAL_ENV/bin/python" -m pytest benchmarks/ \ | |
| --ignore=benchmarks/test_concurrency_perf.py \ | |
| -m "gate or informational" --codspeed -o addopts= -p no:cacheprovider | |
| - name: Compare against committed baseline (report-only) | |
| if: ${{ !inputs.regen }} | |
| # report only, never fails; add --enforce once trusted | |
| run: | | |
| uv run --no-sync python benchmarks/compare_baseline.py compare \ | |
| --profiles profiles --baseline benchmarks/baseline.json \ | |
| --submodule-sha "${{ steps.duckdb_sha.outputs.sha }}" \ | |
| --pins benchmarks/requirements-bench.txt | |
| - name: Regenerate baseline (upload artifact to commit deliberately) | |
| if: ${{ inputs.regen }} | |
| run: | | |
| uv run --no-sync python benchmarks/compare_baseline.py regen \ | |
| --profiles profiles --out benchmarks/baseline.json --gate-list gate_list.txt \ | |
| --git-commit "${{ github.sha }}" --submodule-sha "${{ steps.duckdb_sha.outputs.sha }}" \ | |
| --pins benchmarks/requirements-bench.txt | |
| - name: Upload regenerated baseline | |
| if: ${{ inputs.regen }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: baseline-update | |
| path: benchmarks/baseline.json |