From 75b064aacc3cf5b360994924e876e8096c216f64 Mon Sep 17 00:00:00 2001 From: "Ilya (Marshal)" Date: Thu, 20 Nov 2025 16:53:44 +0100 Subject: [PATCH 1/3] Add `pytest-benchmark` GHA workflow --- .github/workflows/benchmark.yml | 69 +++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/benchmark.yml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..4328dff --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,69 @@ +name: Benchmark +on: [ pull_request, push, workflow_dispatch ] + +permissions: + contents: read + +env: + UV_FROZEN: true + UV_PYTHON: 3.14 + RUST_VERSION: "1.90.0" + +jobs: + benchmark: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository. + uses: actions/checkout@v4 + + - name: Install UV. + uses: astral-sh/setup-uv@v6 + + - name: Install dependencies. + run: uv sync --group testing + + - name: Compile. + run: uv pip install -v -e . + env: + RUST_BACKTRACE: 1 + + - name: Run Benchmarks. + run: uv run pytest . --benchmark-enable --benchmark-json=output.json + + - name: Upload benchmark artifact + uses: actions/upload-artifact@v4 + with: + name: benchmark-output + path: output.json + + - name: Append benchmark table to summary + if: always() + run: | + if [ -f output.json ]; then + echo '## Pytest Benchmark Results' >> "$GITHUB_STEP_SUMMARY" + echo '' >> "$GITHUB_STEP_SUMMARY" + jq -r ' + def fmt_ns(v): + if v >= 1e9 then + (100 * (v / 1e9) | round / 100 | tostring) + " s" + elif v >= 1e6 then + (100 * (v / 1e6) | round / 100 | tostring) + " ms" + elif v >= 1e3 then + (100 * (v / 1e3) | round / 100 | tostring) + " µs" + else + (100 * v | round / 100 | tostring) + " ns" + end; + + .benchmarks + | ("| Benchmark | Mean | Min | Max | Rounds | Iterations |", "|---|---:|---:|---:|---:|---:|") , + (.[] | "| " + .name + + " | " + fmt_ns(.stats.mean * 1e9) + + " | " + fmt_ns(.stats.min * 1e9) + + " | " + fmt_ns(.stats.max * 1e9) + + " | " + (.stats.rounds | tostring) + + " | " + (.stats.iterations | tostring) + " |") + ' output.json >> "$GITHUB_STEP_SUMMARY" + else + echo 'No benchmark output.json found.' >> "$GITHUB_STEP_SUMMARY" + fi From 90c4a91a174516fb84a12db4c308a0c040f43316 Mon Sep 17 00:00:00 2001 From: "Ilya (Marshal)" Date: Thu, 20 Nov 2025 20:29:43 +0100 Subject: [PATCH 2/3] use github-action-benchmark --- .github/workflows/benchmark.yml | 58 +++++++++++++-------------------- 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4328dff..d49a42b 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,5 +1,11 @@ name: Benchmark -on: [ pull_request, push, workflow_dispatch ] + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: permissions: contents: read @@ -29,41 +35,21 @@ jobs: RUST_BACKTRACE: 1 - name: Run Benchmarks. - run: uv run pytest . --benchmark-enable --benchmark-json=output.json + run: uv run pytest . -m benchmark_main --benchmark-enable --benchmark-json=output.json - - name: Upload benchmark artifact - uses: actions/upload-artifact@v4 + - name: Download previous benchmark data. + uses: actions/cache@v4 with: - name: benchmark-output - path: output.json - - - name: Append benchmark table to summary - if: always() - run: | - if [ -f output.json ]; then - echo '## Pytest Benchmark Results' >> "$GITHUB_STEP_SUMMARY" - echo '' >> "$GITHUB_STEP_SUMMARY" - jq -r ' - def fmt_ns(v): - if v >= 1e9 then - (100 * (v / 1e9) | round / 100 | tostring) + " s" - elif v >= 1e6 then - (100 * (v / 1e6) | round / 100 | tostring) + " ms" - elif v >= 1e3 then - (100 * (v / 1e3) | round / 100 | tostring) + " µs" - else - (100 * v | round / 100 | tostring) + " ns" - end; + path: ./benchmark-data + key: benchmark-${{ github.base_ref || github.ref_name }} - .benchmarks - | ("| Benchmark | Mean | Min | Max | Rounds | Iterations |", "|---|---:|---:|---:|---:|---:|") , - (.[] | "| " + .name + - " | " + fmt_ns(.stats.mean * 1e9) + - " | " + fmt_ns(.stats.min * 1e9) + - " | " + fmt_ns(.stats.max * 1e9) + - " | " + (.stats.rounds | tostring) + - " | " + (.stats.iterations | tostring) + " |") - ' output.json >> "$GITHUB_STEP_SUMMARY" - else - echo 'No benchmark output.json found.' >> "$GITHUB_STEP_SUMMARY" - fi + - name: Benchmark results. + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'pytest' + output-file-path: output.json + external-data-json-path: ./benchmark-data/output.json + summary-always: true + fail-on-alert: true + skip-fetch-gh-pages: true + save-data-file: ${{ github.event_name != 'pull_request' }} From 0ea357234dff3aeaefe990feb51639134652cac7 Mon Sep 17 00:00:00 2001 From: "Ilya (Marshal)" Date: Thu, 20 Nov 2025 20:32:47 +0100 Subject: [PATCH 3/3] fix pytest group --- pyproject.toml | 3 +++ pytests/test_dag_cbor.py | 3 +++ pytests/test_decode_car.py | 1 + 3 files changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 4a81447..613221f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,9 @@ all = [ ] [tool.pytest.ini_options] +markers = [ + 'benchmark_main: marks tests as main benchmarks to run selectively' +] addopts = [ '--benchmark-columns', 'min,mean,stddev,outliers,rounds,iterations', '--benchmark-disable', # use --benchmark-enable diff --git a/pytests/test_dag_cbor.py b/pytests/test_dag_cbor.py index fee7708..c48c2c6 100644 --- a/pytests/test_dag_cbor.py +++ b/pytests/test_dag_cbor.py @@ -101,11 +101,13 @@ def test_dag_cbor_decode(benchmark, data) -> None: _dag_cbor_roundtrip(benchmark, data) +@pytest.mark.benchmark_main @pytest.mark.parametrize('data', load_json_data_fixtures(_REAL_DATA_DIR), ids=lambda data: data[0]) def test_dag_cbor_encode_real_data(benchmark, data) -> None: _dag_cbor_encode(benchmark, data) +@pytest.mark.benchmark_main @pytest.mark.parametrize('data', load_json_data_fixtures(_REAL_DATA_DIR), ids=lambda data: data[0]) def test_dag_cbor_decode_real_data(benchmark, data) -> None: _dag_cbor_roundtrip(benchmark, data) @@ -116,6 +118,7 @@ def test_dag_cbor_decode_fixtures(benchmark, data) -> None: _dag_cbor_decode(benchmark, data) +@pytest.mark.benchmark_main def test_dag_cbor_decode_torture_cids(benchmark) -> None: dag_cbor = open(_TORTURE_CIDS_DAG_CBOR_PATH, 'rb').read() benchmark(libipld.decode_dag_cbor, dag_cbor) diff --git a/pytests/test_decode_car.py b/pytests/test_decode_car.py index 76ea661..15d40c5 100644 --- a/pytests/test_decode_car.py +++ b/pytests/test_decode_car.py @@ -14,6 +14,7 @@ def car() -> bytes: return load_car_fixture(_DID, _REPO_CAR_PATH) +@pytest.mark.benchmark_main def test_decode_car(benchmark, car) -> None: header, blocks = benchmark(libipld.decode_car, car)