diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml new file mode 100644 index 00000000..9a4c9332 --- /dev/null +++ b/.github/workflows/benchmarks.yaml @@ -0,0 +1,80 @@ +name: Benchmarks + +on: + pull_request: + types: + - opened + - synchronize + +jobs: + lint: + name: Benchmark tests + runs-on: ubuntu-latest + strategy: + matrix: + python_version: [3.12] + steps: + - name: Checkout branch + uses: actions/checkout@v4 + with: + path: pr + + - name: Checkout main + uses: actions/checkout@v4 + with: + ref: main + path: main + + - name: Install python + uses: actions/setup-python@v5 + with: + python-version: ${{matrix.python_version}} + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + cache-dependency-glob: "main/uv.lock" + + - name: Setup benchmarks + run: | + echo "BASE_SHA=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-8)" >> $GITHUB_ENV + echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV + echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV + + - name: Run benchmarks on PR + working-directory: ./pr + run: | + uv sync --group test + uv run pytest --benchmark-only --benchmark-save=pr + + - name: Run benchmarks on main + working-directory: ./main + continue-on-error: true + run: | + uv sync --group test + uv run pytest --benchmark-only --benchmark-save=base + + - name: Compare results + continue-on-error: false + run: | + uvx pytest-benchmark compare **/.benchmarks/**/*.json | tee cmp_results + + echo 'Benchmark comparison for [`${{ env.BASE_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.base.sha }}) (base) vs [`${{ env.HEAD_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.head.sha }}) (PR)' >> pr_comment + echo '```' >> pr_comment + cat cmp_results >> pr_comment + echo '```' >> pr_comment + cat pr_comment > ${{ env.PR_COMMENT }} + + - name: Comment on PR + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: require('fs').readFileSync('${{ env.PR_COMMENT }}').toString() + }); + diff --git a/.gitignore b/.gitignore index e96b0844..8cba4ba3 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,9 @@ coverage.json .pytest_cache/ cover/ +# Benchmarking +.benchmarks/ + # Translations *.mo *.pot diff --git a/pyproject.toml b/pyproject.toml index 39214324..5328d5f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ test = [ "optuna>=3.0,<4", "pytest>=8.3,<9", "pytest-asyncio>=1.0,<2", + "pytest-benchmark>=5.1.0", "pytest-cases>=3.8,<4", "pytest-env>=1.1,<2", "pytest-rerunfailures>=15.0,<16", diff --git a/tests/benchmark/__init__.py b/tests/benchmark/__init__.py new file mode 100644 index 00000000..ef5f8ae4 --- /dev/null +++ b/tests/benchmark/__init__.py @@ -0,0 +1 @@ +"""Provides benchmark tests for Plugboard.""" diff --git a/tests/benchmark/test_benchmarking.py b/tests/benchmark/test_benchmarking.py new file mode 100644 index 00000000..7554a7a0 --- /dev/null +++ b/tests/benchmark/test_benchmarking.py @@ -0,0 +1,35 @@ +"""Simple benchmark tests for Plugboard models.""" + +import asyncio + +from pytest_benchmark.fixture import BenchmarkFixture + +from plugboard.connector import AsyncioConnector +from plugboard.process import LocalProcess, Process +from plugboard.schemas import ConnectorSpec +from tests.integration.test_process_with_components_run import A, B + + +def _setup_process() -> tuple[tuple[Process], dict]: + comp_a = A(name="comp_a", iters=1000) + comp_b1 = B(name="comp_b1", factor=1) + comp_b2 = B(name="comp_b2", factor=2) + components = [comp_a, comp_b1, comp_b2] + connectors = [ + AsyncioConnector(spec=ConnectorSpec(source="comp_a.out_1", target="comp_b1.in_1")), + AsyncioConnector(spec=ConnectorSpec(source="comp_b1.out_1", target="comp_b2.in_1")), + ] + process = LocalProcess(components=components, connectors=connectors) + # Initialise process so that this is excluded from the benchmark timing + asyncio.run(process.init()) + # Return args and kwargs tuple for benchmark.pedantic + return (process,), {} + + +def _run_process(process: Process) -> None: + asyncio.run(process.run()) + + +def test_benchmark_process_run(benchmark: BenchmarkFixture) -> None: + """Benchmark the running of a Plugboard Process.""" + benchmark.pedantic(_run_process, setup=_setup_process, rounds=5) diff --git a/uv.lock b/uv.lock index 533439c0..1fa8c437 100644 --- a/uv.lock +++ b/uv.lock @@ -1,5 +1,5 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.12, <4.0" resolution-markers = [ "python_full_version >= '3.14'", @@ -3274,6 +3274,7 @@ test = [ { name = "optuna" }, { name = "pytest" }, { name = "pytest-asyncio" }, + { name = "pytest-benchmark" }, { name = "pytest-cases" }, { name = "pytest-env" }, { name = "pytest-rerunfailures" }, @@ -3345,6 +3346,7 @@ test = [ { name = "optuna", specifier = ">=3.0,<4" }, { name = "pytest", specifier = ">=8.3,<9" }, { name = "pytest-asyncio", specifier = ">=1.0,<2" }, + { name = "pytest-benchmark", specifier = ">=5.1.0" }, { name = "pytest-cases", specifier = ">=3.8,<4" }, { name = "pytest-env", specifier = ">=1.1,<2" }, { name = "pytest-rerunfailures", specifier = ">=15.0,<16" }, @@ -3525,6 +3527,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + [[package]] name = "py-partiql-parser" version = "0.6.1" @@ -3758,6 +3769,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/9d/bf86eddabf8c6c9cb1ea9a869d6873b46f105a5d292d3a6f7071f5b07935/pytest_asyncio-1.1.0-py3-none-any.whl", hash = "sha256:5fe2d69607b0bd75c656d1211f969cadba035030156745ee09e7d71740e58ecf", size = 15157, upload-time = "2025-07-16T04:29:24.929Z" }, ] +[[package]] +name = "pytest-benchmark" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "py-cpuinfo" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810, upload-time = "2024-10-30T11:51:48.521Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259, upload-time = "2024-10-30T11:51:45.94Z" }, +] + [[package]] name = "pytest-cases" version = "3.9.1"