From 5870a606e3d9b9cf4a39865cae5b23a9c878643d Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 13:32:04 +0100
Subject: [PATCH 01/18] Add pytest-benchmark

---
 pyproject.toml |  1 +
 uv.lock        | 26 +++++++++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 39214324..5328d5f9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -75,6 +75,7 @@ test = [
     "optuna>=3.0,<4",
     "pytest>=8.3,<9",
     "pytest-asyncio>=1.0,<2",
+    "pytest-benchmark>=5.1.0",
     "pytest-cases>=3.8,<4",
     "pytest-env>=1.1,<2",
     "pytest-rerunfailures>=15.0,<16",
diff --git a/uv.lock b/uv.lock
index 533439c0..1fa8c437 100644
--- a/uv.lock
+++ b/uv.lock
@@ -1,5 +1,5 @@
 version = 1
-revision = 2
+revision = 3
 requires-python = ">=3.12, <4.0"
 resolution-markers = [
     "python_full_version >= '3.14'",
@@ -3274,6 +3274,7 @@ test = [
     { name = "optuna" },
     { name = "pytest" },
     { name = "pytest-asyncio" },
+    { name = "pytest-benchmark" },
     { name = "pytest-cases" },
     { name = "pytest-env" },
     { name = "pytest-rerunfailures" },
@@ -3345,6 +3346,7 @@ test = [
     { name = "optuna", specifier = ">=3.0,<4" },
     { name = "pytest", specifier = ">=8.3,<9" },
     { name = "pytest-asyncio", specifier = ">=1.0,<2" },
+    { name = "pytest-benchmark", specifier = ">=5.1.0" },
     { name = "pytest-cases", specifier = ">=3.8,<4" },
     { name = "pytest-env", specifier = ">=1.1,<2" },
     { name = "pytest-rerunfailures", specifier = ">=15.0,<16" },
@@ -3525,6 +3527,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" },
 ]
 
+[[package]]
+name = "py-cpuinfo"
+version = "9.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" },
+]
+
 [[package]]
 name = "py-partiql-parser"
 version = "0.6.1"
@@ -3758,6 +3769,19 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c7/9d/bf86eddabf8c6c9cb1ea9a869d6873b46f105a5d292d3a6f7071f5b07935/pytest_asyncio-1.1.0-py3-none-any.whl", hash = "sha256:5fe2d69607b0bd75c656d1211f969cadba035030156745ee09e7d71740e58ecf", size = 15157, upload-time = "2025-07-16T04:29:24.929Z" },
 ]
 
+[[package]]
+name = "pytest-benchmark"
+version = "5.1.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "py-cpuinfo" },
+    { name = "pytest" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/39/d0/a8bd08d641b393db3be3819b03e2d9bb8760ca8479080a26a5f6e540e99c/pytest-benchmark-5.1.0.tar.gz", hash = "sha256:9ea661cdc292e8231f7cd4c10b0319e56a2118e2c09d9f50e1b3d150d2aca105", size = 337810, upload-time = "2024-10-30T11:51:48.521Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/9e/d6/b41653199ea09d5969d4e385df9bbfd9a100f28ca7e824ce7c0a016e3053/pytest_benchmark-5.1.0-py3-none-any.whl", hash = "sha256:922de2dfa3033c227c96da942d1878191afa135a29485fb942e85dff1c592c89", size = 44259, upload-time = "2024-10-30T11:51:45.94Z" },
+]
+
 [[package]]
 name = "pytest-cases"
 version = "3.9.1"

From 7c1e28d2c5228713c65d232242806184effa021c Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 18:58:26 +0100
Subject: [PATCH 02/18] Add simple benchmark

---
 pyproject.toml                       |  3 ++-
 tests/benchmark/__init__.py          |  1 +
 tests/benchmark/test_benchmarking.py | 38 ++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+), 1 deletion(-)
 create mode 100644 tests/benchmark/__init__.py
 create mode 100644 tests/benchmark/test_benchmarking.py

diff --git a/pyproject.toml b/pyproject.toml
index 5328d5f9..0a0388f5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -112,7 +112,8 @@ asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
 asyncio_default_test_loop_scope = "session"
 markers = [
-    "smoke: marks tests as smoke tests (deselect with '-m \"not smoke\"')"
+    "smoke: marks tests as smoke tests (deselect with '-m \"not smoke\"')",
+    "benchmark: marks tests as benchmarking tests (deselect with '-m \"not benchmark\"')"
 ]
 
 [tool.coverage.run]
diff --git a/tests/benchmark/__init__.py b/tests/benchmark/__init__.py
new file mode 100644
index 00000000..ef5f8ae4
--- /dev/null
+++ b/tests/benchmark/__init__.py
@@ -0,0 +1 @@
+"""Provides benchmark tests for Plugboard."""
diff --git a/tests/benchmark/test_benchmarking.py b/tests/benchmark/test_benchmarking.py
new file mode 100644
index 00000000..f77818e1
--- /dev/null
+++ b/tests/benchmark/test_benchmarking.py
@@ -0,0 +1,38 @@
+"""Simple benchmark tests for Plugboard models."""
+
+import asyncio
+
+import pytest
+from pytest_benchmark.fixture import BenchmarkFixture
+
+from plugboard.connector import AsyncioConnector
+from plugboard.process import LocalProcess, Process
+from plugboard.schemas import ConnectorSpec
+from tests.integration.test_process_with_components_run import A, B
+
+
+def _setup_process() -> tuple[tuple[Process], dict]:
+    comp_a = A(name="comp_a", iters=1000)
+    comp_b1 = B(name="comp_b1", factor=1)
+    comp_b2 = B(name="comp_b2", factor=2)
+    components = [comp_a, comp_b1, comp_b2]
+    connectors = [
+        AsyncioConnector(spec=ConnectorSpec(source="comp_a.out_1", target="comp_b1.in_1")),
+        AsyncioConnector(spec=ConnectorSpec(source="comp_b1.out_1", target="comp_b2.in_1")),
+    ]
+    process = LocalProcess(components=components, connectors=connectors)
+    # Initialise process so that this is excluded from the benchmark timing
+    asyncio.run(process.init())
+    # Return args and kwargs tuple for benchmark.pedantic
+    return (process,), {}
+
+
+def _run_process(process: Process) -> None:
+    asyncio.run(process.run())
+
+
+@pytest.mark.benchmark
+@pytest.mark.asyncio
+def test_benchmark_process_run(benchmark: BenchmarkFixture) -> None:
+    """Benchmark the running of a Plugboard Process."""
+    benchmark.pedantic(_run_process, setup=_setup_process, rounds=5)

From e78c5ec031e75319e36aa554da846c8113c7770d Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 19:11:36 +0100
Subject: [PATCH 03/18] Marker not needed

---
 pyproject.toml                       | 3 +--
 tests/benchmark/test_benchmarking.py | 1 -
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 0a0388f5..5328d5f9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -112,8 +112,7 @@ asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
 asyncio_default_test_loop_scope = "session"
 markers = [
-    "smoke: marks tests as smoke tests (deselect with '-m \"not smoke\"')",
-    "benchmark: marks tests as benchmarking tests (deselect with '-m \"not benchmark\"')"
+    "smoke: marks tests as smoke tests (deselect with '-m \"not smoke\"')"
 ]
 
 [tool.coverage.run]
diff --git a/tests/benchmark/test_benchmarking.py b/tests/benchmark/test_benchmarking.py
index f77818e1..57bf2709 100644
--- a/tests/benchmark/test_benchmarking.py
+++ b/tests/benchmark/test_benchmarking.py
@@ -31,7 +31,6 @@ def _run_process(process: Process) -> None:
     asyncio.run(process.run())
 
 
-@pytest.mark.benchmark
 @pytest.mark.asyncio
 def test_benchmark_process_run(benchmark: BenchmarkFixture) -> None:
     """Benchmark the running of a Plugboard Process."""

From 818e369cc6b57e20c3c9818b37b1b9c50f5a18ee Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 19:18:57 +0100
Subject: [PATCH 04/18] Ignore benchmark files

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index e96b0844..8cba4ba3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,6 +54,9 @@ coverage.json
 .pytest_cache/
 cover/
 
+# Benchmarking
+.benchmarks/
+
 # Translations
 *.mo
 *.pot

From 159a8bb20412511016af7c06944b6369b3bf0299 Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 19:35:35 +0100
Subject: [PATCH 05/18] Add github action

---
 .github/workflows/benchmarks.yaml | 66 +++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 .github/workflows/benchmarks.yaml

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
new file mode 100644
index 00000000..4d741b84
--- /dev/null
+++ b/.github/workflows/benchmarks.yaml
@@ -0,0 +1,66 @@
+name: Benchmarks
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+jobs:
+  lint:
+    name: Benchmark tests
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python_version: [3.12]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{matrix.python_version}}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+
+      - name: Install project
+        run: uv sync --group test
+
+      - name: Run benchmarks
+        run: |
+          git checkout ${{ github.event.pull_request.base.sha }}
+          pytest --benchmark-only --benchmark-autosave --benchmark-save=base
+
+          git checkout ${{ github.event.pull_request.head.sha }}
+          pytest --benchmark-only --benchmark-autosave --benchmark-save=branch
+
+      - name: Compare results
+        run: |
+          pytest-benchmark compare | tee cmp_results
+
+          echo 'Benchmark comparison for [`${{ env.BASE_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.base.sha }}) (base) vs [`${{ env.HEAD_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.head.sha }}) (PR)' >> pr_comment
+          echo '```' >> pr_comment
+          echo cmp_results >> pr_comment
+          echo '```' >> pr_comment
+          cat pr_comment > ${{ env.PR_COMMENT }}
+      
+      - name: Comment on PR
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            github.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: require('fs').readFileSync('${{ env.PR_COMMENT }}').toString()
+            });
+

From 15649fe8711661e0e7cd2bd6dcaff2ce7aa9a2cf Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 19:37:14 +0100
Subject: [PATCH 06/18] Simplify

---
 .github/workflows/benchmarks.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index 4d741b84..2e78f74d 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -37,10 +37,10 @@ jobs:
       - name: Run benchmarks
         run: |
           git checkout ${{ github.event.pull_request.base.sha }}
-          pytest --benchmark-only --benchmark-autosave --benchmark-save=base
+          pytest --benchmark-only --benchmark-save=base
 
           git checkout ${{ github.event.pull_request.head.sha }}
-          pytest --benchmark-only --benchmark-autosave --benchmark-save=branch
+          pytest --benchmark-only --benchmark-save=branch
 
       - name: Compare results
         run: |

From c22924dbd9d9d0e9d58a2969303e48d4a16e0a85 Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 19:38:48 +0100
Subject: [PATCH 07/18] Only comment on open

---
 .github/workflows/benchmarks.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index 2e78f74d..979eb1c1 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -4,8 +4,6 @@ on:
   pull_request:
     types:
       - opened
-      - synchronize
-      - reopened
 
 jobs:
   lint:

From 6eeb628205c6bfa33d59ba262b22c40686cca0fa Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 19:44:16 +0100
Subject: [PATCH 08/18] Fixup

---
 .github/workflows/benchmarks.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index 979eb1c1..09839800 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -4,6 +4,7 @@ on:
   pull_request:
     types:
       - opened
+      - synchronize
 
 jobs:
   lint:
@@ -35,14 +36,14 @@ jobs:
       - name: Run benchmarks
         run: |
           git checkout ${{ github.event.pull_request.base.sha }}
-          pytest --benchmark-only --benchmark-save=base
+          uv run pytest --benchmark-only --benchmark-save=base
 
           git checkout ${{ github.event.pull_request.head.sha }}
-          pytest --benchmark-only --benchmark-save=branch
+          uv run pytest --benchmark-only --benchmark-save=branch
 
       - name: Compare results
         run: |
-          pytest-benchmark compare | tee cmp_results
+          uv run pytest-benchmark compare | tee cmp_results
 
           echo 'Benchmark comparison for [`${{ env.BASE_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.base.sha }}) (base) vs [`${{ env.HEAD_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.head.sha }}) (PR)' >> pr_comment
           echo '```' >> pr_comment

From d5ddcf467137bd45e9ee064937054efdb174de1d Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 19:50:18 +0100
Subject: [PATCH 09/18] Add missing setup stage

---
 .github/workflows/benchmarks.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index 09839800..a80ce284 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -33,6 +33,12 @@ jobs:
       - name: Install project
         run: uv sync --group test
 
+      - name: Setup benchmarks
+        run: |
+          echo "BASE_SHA=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-8)" >> $GITHUB_ENV
+          echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV
+          echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV
+
       - name: Run benchmarks
         run: |
           git checkout ${{ github.event.pull_request.base.sha }}

From 93f4dee8112aaa616bbd3d5ebff48bca19a1c92f Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 19:53:08 +0100
Subject: [PATCH 10/18] Update createComment syntax

---
 .github/workflows/benchmarks.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index a80ce284..8dc21aeb 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -62,7 +62,7 @@ jobs:
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
-            github.issues.createComment({
+            github.rest.issues.createComment({
               issue_number: context.issue.number,
               owner: context.repo.owner,
               repo: context.repo.repo,

From 9249d512e5739e9ed18bf6eb99b72ac03ffd2446 Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 19:55:48 +0100
Subject: [PATCH 11/18] Remove marker

---
 tests/benchmark/test_benchmarking.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/benchmark/test_benchmarking.py b/tests/benchmark/test_benchmarking.py
index 57bf2709..7554a7a0 100644
--- a/tests/benchmark/test_benchmarking.py
+++ b/tests/benchmark/test_benchmarking.py
@@ -2,7 +2,6 @@
 
 import asyncio
 
-import pytest
 from pytest_benchmark.fixture import BenchmarkFixture
 
 from plugboard.connector import AsyncioConnector
@@ -31,7 +30,6 @@ def _run_process(process: Process) -> None:
     asyncio.run(process.run())
 
 
-@pytest.mark.asyncio
 def test_benchmark_process_run(benchmark: BenchmarkFixture) -> None:
     """Benchmark the running of a Plugboard Process."""
     benchmark.pedantic(_run_process, setup=_setup_process, rounds=5)

From 4917f5c59a42e1e544964fb393d0095c4423e885 Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 19:58:05 +0100
Subject: [PATCH 12/18] Fix

---
 .github/workflows/benchmarks.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index 8dc21aeb..1ae063e2 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -53,7 +53,7 @@ jobs:
 
           echo 'Benchmark comparison for [`${{ env.BASE_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.base.sha }}) (base) vs [`${{ env.HEAD_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.head.sha }}) (PR)' >> pr_comment
           echo '```' >> pr_comment
-          echo cmp_results >> pr_comment
+          cat cmp_results >> pr_comment
           echo '```' >> pr_comment
           cat pr_comment > ${{ env.PR_COMMENT }}
       

From b1b89699dd5801410b22ba3702df41fa1c73fa0e Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 20:40:49 +0100
Subject: [PATCH 13/18] Try something different

---
 .github/workflows/benchmarks.yaml | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index 1ae063e2..a3b274c2 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -14,10 +14,16 @@ jobs:
       matrix:
         python_version: [3.12]
     steps:
-      - name: Checkout
+      - name: Checkout branch
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0
+          path: pr
+
+      - name: Checkout main
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          path: main
 
       - name: Install python
         uses: actions/setup-python@v5
@@ -30,26 +36,27 @@ jobs:
           enable-cache: true
           cache-dependency-glob: "uv.lock"
 
-      - name: Install project
-        run: uv sync --group test
-
       - name: Setup benchmarks
         run: |
           echo "BASE_SHA=$(echo ${{ github.event.pull_request.base.sha }} | cut -c1-8)" >> $GITHUB_ENV
           echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV
           echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV
 
-      - name: Run benchmarks
+      - name: Run benchmarks on main
+        working-directory: ./main
         run: |
-          git checkout ${{ github.event.pull_request.base.sha }}
+          uv sync --group test
           uv run pytest --benchmark-only --benchmark-save=base
 
-          git checkout ${{ github.event.pull_request.head.sha }}
-          uv run pytest --benchmark-only --benchmark-save=branch
+      - name: Run benchmarks on PR
+        working-directory: ./pr
+        run: |
+          uv sync --group test
+          uv run pytest --benchmark-only --benchmark-save=pr
 
       - name: Compare results
         run: |
-          uv run pytest-benchmark compare | tee cmp_results
+          uv run pytest-benchmark compare main/.benchmarks/**/* pr/.benchmarks/**/* | tee cmp_results
 
           echo 'Benchmark comparison for [`${{ env.BASE_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.base.sha }}) (base) vs [`${{ env.HEAD_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.head.sha }}) (PR)' >> pr_comment
           echo '```' >> pr_comment

From 62544ac1f78f76ce7539afdd683b72c47056e23c Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 20:45:42 +0100
Subject: [PATCH 14/18] Fix uv install

---
 .github/workflows/benchmarks.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index a3b274c2..50638567 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -34,7 +34,7 @@ jobs:
         uses: astral-sh/setup-uv@v4
         with:
           enable-cache: true
-          cache-dependency-glob: "uv.lock"
+          cache-dependency-glob: "main/uv.lock"
 
       - name: Setup benchmarks
         run: |

From 6ab0a94d514f45c47b30dbe3f1cc192feadf2410 Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 20:47:26 +0100
Subject: [PATCH 15/18] Change order

---
 .github/workflows/benchmarks.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index 50638567..063a26d5 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -42,18 +42,18 @@ jobs:
           echo "HEAD_SHA=$(echo ${{ github.event.pull_request.head.sha }} | cut -c1-8)" >> $GITHUB_ENV
           echo "PR_COMMENT=$(mktemp)" >> $GITHUB_ENV
 
-      - name: Run benchmarks on main
-        working-directory: ./main
-        run: |
-          uv sync --group test
-          uv run pytest --benchmark-only --benchmark-save=base
-
       - name: Run benchmarks on PR
         working-directory: ./pr
         run: |
           uv sync --group test
           uv run pytest --benchmark-only --benchmark-save=pr
 
+      - name: Run benchmarks on main
+        working-directory: ./main
+        run: |
+          uv sync --group test
+          uv run pytest --benchmark-only --benchmark-save=base
+
       - name: Compare results
         run: |
           uv run pytest-benchmark compare main/.benchmarks/**/* pr/.benchmarks/**/* | tee cmp_results

From 1a2834c86dbe075ecca98df50ea5e789753ca539 Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 20:56:00 +0100
Subject: [PATCH 16/18] Fix

---
 .github/workflows/benchmarks.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index 063a26d5..e95bdc35 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -50,6 +50,7 @@ jobs:
 
       - name: Run benchmarks on main
         working-directory: ./main
+        continue-on-error: true
         run: |
           uv sync --group test
           uv run pytest --benchmark-only --benchmark-save=base

From 06fe890e847b16882e0bcf22f4bd552c6b4e7200 Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 20:59:57 +0100
Subject: [PATCH 17/18] Fix comparison

---
 .github/workflows/benchmarks.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index e95bdc35..bc14cd18 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -57,7 +57,7 @@ jobs:
 
       - name: Compare results
         run: |
-          uv run pytest-benchmark compare main/.benchmarks/**/* pr/.benchmarks/**/* | tee cmp_results
+          uv run pytest-benchmark compare **/.benchmarks/**/*.json | tee cmp_results
 
           echo 'Benchmark comparison for [`${{ env.BASE_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.base.sha }}) (base) vs [`${{ env.HEAD_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.head.sha }}) (PR)' >> pr_comment
           echo '```' >> pr_comment

From b5af6f72e8a1db71eba1f4c15dcf7f5bef6dac74 Mon Sep 17 00:00:00 2001
From: Toby Coleman <toby@tobycoleman.com>
Date: Sat, 23 Aug 2025 21:02:47 +0100
Subject: [PATCH 18/18] Working?

---
 .github/workflows/benchmarks.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index bc14cd18..9a4c9332 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -56,8 +56,9 @@ jobs:
           uv run pytest --benchmark-only --benchmark-save=base
 
       - name: Compare results
+        continue-on-error: false
         run: |
-          uv run pytest-benchmark compare **/.benchmarks/**/*.json | tee cmp_results
+          uvx pytest-benchmark compare **/.benchmarks/**/*.json | tee cmp_results
 
           echo 'Benchmark comparison for [`${{ env.BASE_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.base.sha }}) (base) vs [`${{ env.HEAD_SHA }}`](${{ github.event.repository.html_url }}/commit/${{ github.event.pull_request.head.sha }}) (PR)' >> pr_comment
           echo '```' >> pr_comment