feat: make sure the benchmark fixture can only be called once per bench

Author: art049

src/pytest_codspeed/plugin.py

@@ -318,14 +318,20 @@ def __init__(self, request: pytest.FixtureRequest):
         self.extra_info: dict = {}
 
         self._request = request
-
-    def __call__(self, func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T:
-        config = self._request.config
-        plugin = get_plugin(config)
-        if plugin.is_codspeed_enabled:
-            return _measure(plugin, self._request.node, config, func, *args, **kwargs)
+        self._config = self._request.config
+        self._plugin = get_plugin(self._config)
+        self._called = False
+
+    def __call__(self, target: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T:
+        if self._called:
+            raise RuntimeError("The benchmark fixture can only be used once per test")
+        self._called = True
+        if self._plugin.is_codspeed_enabled:
+            return _measure(
+                self._plugin, self._request.node, self._config, target, *args, **kwargs
+            )
         else:
-            return func(*args, **kwargs)
+            return target(*args, **kwargs)
 
 
 @pytest.fixture(scope="function")

tests/test_pytest_plugin.py

@@ -338,3 +338,25 @@ def test_capsys(capsys):
     result.assert_outcomes(passed=1)
     result.stdout.no_fnmatch_line("*print to stdout*")
     result.stderr.no_fnmatch_line("*print to stderr*")
+
+@pytest.mark.parametrize("mode", [*MeasurementMode])
+def test_benchmark_fixture_used_twice(
+    pytester: pytest.Pytester, mode: MeasurementMode
+) -> None:
+    """Test that using the benchmark fixture twice in a test raises an error."""
+    pytester.makepyfile(
+        """
+        def test_benchmark_used_twice(benchmark):
+            def foo():
+                pass
+
+            benchmark(foo)
+            benchmark(foo)
+        """
+    )
+    result = run_pytest_codspeed_with_mode(pytester, mode)
+    assert result.ret == 1, "the run should have failed"
+    result.stdout.fnmatch_lines(
+        ["*RuntimeError: The benchmark fixture can only be used once per test*"]
+    )
+