--wip-- [with ci]

Author: art049

src/pytest_codspeed/instruments/__init__.py

@@ -4,6 +4,8 @@
 from enum import Enum
 from typing import TYPE_CHECKING
 
+from pytest_codspeed.plugin import BenchmarkMarkerOptions
+
 if TYPE_CHECKING:
     from typing import Any, Callable, ClassVar, ParamSpec, TypeVar
 
@@ -27,6 +29,7 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]: ...
     @abstractmethod
     def measure(
         self,
+        marker_options: BenchmarkMarkerOptions,
         name: str,
         uri: str,
         fn: Callable[P, T],

src/pytest_codspeed/instruments/hooks/instrument-hooks

@@ -0,0 +1 @@
+Subproject commit b003e5024d61cfb784d6ac6f3ffd7d61bf7b9ec9

src/pytest_codspeed/instruments/valgrind/__init__.py

@@ -7,6 +7,7 @@
 from pytest_codspeed import __semver_version__
 from pytest_codspeed.instruments import Instrument
 from pytest_codspeed.instruments.valgrind._wrapper import get_lib
+from pytest_codspeed.plugin import BenchmarkMarkerOptions
 
 if TYPE_CHECKING:
     from typing import Any, Callable
@@ -40,7 +41,7 @@ def __init__(self, config: CodSpeedConfig) -> None:
     def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
         config = (
             f"mode: instrumentation, "
-            f"callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE  else 'not supported'}"
+            f"callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE else 'not supported'}"
         )
         warnings = []
         if not self.should_measure:
@@ -54,6 +55,7 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
 
     def measure(
         self,
+        marker_options: BenchmarkMarkerOptions,
         name: str,
         uri: str,
         fn: Callable[P, T],

src/pytest_codspeed/instruments/walltime.py

@@ -12,6 +12,7 @@
 from rich.text import Text
 
 from pytest_codspeed.instruments import Instrument
+from pytest_codspeed.plugin import BenchmarkMarkerOptions
 
 if TYPE_CHECKING:
     from typing import Any, Callable
@@ -24,7 +25,7 @@
 DEFAULT_WARMUP_TIME_NS = 1_000_000_000
 DEFAULT_MAX_TIME_NS = 3_000_000_000
 TIMER_RESOLUTION_NS = get_clock_info("perf_counter").resolution * 1e9
-DEFAULT_MIN_ROUND_TIME_NS = TIMER_RESOLUTION_NS * 1_000_000
+DEFAULT_MIN_ROUND_TIME_NS = int(TIMER_RESOLUTION_NS * 1_000_000)
 
 IQR_OUTLIER_FACTOR = 1.5
 STDEV_OUTLIER_FACTOR = 3
@@ -38,16 +39,35 @@ class BenchmarkConfig:
     max_rounds: int | None
 
     @classmethod
-    def from_codspeed_config(cls, config: CodSpeedConfig) -> BenchmarkConfig:
+    def from_codspeed_config_and_marker_data(
+        cls, config: CodSpeedConfig, marker_data: BenchmarkMarkerOptions
+    ) -> BenchmarkConfig:
+        if marker_data.max_time is not None:
+            max_time_ns = int(marker_data.max_time * 1e9)
+        elif config.max_time_ns is not None:
+            max_time_ns = config.max_time_ns
+        else:
+            max_time_ns = DEFAULT_MAX_TIME_NS
+
+        if marker_data.max_rounds is not None:
+            max_rounds = marker_data.max_rounds
+        elif config.max_rounds is not None:
+            max_rounds = config.max_rounds
+        else:
+            max_rounds = None
+
+        if marker_data.min_time is not None:
+            min_round_time_ns = int(marker_data.min_time * 1e9)
+        else:
+            min_round_time_ns = DEFAULT_MIN_ROUND_TIME_NS
+
         return cls(
             warmup_time_ns=config.warmup_time_ns
             if config.warmup_time_ns is not None
             else DEFAULT_WARMUP_TIME_NS,
-            min_round_time_ns=DEFAULT_MIN_ROUND_TIME_NS,
-            max_time_ns=config.max_time_ns
-            if config.max_time_ns is not None
-            else DEFAULT_MAX_TIME_NS,
-            max_rounds=config.max_rounds,
+            min_round_time_ns=min_round_time_ns,
+            max_time_ns=max_time_ns,
+            max_rounds=max_rounds,
         )
 
 
@@ -202,6 +222,7 @@ def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
 
     def measure(
         self,
+        marker_options: BenchmarkMarkerOptions,
         name: str,
         uri: str,
         fn: Callable[P, T],
@@ -214,7 +235,9 @@ def measure(
             fn=fn,
             args=args,
             kwargs=kwargs,
-            config=BenchmarkConfig.from_codspeed_config(self.config),
+            config=BenchmarkConfig.from_codspeed_config_and_marker_data(
+                self.config, marker_options
+            ),
         )
         self.benchmarks.append(bench)
         return out

src/pytest_codspeed/plugin.py

@@ -58,8 +58,7 @@ def pytest_addoption(parser: pytest.Parser):
         action="store",
         type=float,
         help=(
-            "The time to warm up the benchmark for (in seconds), "
-            "only for walltime mode"
+            "The time to warm up the benchmark for (in seconds), only for walltime mode"
         ),
     )
     group.addoption(
@@ -223,6 +222,43 @@ def has_benchmark_fixture(item: pytest.Item) -> bool:
     return "benchmark" in item_fixtures or "codspeed_benchmark" in item_fixtures
 
 
+@dataclass(frozen=True)
+class BenchmarkMarkerOptions:
+    group: str | None = None
+    """The group name to use for the benchmark"""
+    min_time: int | None = None
+    """The minimum time of a round (in seconds)"""
+    max_time: int | None = None
+    """The maximum time to run the benchmark for (in seconds)"""
+    max_rounds: int | None = None
+    """The maximum number of rounds to run the benchmark for. Takes precedence over
+    max_time."""
+
+
+def get_benchmark_marker_options(item: pytest.Item) -> BenchmarkMarkerOptions:
+    marker = item.get_closest_marker("codspeed_benchmark") or item.get_closest_marker(
+        "benchmark"
+    )
+    if marker is None:
+        return BenchmarkMarkerOptions()
+    if len(marker.args) > 0:
+        raise ValueError("Positional arguments are not allowed in the benchmark marker")
+
+    options = BenchmarkMarkerOptions(
+        group=marker.kwargs.pop("group", None),
+        max_time=marker.kwargs.pop("max_time", None),
+        min_time=marker.kwargs.pop("min_time", None),
+        max_rounds=marker.kwargs.pop("max_rounds", None),
+    )
+
+    if len(marker.kwargs) > 0:
+        raise ValueError(
+            "Unknown kwargs passed to benchmark marker: "
+            + ", ".join(marker.kwargs.keys())
+        )
+    return options
+
+
 def has_benchmark_marker(item: pytest.Item) -> bool:
     return (
         item.get_closest_marker("codspeed_benchmark") is not None
@@ -254,7 +290,7 @@ def pytest_collection_modifyitems(
 
 def _measure(
     plugin: CodSpeedPlugin,
-    nodeid: str,
+    node: pytest.Item,
     config: pytest.Config,
     fn: Callable[P, T],
     *args: P.args,
@@ -266,8 +302,9 @@ def _measure(
         gc.collect()
         gc.disable()
     try:
-        uri, name = get_git_relative_uri_and_name(nodeid, config.rootpath)
-        return plugin.instrument.measure(name, uri, fn, *args, **kwargs)
+        uri, name = get_git_relative_uri_and_name(node.nodeid, config.rootpath)
+        marker_options = get_benchmark_marker_options(node)
+        return plugin.instrument.measure(marker_options, name, uri, fn, *args, **kwargs)
     finally:
         # Ensure GC is re-enabled even if the test failed
         if is_gc_enabled:
@@ -276,13 +313,13 @@ def _measure(
 
 def wrap_runtest(
     plugin: CodSpeedPlugin,
-    nodeid: str,
+    node: pytest.Item,
     config: pytest.Config,
     fn: Callable[P, T],
 ) -> Callable[P, T]:
     @functools.wraps(fn)
     def wrapped(*args: P.args, **kwargs: P.kwargs) -> T:
-        return _measure(plugin, nodeid, config, fn, *args, **kwargs)
+        return _measure(plugin, node, config, fn, *args, **kwargs)
 
     return wrapped
 
@@ -299,7 +336,7 @@ def pytest_runtest_protocol(item: pytest.Item, nextitem: pytest.Item | None):
         return None
 
     # Wrap runtest and defer to default protocol
-    item.runtest = wrap_runtest(plugin, item.nodeid, item.config, item.runtest)
+    item.runtest = wrap_runtest(plugin, item, item.config, item.runtest)
     return None
 
 
@@ -343,9 +380,7 @@ def __call__(self, func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T
         config = self._request.config
         plugin = get_plugin(config)
         if plugin.is_codspeed_enabled:
-            return _measure(
-                plugin, self._request.node.nodeid, config, func, *args, **kwargs
-            )
+            return _measure(plugin, self._request.node, config, func, *args, **kwargs)
         else:
             return func(*args, **kwargs)