feat: support memory profiling

Author: not-matthias

src/pytest_codspeed/instruments/__init__.py

@@ -21,7 +21,7 @@ class Instrument(metaclass=ABCMeta):
     instrument: ClassVar[str]
 
     @abstractmethod
-    def __init__(self, config: CodSpeedConfig): ...
+    def __init__(self, config: CodSpeedConfig, mode: MeasurementMode): ...
 
     @abstractmethod
     def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]: ...
@@ -57,6 +57,7 @@ def get_result_dict(
 
 class MeasurementMode(str, Enum):
     Simulation = "simulation"
+    Memory = "memory"
     WallTime = "walltime"
 
     @classmethod
@@ -68,12 +69,12 @@ def _missing_(cls, value: object):
 
 
 def get_instrument_from_mode(mode: MeasurementMode) -> type[Instrument]:
-    from pytest_codspeed.instruments.valgrind import (
-        ValgrindInstrument,
+    from pytest_codspeed.instruments.analysis import (
+        AnalysisInstrument,
     )
     from pytest_codspeed.instruments.walltime import WallTimeInstrument
 
-    if mode == MeasurementMode.Simulation:
-        return ValgrindInstrument
+    if mode in (MeasurementMode.Simulation, MeasurementMode.Memory):
+        return AnalysisInstrument
     else:
         return WallTimeInstrument

…/pytest_codspeed/instruments/valgrind.py …/pytest_codspeed/instruments/analysis.pysrc/pytest_codspeed/instruments/valgrind.py renamed to src/pytest_codspeed/instruments/analysis.py src/pytest_codspeed/instruments/valgrind.py renamed to src/pytest_codspeed/instruments/analysis.py

@@ -15,42 +15,44 @@
     from pytest import Session
 
     from pytest_codspeed.config import PedanticOptions
-    from pytest_codspeed.instruments import P, T
+    from pytest_codspeed.instruments import MeasurementMode, P, T
     from pytest_codspeed.plugin import BenchmarkMarkerOptions, CodSpeedConfig
 
 
-class ValgrindInstrument(Instrument):
-    instrument = "valgrind"
+class AnalysisInstrument(Instrument):
+    instrument = "analysis"
     instrument_hooks: InstrumentHooks | None
+    mode: MeasurementMode
 
-    def __init__(self, config: CodSpeedConfig) -> None:
+    def __init__(self, config: CodSpeedConfig, mode: MeasurementMode) -> None:
+        self.mode = mode
         self.benchmark_count = 0
         try:
             self.instrument_hooks = InstrumentHooks()
             self.instrument_hooks.set_integration("pytest-codspeed", __semver_version__)
         except RuntimeError as e:
             if os.environ.get("CODSPEED_ENV") is not None:
                 raise Exception(
-                    "Failed to initialize CPU simulation instrument hooks"
+                    f"Failed to initialize {self.mode.value} instrument hooks"
                 ) from e
             self.instrument_hooks = None
 
         self.should_measure = self.instrument_hooks is not None
 
     def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
         config = (
-            f"mode: simulation, "
+            f"instrument: {self.mode.value}, "
             f"callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE else 'not supported'}"
         )
-        warnings = []
+        warns = []
         if not self.should_measure:
-            warnings.append(
+            warns.append(
                 "\033[1m"
                 "NOTICE: codspeed is enabled, but no performance measurement"
                 " will be made since it's running in an unknown environment."
                 "\033[0m"
             )
-        return config, warnings
+        return config, warns
 
     def measure(
         self,
@@ -73,14 +75,12 @@ def __codspeed_root_frame__() -> T:
             # Warmup CPython performance map cache
             __codspeed_root_frame__()
 
-        # Manually call the library function to avoid an extra stack frame. Also
-        # call the callgrind markers directly to avoid extra overhead.
-        self.instrument_hooks.lib.callgrind_start_instrumentation()
+        self.instrument_hooks.start_benchmark()
         try:
             return __codspeed_root_frame__()
         finally:
             # Ensure instrumentation is stopped even if the test failed
-            self.instrument_hooks.lib.callgrind_stop_instrumentation()
+            self.instrument_hooks.stop_benchmark()
             self.instrument_hooks.set_executed_benchmark(uri)
 
     def measure_pedantic(
@@ -92,8 +92,8 @@ def measure_pedantic(
     ) -> T:
         if pedantic_options.rounds != 1 or pedantic_options.iterations != 1:
             warnings.warn(
-                "Valgrind instrument ignores rounds and iterations settings "
-                "in pedantic mode"
+                f"{self.mode.value.capitalize()} instrument ignores rounds and "
+                "iterations settings in pedantic mode"
             )
         if not self.instrument_hooks:
             args, kwargs = pedantic_options.setup_and_get_args_kwargs()
@@ -117,11 +117,12 @@ def __codspeed_root_frame__(*args, **kwargs) -> T:
 
         # Compute the actual result of the function
         args, kwargs = pedantic_options.setup_and_get_args_kwargs()
-        self.instrument_hooks.lib.callgrind_start_instrumentation()
+
+        self.instrument_hooks.start_benchmark()
         try:
             out = __codspeed_root_frame__(*args, **kwargs)
         finally:
-            self.instrument_hooks.lib.callgrind_stop_instrumentation()
+            self.instrument_hooks.stop_benchmark()
             self.instrument_hooks.set_executed_benchmark(uri)
             if pedantic_options.teardown is not None:
                 pedantic_options.teardown(*args, **kwargs)
@@ -140,5 +141,5 @@ def report(self, session: Session) -> None:
     def get_result_dict(self) -> dict[str, Any]:
         return {
             "instrument": {"type": self.instrument},
-            # bench results will be dumped by valgrind
+            # bench results will be dumped by the runner
         }

src/pytest_codspeed/instruments/hooks/instrument-hooks

@@ -24,7 +24,7 @@
     from pytest import Session
 
     from pytest_codspeed.config import PedanticOptions
-    from pytest_codspeed.instruments import P, T
+    from pytest_codspeed.instruments import MeasurementMode, P, T
     from pytest_codspeed.plugin import BenchmarkMarkerOptions, CodSpeedConfig
 
 DEFAULT_WARMUP_TIME_NS = 1_000_000_000
@@ -159,7 +159,7 @@ class WallTimeInstrument(Instrument):
     instrument = "walltime"
     instrument_hooks: InstrumentHooks | None
 
-    def __init__(self, config: CodSpeedConfig) -> None:
+    def __init__(self, config: CodSpeedConfig, mode: MeasurementMode) -> None:
         try:
             self.instrument_hooks = InstrumentHooks()
             self.instrument_hooks.set_integration("pytest-codspeed", __semver_version__)

src/pytest_codspeed/instruments/walltime.py

@@ -112,8 +112,11 @@ def pytest_configure(config: pytest.Config):
     )
 
     if os.environ.get("CODSPEED_ENV") is not None:
-        if os.environ.get("CODSPEED_RUNNER_MODE") == "walltime":
+        runner_mode = os.environ.get("CODSPEED_RUNNER_MODE")
+        if runner_mode == "walltime":
             default_mode = MeasurementMode.WallTime.value
+        elif runner_mode == "memory":
+            default_mode = MeasurementMode.Memory.value
         else:
             default_mode = MeasurementMode.Simulation.value
     else:
@@ -142,7 +145,7 @@ def pytest_configure(config: pytest.Config):
         disabled_plugins=tuple(disabled_plugins),
         is_codspeed_enabled=is_codspeed_enabled,
         mode=mode,
-        instrument=instrument(codspeed_config),
+        instrument=instrument(codspeed_config, mode),
         config=codspeed_config,
         profile_folder=Path(profile_folder) if profile_folder else None,
     )

src/pytest_codspeed/plugin.py

@@ -76,7 +76,7 @@ def fixtured_child():
     with open(perf_filepath) as perf_file:
         lines = perf_file.readlines()
         assert any(
-            "py::ValgrindInstrument.measure.<locals>.__codspeed_root_frame__" in line
+            "py::AnalysisInstrument.measure.<locals>.__codspeed_root_frame__" in line
             for line in lines
         ), "No root frame found in perf map"
         assert any("py::test_some_addition_marked" in line for line in lines), (
@@ -135,8 +135,8 @@ def foo():
     result = run_pytest_codspeed_with_mode(pytester, MeasurementMode.Simulation)
     result.stdout.fnmatch_lines(
         [
-            "*UserWarning: Valgrind instrument ignores rounds and iterations settings "
-            "in pedantic mode*"
+            "*UserWarning: Simulation instrument ignores rounds and iterations settings"
+            " in pedantic mode*"
         ]
     )
     result.assert_outcomes(passed=1)