chore: wip [skip ci]

Author: not-matthias

src/pytest_codspeed/instruments/__init__.py

@@ -57,6 +57,7 @@ def get_result_dict(
 
 class MeasurementMode(str, Enum):
     Simulation = "simulation"
+    Memory = "memory"
     WallTime = "walltime"
 
     @classmethod
@@ -68,12 +69,12 @@ def _missing_(cls, value: object):
 
 
 def get_instrument_from_mode(mode: MeasurementMode) -> type[Instrument]:
-    from pytest_codspeed.instruments.valgrind import (
-        ValgrindInstrument,
+    from pytest_codspeed.instruments.analysis import (
+        AnalysisInstrument,
     )
     from pytest_codspeed.instruments.walltime import WallTimeInstrument
 
-    if mode == MeasurementMode.Simulation:
-        return ValgrindInstrument
+    if mode in (MeasurementMode.Simulation, MeasurementMode.Memory):
+        return AnalysisInstrument
     else:
         return WallTimeInstrument

…/pytest_codspeed/instruments/valgrind.py …/pytest_codspeed/instruments/analysis.pysrc/pytest_codspeed/instruments/valgrind.py renamed to src/pytest_codspeed/instruments/analysis.py src/pytest_codspeed/instruments/valgrind.py renamed to src/pytest_codspeed/instruments/analysis.py

@@ -19,8 +19,8 @@
     from pytest_codspeed.plugin import BenchmarkMarkerOptions, CodSpeedConfig
 
 
-class ValgrindInstrument(Instrument):
-    instrument = "valgrind"
+class AnalysisInstrument(Instrument):
+    instrument = "analysis"
     instrument_hooks: InstrumentHooks | None
 
     def __init__(self, config: CodSpeedConfig) -> None:
@@ -30,27 +30,29 @@ def __init__(self, config: CodSpeedConfig) -> None:
             self.instrument_hooks.set_integration("pytest-codspeed", __semver_version__)
         except RuntimeError as e:
             if os.environ.get("CODSPEED_ENV") is not None:
+                # FIXME: use the actual mode str here
                 raise Exception(
-                    "Failed to initialize CPU simulation instrument hooks"
+                    "Failed to initialize analysis instrument hooks"
                 ) from e
             self.instrument_hooks = None
 
         self.should_measure = self.instrument_hooks is not None
 
     def get_instrument_config_str_and_warns(self) -> tuple[str, list[str]]:
+        # FIXME: use the actual mode str here
         config = (
-            f"mode: simulation, "
+            f"instrument: analysis, "
             f"callgraph: {'enabled' if SUPPORTS_PERF_TRAMPOLINE else 'not supported'}"
         )
-        warnings = []
+        warns = []
         if not self.should_measure:
-            warnings.append(
+            warns.append(
                 "\033[1m"
                 "NOTICE: codspeed is enabled, but no performance measurement"
                 " will be made since it's running in an unknown environment."
                 "\033[0m"
             )
-        return config, warnings
+        return config, warns
 
     def measure(
         self,
@@ -73,14 +75,12 @@ def __codspeed_root_frame__() -> T:
             # Warmup CPython performance map cache
             __codspeed_root_frame__()
 
-        # Manually call the library function to avoid an extra stack frame. Also
-        # call the callgrind markers directly to avoid extra overhead.
-        self.instrument_hooks.lib.callgrind_start_instrumentation()
+        self.instrument_hooks.start_benchmark()
         try:
             return __codspeed_root_frame__()
         finally:
             # Ensure instrumentation is stopped even if the test failed
-            self.instrument_hooks.lib.callgrind_stop_instrumentation()
+            self.instrument_hooks.stop_benchmark()
             self.instrument_hooks.set_executed_benchmark(uri)
 
     def measure_pedantic(
@@ -91,8 +91,9 @@ def measure_pedantic(
         uri: str,
     ) -> T:
         if pedantic_options.rounds != 1 or pedantic_options.iterations != 1:
+            # FIXME: Use the real instrument here
             warnings.warn(
-                "Valgrind instrument ignores rounds and iterations settings "
+                "Analysis instrument ignores rounds and iterations settings "
                 "in pedantic mode"
             )
         if not self.instrument_hooks:
@@ -117,11 +118,12 @@ def __codspeed_root_frame__(*args, **kwargs) -> T:
 
         # Compute the actual result of the function
         args, kwargs = pedantic_options.setup_and_get_args_kwargs()
-        self.instrument_hooks.lib.callgrind_start_instrumentation()
+
+        self.instrument_hooks.start_benchmark()
         try:
             out = __codspeed_root_frame__(*args, **kwargs)
         finally:
-            self.instrument_hooks.lib.callgrind_stop_instrumentation()
+            self.instrument_hooks.stop_benchmark()
             self.instrument_hooks.set_executed_benchmark(uri)
             if pedantic_options.teardown is not None:
                 pedantic_options.teardown(*args, **kwargs)
@@ -140,5 +142,5 @@ def report(self, session: Session) -> None:
     def get_result_dict(self) -> dict[str, Any]:
         return {
             "instrument": {"type": self.instrument},
-            # bench results will be dumped by valgrind
+            # bench results will be dumped by the runner
         }

src/pytest_codspeed/instruments/hooks/instrument-hooks

@@ -112,8 +112,11 @@ def pytest_configure(config: pytest.Config):
     )
 
     if os.environ.get("CODSPEED_ENV") is not None:
-        if os.environ.get("CODSPEED_RUNNER_MODE") == "walltime":
+        runner_mode = os.environ.get("CODSPEED_RUNNER_MODE")
+        if runner_mode == "walltime":
             default_mode = MeasurementMode.WallTime.value
+        elif runner_mode == "memory":
+            default_mode = MeasurementMode.Memory.value
         else:
             default_mode = MeasurementMode.Simulation.value
     else:

src/pytest_codspeed/plugin.py

@@ -76,7 +76,7 @@ def fixtured_child():
     with open(perf_filepath) as perf_file:
         lines = perf_file.readlines()
         assert any(
-            "py::ValgrindInstrument.measure.<locals>.__codspeed_root_frame__" in line
+            "py::AnalysisInstrument.measure.<locals>.__codspeed_root_frame__" in line
             for line in lines
         ), "No root frame found in perf map"
         assert any("py::test_some_addition_marked" in line for line in lines), (
@@ -135,7 +135,7 @@ def foo():
     result = run_pytest_codspeed_with_mode(pytester, MeasurementMode.Simulation)
     result.stdout.fnmatch_lines(
         [
-            "*UserWarning: Valgrind instrument ignores rounds and iterations settings "
+            "*UserWarning: Analysis instrument ignores rounds and iterations settings "
             "in pedantic mode*"
         ]
     )