test: strengthen integration tests and eliminate helper duplication

- Move _make_result/_bench helpers to conftest.py — single source of truth
- Add result.assert_outcomes(passed=1) to all 9 integration tests so a
  broken feature cannot hide behind a passing stdout check
- Fix test_no_comparison_with_profile_folder: CODSPEED_PROFILE_FOLDER is
  an env var, not a CLI flag; use monkeypatch.setenv instead
- Ruff TC003/I001 fixes (Path in TYPE_CHECKING block, import sort)

Author: StephaneWamba

tests/conftest.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import importlib.util
+import json
 import os
 import shutil
 import sys
@@ -13,6 +14,8 @@
 from pytest_codspeed.utils import IS_PYTEST_BENCHMARK_INSTALLED
 
 if TYPE_CHECKING:
+    from pathlib import Path
+
     from _pytest.pytester import RunResult
 
 pytest_plugins = ["pytester"]
@@ -71,6 +74,28 @@ def ctx_manager():
     return ctx_manager
 
 
+# ---------------------------------------------------------------------------
+# Shared test helpers
+# ---------------------------------------------------------------------------
+
+
+def _make_result(tmp_path: Path, filename: str, benchmarks: list) -> Path:
+    """Write a minimal CodSpeed result JSON and return its path."""
+    path = tmp_path / filename
+    path.write_text(
+        json.dumps({"instrument": {"type": "walltime"}, "benchmarks": benchmarks})
+    )
+    return path
+
+
+def _bench(uri: str, mean_ns: float, output_hash=None) -> dict:
+    """Return a minimal benchmark entry, optionally with an output hash."""
+    entry: dict = {"uri": uri, "stats": {"mean_ns": mean_ns}}
+    if output_hash is not None:
+        entry["output_hash"] = output_hash
+    return entry
+
+
 def run_pytest_codspeed_with_mode(
     pytester: pytest.Pytester, mode: MeasurementMode, *args, **kwargs
 ) -> RunResult:

tests/test_comparison.py

@@ -12,6 +12,7 @@
 from pathlib import Path
 
 import pytest
+from conftest import _bench, _make_result
 
 from pytest_codspeed.comparison import (
     BenchmarkDiff,
@@ -21,32 +22,6 @@
     print_comparison_report,
 )
 
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-def _make_result(tmp_path: Path, filename: str, benchmarks: list) -> Path:
-    """Write a minimal CodSpeed result JSON and return its path."""
-    path = tmp_path / filename
-    path.write_text(
-        json.dumps(
-            {
-                "instrument": {"type": "walltime"},
-                "benchmarks": benchmarks,
-            }
-        )
-    )
-    return path
-
-
-def _bench(uri: str, mean_ns: float) -> dict:
-    """Return a minimal benchmark entry."""
-    return {
-        "uri": uri,
-        "stats": {"mean_ns": mean_ns},
-    }
-
-
 # ---------------------------------------------------------------------------
 # find_baseline
 # ---------------------------------------------------------------------------

tests/test_comparison_integration.py

@@ -32,6 +32,7 @@ def test_no_comparison_on_first_run(self, pytester: pytest.Pytester) -> None:
 
         result = run_pytest_codspeed_with_mode(pytester, MeasurementMode.WallTime)
 
+        result.assert_outcomes(passed=1)
         result.stdout.no_fnmatch_line("*CodSpeed local comparison*")
 
     def test_comparison_appears_on_second_run(self, pytester: pytest.Pytester) -> None:
@@ -44,6 +45,7 @@ def test_comparison_appears_on_second_run(self, pytester: pytest.Pytester) -> No
         # Second run — finds the first JSON as baseline
         result = run_pytest_codspeed_with_mode(pytester, MeasurementMode.WallTime)
 
+        result.assert_outcomes(passed=1)
         result.stdout.fnmatch_lines(["*CodSpeed local comparison*"])
 
     def test_comparison_shows_benchmark_count(self, pytester: pytest.Pytester) -> None:
@@ -53,33 +55,31 @@ def test_comparison_shows_benchmark_count(self, pytester: pytest.Pytester) -> No
         run_pytest_codspeed_with_mode(pytester, MeasurementMode.WallTime)
         result = run_pytest_codspeed_with_mode(pytester, MeasurementMode.WallTime)
 
-        # "1 compared" is the footer line produced by print_comparison_report
+        result.assert_outcomes(passed=1)
         result.stdout.fnmatch_lines(["*1 compared*"])
 
-    def test_no_comparison_with_profile_folder(self, pytester: pytest.Pytester) -> None:
-        """When --codspeed-profile-folder is set, comparison is skipped.
+    def test_no_comparison_with_profile_folder(
+        self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """When CODSPEED_PROFILE_FOLDER is set, comparison is skipped.
 
-        The profile folder path is used by the CodSpeed runner in CI; in
+        The profile folder env var is used by the CodSpeed runner in CI; in
         that context the result file is written to a custom location and
         the local comparison loop makes no sense.
         """
         pytester.makepyfile(_BENCH_FILE)
         profile_dir = pytester.path / "profile"
         profile_dir.mkdir()
+        (profile_dir / "results").mkdir()
+
+        monkeypatch.setenv("CODSPEED_PROFILE_FOLDER", str(profile_dir))
 
         # First run with profile folder
-        run_pytest_codspeed_with_mode(
-            pytester,
-            MeasurementMode.WallTime,
-            f"--codspeed-profile-folder={profile_dir}",
-        )
+        run_pytest_codspeed_with_mode(pytester, MeasurementMode.WallTime)
         # Second run with profile folder — still no comparison
-        result = run_pytest_codspeed_with_mode(
-            pytester,
-            MeasurementMode.WallTime,
-            f"--codspeed-profile-folder={profile_dir}",
-        )
+        result = run_pytest_codspeed_with_mode(pytester, MeasurementMode.WallTime)
 
+        result.assert_outcomes(passed=1)
         result.stdout.no_fnmatch_line("*CodSpeed local comparison*")
 
     def test_no_comparison_in_simulation_mode(self, pytester: pytest.Pytester) -> None:
@@ -91,4 +91,5 @@ def test_no_comparison_in_simulation_mode(self, pytester: pytest.Pytester) -> No
         # Second run in simulation mode — no benchmarks to compare
         result = run_pytest_codspeed_with_mode(pytester, MeasurementMode.Simulation)
 
+        result.assert_outcomes(passed=1)
         result.stdout.no_fnmatch_line("*CodSpeed local comparison*")

tests/test_eval_harness.py

@@ -1,35 +1,14 @@
 """Unit tests for pytest_codspeed.eval_harness and output_changed in comparison."""
 
-import json
 import math
 from pathlib import Path
 
 import pytest
+from conftest import _bench, _make_result
 
 from pytest_codspeed.comparison import BenchmarkDiff, ComparisonReport, compare_results
 from pytest_codspeed.eval_harness import EvalReport, compute_score
 
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _make_result(tmp_path: Path, filename: str, benchmarks: list) -> Path:
-    path = tmp_path / filename
-    path.write_text(
-        json.dumps({"instrument": {"type": "walltime"}, "benchmarks": benchmarks})
-    )
-    return path
-
-
-def _bench(uri: str, mean_ns: float, output_hash=None) -> dict:
-    entry: dict = {"uri": uri, "stats": {"mean_ns": mean_ns}}
-    if output_hash is not None:
-        entry["output_hash"] = output_hash
-    return entry
-
-
 # ---------------------------------------------------------------------------
 # compute_score
 # ---------------------------------------------------------------------------

tests/test_eval_harness_integration.py

@@ -36,6 +36,7 @@ def test_no_warning_when_output_stable(
             pytester, MeasurementMode.WallTime, "--codspeed-capture-output"
         )
 
+        result.assert_outcomes(passed=1)
         result.stdout.no_fnmatch_line("*output changed*")
 
     def test_warning_when_output_changes(
@@ -51,6 +52,7 @@ def test_warning_when_output_changes(
             pytester, MeasurementMode.WallTime, "--codspeed-capture-output"
         )
 
+        result.assert_outcomes(passed=1)
         result.stdout.fnmatch_lines(["*output changed*"])
 
     def test_no_warning_without_flag(self, pytester: pytest.Pytester) -> None:
@@ -60,6 +62,7 @@ def test_no_warning_without_flag(self, pytester: pytest.Pytester) -> None:
         pytester.makepyfile(_BROKEN_BENCH)
         result = run_pytest_codspeed_with_mode(pytester, MeasurementMode.WallTime)
 
+        result.assert_outcomes(passed=1)
         result.stdout.no_fnmatch_line("*output changed*")
 
     def test_correctness_warning_in_footer(
@@ -75,4 +78,5 @@ def test_correctness_warning_in_footer(
             pytester, MeasurementMode.WallTime, "--codspeed-capture-output"
         )
 
+        result.assert_outcomes(passed=1)
         result.stdout.fnmatch_lines(["*1 correctness warning(s)*"])