Test the test coherence (#7318)

Author: cbeauchesne

.cursor/rules/test_the_test.mdc

@@ -0,0 +1,9 @@
+---
+description: Rules for tests inside tests/test_the_test
+globs: 
+alwaysApply: true
+---
+
+# Which scenario contains tests/test_the_test functions and classes
+
+All tests inside tests/test_the_test should belong to either TEST_THE_TEST, MOCK_THE_TEST or MOCK_THE_TEST_2. For almost all situation, they should belong to TEST_THE_TEST,

tests/test_the_test/test_itself.py

@@ -0,0 +1,15 @@
+"""It's easy to forget to attach a test inside this folder to the right scenario"""
+
+from utils import scenarios
+from .utils import get_scenario_map
+
+
+@scenarios.test_the_test
+def test_the_test_coherence():
+    valid_scenarios = ("TEST_THE_TEST", "MOCK_THE_TEST", "MOCK_THE_TEST_2")
+    scenario_map = get_scenario_map()
+
+    for node_id, scenario_list in scenario_map.items():
+        if node_id.startswith("tests/test_the_test/"):
+            for scenario in scenario_list:
+                assert scenario in valid_scenarios, f"{node_id} should belong to TEST_THE_TEST"

tests/test_the_test/test_start_stop_agent.py

@@ -7,6 +7,7 @@
 from utils.docker_fixtures._test_agent import DEFAULT_OTLP_HTTP_PORT, DEFAULT_OTLP_GRPC_PORT
 
 
+@scenarios.test_the_test
 def test_start_agent_then_stop(request: pytest.FixtureRequest):
     # Docker-backed smoke test. Gate at runtime rather than with a module-level
     # pytest.mark.skipif: the root conftest's _item_must_pass iterates skipif

tests/test_the_test/test_test_agent_pool.py

@@ -1,6 +1,7 @@
 import contextlib
 from collections.abc import Iterator
 
+from utils import scenarios
 from utils.docker_fixtures._test_agent_pool import WorkerAgentPool, agent_env_key
 
 
@@ -36,10 +37,12 @@ def __call__(self, request: object, agent_env: dict[str, str]) -> Iterator[_Fake
             self.stopped += 1
 
 
+@scenarios.test_the_test
 def test_env_key_is_order_independent():
     assert agent_env_key({"A": "1", "B": "2"}) == agent_env_key({"B": "2", "A": "1"})
 
 
+@scenarios.test_the_test
 def test_same_env_reuses_and_clears():
     creator = _FakeCreator()
     with WorkerAgentPool(creator) as pool:
@@ -53,6 +56,7 @@ def test_same_env_reuses_and_clears():
         assert api1.rebind_calls == ["req2"]  # rebound only on reuse, not on first acquire
 
 
+@scenarios.test_the_test
 def test_distinct_env_creates_separate_agents():
     creator = _FakeCreator()
     with WorkerAgentPool(creator) as pool:
@@ -63,6 +67,7 @@ def test_distinct_env_creates_separate_agents():
         assert len(creator.created_envs) == 2
 
 
+@scenarios.test_the_test
 def test_exit_tears_down_every_agent():
     creator = _FakeCreator()
     with WorkerAgentPool(creator) as pool:

tests/test_the_test/utils.py

@@ -1,9 +1,16 @@
 import json
 import subprocess
+from pathlib import Path
 
 from utils import logger
 
 
+def get_scenario_map() -> dict[str, list[str]]:
+    _execute_system_tests(scenario_map=True, print_stdout=False)
+    with Path("logs_mock_the_test/scenarios.json").open(encoding="utf-8") as f:
+        return json.load(f)
+
+
 def run_system_tests(
     scenario: str = "MOCK_THE_TEST",
     test_path: str | None = None,
@@ -14,6 +21,37 @@ def run_system_tests(
     xfail_strict: bool = False,
     env: dict[str, str] | None = None,
     expected_return_code: int = 0,
+) -> dict[str, dict]:
+    _execute_system_tests(
+        scenario=scenario,
+        test_path=test_path,
+        verbose=verbose,
+        forced_test=forced_test,
+        use_xdist=use_xdist,
+        xfail_strict=xfail_strict,
+        env=env,
+        expected_return_code=expected_return_code,
+    )
+
+    scenario = scenario or "DEFAULT"
+    with open(f"logs_{scenario.lower()}/feature_parity.json", encoding="utf-8") as f:
+        report = json.load(f)
+
+    return {test["path"]: test for test in report["tests"]}
+
+
+def _execute_system_tests(
+    scenario: str = "MOCK_THE_TEST",
+    test_path: str | None = None,
+    *,
+    verbose: bool = False,
+    forced_test: str | None = None,
+    use_xdist: bool = False,
+    xfail_strict: bool = False,
+    scenario_map: bool = False,
+    env: dict[str, str] | None = None,
+    print_stdout: bool = False,
+    expected_return_code: int = 0,
 ):
     cmd_parts = ["./run.sh"]
 
@@ -29,15 +67,12 @@ def run_system_tests(
         cmd_parts.extend(["-o", "xfail_strict=True"])
     if use_xdist:
         cmd_parts.extend(["-n=4"])
+    if scenario_map:
+        cmd_parts.extend(["--collect-only", "--scenario-report"])
 
     logger.info(" ".join(cmd_parts))
     result = subprocess.run(cmd_parts, capture_output=True, text=True, check=False, env=env)
 
-    logger.info(result.stdout)
+    if print_stdout:
+        logger.info(result.stdout)
     assert result.returncode == expected_return_code, f"Command failed with return code {result.returncode}"
-
-    scenario = scenario if scenario else "DEFAULT"
-    with open(f"logs_{scenario.lower()}/feature_parity.json", encoding="utf-8") as f:
-        report = json.load(f)
-
-    return {test["path"]: test for test in report["tests"]}