feat(test-commands): Allow pytest valid fork markers as params; add unit tests

fselmo · fselmo · commit c69fff690bde · 2026-01-12T16:25:48.000-07:00
diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/forks.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/forks.py
@@ -1208,3 +1208,54 @@ def parametrize_fork(
     metafunc.parametrize(
         param_names, param_values, scope="function", indirect=indirect
     )
+
+
+def pytest_collection_modifyitems(
+    config: pytest.Config, items: List[pytest.Item]
+) -> None:
+    """
+    Filter tests based on param-level validity markers.
+
+    The pytest_generate_tests hook only considers function-level validity markers.
+    This hook runs after parametrization and can access all markers including
+    param-level ones, allowing us to properly filter tests based on param-level
+    valid_from/valid_until markers.
+    """
+    items_to_remove = []
+
+    for i, item in enumerate(items):
+        # Get fork from params if available
+        params = None
+        if hasattr(item, "callspec"):
+            params = item.callspec.params
+        elif hasattr(item, "params"):
+            params = item.params
+
+        if not params or "fork" not in params or params["fork"] is None:
+            continue
+
+        fork: Fork = params["fork"]
+
+        # Get all markers including param-level ones
+        markers = item.iter_markers()
+
+        # Calculate valid fork set from all markers
+        # If this raises (e.g., duplicate markers from combining function-level
+        # and param-level), exit immediately with error
+        try:
+            valid_fork_set = ValidityMarker.get_test_fork_set_from_markers(
+                markers
+            )
+        except Exception as e:
+            pytest.exit(
+                f"Error in test '{item.name}': {e}",
+                returncode=pytest.ExitCode.USAGE_ERROR,
+            )
+
+        # If the fork is not in the valid set, mark for removal
+        if fork not in valid_fork_set:
+            items_to_remove.append(i)
+
+    # Remove items in reverse order to maintain indices
+    for i in reversed(items_to_remove):
+        del items[i]
diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/tests/test_bad_validity_markers.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/tests/test_bad_validity_markers.py
@@ -236,3 +236,73 @@ def test_invalid_validity_markers(
         errors=1,
     )
     assert error_string in "\n".join(result.stdout.lines)
+
+
+# --- Tests for param-level marker errors --- #
+
+
+param_level_marker_error_test_cases = (
+    (
+        "param_level_valid_from_with_function_level_valid_from",
+        (
+            """
+            import pytest
+            @pytest.mark.parametrize(
+                "value",
+                [
+                    pytest.param(True, marks=pytest.mark.valid_from("Paris")),
+                ],
+            )
+            @pytest.mark.valid_from("Berlin")
+            def test_case(state_test, value):
+                assert 1
+            """,
+            "Too many 'valid_from' markers applied to test",
+        ),
+    ),
+    (
+        "param_level_valid_until_with_function_level_valid_until",
+        (
+            """
+            import pytest
+            @pytest.mark.parametrize(
+                "value",
+                [
+                    pytest.param(True, marks=pytest.mark.valid_until("Cancun")),
+                ],
+            )
+            @pytest.mark.valid_until("Prague")
+            def test_case(state_test, value):
+                assert 1
+            """,
+            "Too many 'valid_until' markers applied to test",
+        ),
+    ),
+)
+
+
+@pytest.mark.parametrize(
+    "test_function, error_string",
+    [test_case for _, test_case in param_level_marker_error_test_cases],
+    ids=[test_id for test_id, _ in param_level_marker_error_test_cases],
+)
+def test_param_level_marker_errors(
+    pytester: pytest.Pytester, error_string: str, test_function: str
+) -> None:
+    """
+    Test that combining function-level and param-level validity markers
+    of the same type produces an error.
+
+    Unlike function-level errors (caught during test generation), param-level
+    errors are caught during collection and cause pytest to exit immediately.
+    """
+    pytester.makepyfile(test_function)
+    pytester.copy_example(
+        name="src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini"
+    )
+    result = pytester.runpytest("-c", "pytest-fill.ini")
+
+    # pytest.exit() causes the run to terminate with no test outcomes
+    assert result.ret != 0, "Expected non-zero exit code"
+    stdout = "\n".join(result.stdout.lines)
+    assert error_string in stdout, f"Expected '{error_string}' in output"
diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/tests/test_markers.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/forks/tests/test_markers.py
@@ -222,3 +222,182 @@ def test_fork_markers(
         *pytest_args,
     )
     result.assert_outcomes(**outcomes)
+
+
+# --- Tests for param-level validity markers --- #
+
+
+def generate_param_level_marker_test() -> str:
+    """Generate a test function with param-level fork validity markers."""
+    return """
+import pytest
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        pytest.param(
+            True,
+            id="from_tangerine",
+            marks=pytest.mark.valid_from("TangerineWhistle"),
+        ),
+        pytest.param(
+            False,
+            id="from_paris",
+            marks=pytest.mark.valid_from("Paris"),
+        ),
+    ],
+)
+@pytest.mark.state_test_only
+def test_param_level_valid_from(state_test, value):
+    pass
+"""
+
+
+def generate_param_level_valid_until_test() -> str:
+    """Generate a test function with param-level valid_until markers."""
+    return """
+import pytest
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        pytest.param(
+            True,
+            id="until_cancun",
+            marks=pytest.mark.valid_until("Cancun"),
+        ),
+        pytest.param(
+            False,
+            id="until_paris",
+            marks=pytest.mark.valid_until("Paris"),
+        ),
+    ],
+)
+@pytest.mark.state_test_only
+def test_param_level_valid_until(state_test, value):
+    pass
+"""
+
+
+def generate_param_level_mixed_test() -> str:
+    """Generate a test with both function-level and param-level markers."""
+    return """
+import pytest
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        pytest.param(
+            True,
+            id="all_forks",
+            marks=pytest.mark.valid_from("TangerineWhistle"),
+        ),
+        pytest.param(
+            False,
+            id="paris_only",
+            marks=pytest.mark.valid_from("Paris"),
+        ),
+    ],
+)
+@pytest.mark.valid_until("Cancun")
+@pytest.mark.state_test_only
+def test_mixed_function_and_param_markers(state_test, value):
+    pass
+"""
+
+
+@pytest.mark.parametrize(
+    "test_function,pytest_args,outcomes",
+    [
+        pytest.param(
+            generate_param_level_marker_test(),
+            ["--from=Paris", "--until=Cancun"],
+            # from_tangerine: Paris, Shanghai, Cancun = 3 forks
+            # from_paris: Paris, Shanghai, Cancun = 3 forks
+            # Total: 6 tests
+            {"passed": 6, "failed": 0, "skipped": 0, "errors": 0},
+            id="param_level_valid_from_paris_to_cancun",
+        ),
+        pytest.param(
+            generate_param_level_marker_test(),
+            ["--from=Berlin", "--until=Shanghai"],
+            # from_tangerine: Berlin, London, Paris, Shanghai = 4 forks
+            # from_paris: Paris, Shanghai = 2 forks
+            # Total: 6 tests
+            {"passed": 6, "failed": 0, "skipped": 0, "errors": 0},
+            id="param_level_valid_from_berlin_to_shanghai",
+        ),
+        pytest.param(
+            generate_param_level_marker_test(),
+            ["--from=Berlin", "--until=London"],
+            # from_tangerine: Berlin, London = 2 forks
+            # from_paris: none (Paris > London)
+            # Total: 2 tests
+            {"passed": 2, "failed": 0, "skipped": 0, "errors": 0},
+            id="param_level_valid_from_berlin_to_london",
+        ),
+        pytest.param(
+            generate_param_level_valid_until_test(),
+            ["--from=Paris", "--until=Prague"],
+            # until_cancun: Paris, Shanghai, Cancun = 3 forks
+            # until_paris: Paris = 1 fork
+            # Total: 4 tests
+            {"passed": 4, "failed": 0, "skipped": 0, "errors": 0},
+            id="param_level_valid_until_paris_to_prague",
+        ),
+        pytest.param(
+            generate_param_level_valid_until_test(),
+            ["--from=Shanghai", "--until=Prague"],
+            # until_cancun: Shanghai, Cancun = 2 forks
+            # until_paris: none (Shanghai > Paris)
+            # Total: 2 tests
+            {"passed": 2, "failed": 0, "skipped": 0, "errors": 0},
+            id="param_level_valid_until_shanghai_to_prague",
+        ),
+        pytest.param(
+            generate_param_level_mixed_test(),
+            ["--from=Berlin", "--until=Prague"],
+            # Function marker: valid_until("Cancun") limits to <= Cancun
+            # all_forks (TangerineWhistle): Berlin, London, Paris, Shanghai, Cancun = 5
+            # paris_only: Paris, Shanghai, Cancun = 3
+            # Total: 8 tests
+            {"passed": 8, "failed": 0, "skipped": 0, "errors": 0},
+            id="mixed_markers_berlin_to_prague",
+        ),
+        pytest.param(
+            generate_param_level_mixed_test(),
+            ["--from=Paris", "--until=Shanghai"],
+            # Function marker: valid_until("Cancun") limits to <= Cancun
+            # Command line: --until=Shanghai further limits to <= Shanghai
+            # all_forks: Paris, Shanghai = 2 forks
+            # paris_only: Paris, Shanghai = 2 forks
+            # Total: 4 tests
+            {"passed": 4, "failed": 0, "skipped": 0, "errors": 0},
+            id="mixed_markers_paris_to_shanghai",
+        ),
+    ],
+)
+def test_param_level_validity_markers(
+    pytester: pytest.Pytester,
+    test_function: str,
+    outcomes: dict,
+    pytest_args: List[str],
+) -> None:
+    """
+    Test param-level validity markers (valid_from, valid_until on pytest.param).
+
+    The pytest_collection_modifyitems hook filters tests based on param-level
+    markers after parametrization, allowing different parameter values to have
+    different fork validity ranges.
+    """
+    pytester.makepyfile(test_function)
+    pytester.copy_example(
+        name="src/execution_testing/cli/pytest_commands/pytest_ini_files/pytest-fill.ini"
+    )
+    result = pytester.runpytest(
+        "-c",
+        "pytest-fill.ini",
+        "-v",
+        *pytest_args,
+    )
+    result.assert_outcomes(**outcomes)
diff --git a/tests/tangerine_whistle/eip150_operation_gas_costs/test_eip150_selfdestruct.py b/tests/tangerine_whistle/eip150_operation_gas_costs/test_eip150_selfdestruct.py
@@ -29,7 +29,11 @@
 from execution_testing import (
     Macros as Om,
 )
-from execution_testing.forks import Berlin, Cancun, SpuriousDragon
+from execution_testing.forks import (
+    Berlin,
+    Cancun,
+    SpuriousDragon,
+)
 from execution_testing.forks.helpers import Fork
 
 from .spec import ref_spec_150
@@ -306,7 +310,9 @@ def build_post_state(
 @pytest.mark.parametrize(
     "warm",
     [
-        pytest.param(False, id="cold"),
+        pytest.param(
+            False, id="cold", marks=pytest.mark.valid_from("TangerineWhistle")
+        ),
         pytest.param(True, id="warm", marks=pytest.mark.valid_from("Berlin")),
     ],
 )
@@ -323,7 +329,6 @@ def build_post_state(
     [0, 1],
     ids=["dead_beneficiary", "alive_beneficiary"],
 )
-@pytest.mark.valid_from("TangerineWhistle")
 def test_selfdestruct_to_account(
     pre: Alloc,
     blockchain_test: BlockchainTestFiller,
@@ -429,7 +434,9 @@ def test_selfdestruct_to_account(
 @pytest.mark.parametrize(
     "warm",
     [
-        pytest.param(False, id="cold"),
+        pytest.param(
+            False, id="cold", marks=pytest.mark.valid_from("TangerineWhistle")
+        ),
         pytest.param(True, id="warm", marks=pytest.mark.valid_from("Berlin")),
     ],
 )
@@ -446,7 +453,6 @@ def test_selfdestruct_to_account(
     [0, 1],
     ids=["dead_beneficiary", "alive_beneficiary"],
 )
-@pytest.mark.valid_from("TangerineWhistle")
 def test_selfdestruct_state_access_boundary(
     pre: Alloc,
     blockchain_test: BlockchainTestFiller,
