feat(cli,fill): make the eip version checker a separate pytest cli tool

.github/scripts/generate_eip_report.py

@@ -0,0 +1,147 @@
+"""Generate a markdown report of outdated EIP references from the EIP version checker output."""
+
+import os
+import re
+import sys
+import textwrap
+from string import Template
+
+# Report template using textwrap.dedent for clean multiline strings
+REPORT_TEMPLATE = Template(
+    textwrap.dedent("""\
+    # EIP Version Check Report
+
+    This automated check has detected that some EIP references in test files are outdated. This means that the EIPs have been updated in the [ethereum/EIPs](https://github.com/ethereum/EIPs) repository since our tests were last updated.
+
+    ## Outdated EIP References
+
+    ### Summary Table
+
+    | File | EIP Link | Referenced Version | Latest Version |
+    | ---- | -------- | ------------------ | -------------- |
+    $summary_table
+
+    ### Verbatim Failures
+
+    ```
+    $fail_messages
+    ```
+
+    ### Verbatim Errors
+
+    ```
+    $error_messages
+    ```
+
+    ## Action Required
+
+    1. Please verify whether the affected tests need updating based on changes in the EIP spec.
+    2. Update the `REFERENCE_SPEC_VERSION` in each file with the latest version shown above.
+    3. For detailed instructions, see the [reference specification documentation](https://eest.ethereum.org/main/writing_tests/reference_specification/).
+
+    ## Workflow Information
+
+    For more details, see the [workflow run](https://github.com/ethereum/execution-spec-tests/actions/runs/$run_id).
+""")  # noqa: E501
+)
+
+
+def extract_failures(output):
+    """Extract failure information from the output using regex."""
+    failures = []
+
+    for line in output.split("\n"):
+        if not line.startswith("FAILED"):
+            continue
+
+        # Extract test file path
+        file_match = re.search(r"FAILED (tests/[^:]+\.py)", line)
+        if not file_match:
+            continue
+        file_path = file_match.group(1)
+
+        # Extract EIP number
+        eip_match = re.search(r"eip(\d+)", file_path, re.IGNORECASE)
+        eip_num = f"EIP-{eip_match.group(1)}" if eip_match else "Unknown"
+
+        # Extract full path
+        full_path_match = re.search(r"from '([^']+)'", line)
+        full_path = full_path_match.group(1) if full_path_match else "Unknown"
+
+        # Extract EIP link
+        eip_link_match = re.search(r"Spec: (https://[^ ]+)\.", line)
+        eip_link = eip_link_match.group(1) if eip_link_match else ""
+        eip_link = eip_link.replace("blob/", "commits/") if eip_link else ""
+
+        # Extract versions
+        ref_version_match = re.search(r"Referenced version: ([a-f0-9]+)", line)
+        ref_version = ref_version_match.group(1) if ref_version_match else "Unknown"
+
+        latest_version_match = re.search(r"Latest version: ([a-f0-9]+)", line)
+        latest_version = latest_version_match.group(1) if latest_version_match else "Unknown"
+
+        failures.append((file_path, eip_num, full_path, eip_link, ref_version, latest_version))
+
+    return failures
+
+
+def generate_summary_table(failures):
+    """Generate a markdown summary table from the failures."""
+    rows = []
+    for file_path, eip_num, _, eip_link, ref_version, latest_version in failures:
+        rows.append(
+            f"| `{file_path}` | [{eip_num}]({eip_link}) | `{ref_version}` | `{latest_version}` |"
+        )
+    return "\n".join(rows)
+
+
+def main():
+    """Generate the report."""
+    if len(sys.argv) < 2:
+        print("Usage: uv run python generate_eip_report.py <input_file> [output_file]")
+        sys.exit(1)
+
+    input_file = sys.argv[1]
+    output_file = sys.argv[2] if len(sys.argv) > 2 else "./reports/outdated_eips.md"
+
+    try:
+        with open(input_file, "r") as f:
+            output = f.read()
+    except Exception as e:
+        print(f"Error reading input file: {e}")
+        sys.exit(1)
+
+    failures = extract_failures(output)
+
+    fail_messages = "\n".join(line for line in output.split("\n") if line.startswith("FAILED"))
+    if not fail_messages:
+        fail_messages = (
+            "No test failures were found in the pytest output: No lines start with 'FAILED'."
+        )
+
+    error_messages = "\n".join(line for line in output.split("\n") if line.startswith("ERROR"))
+    if not error_messages:
+        error_messages = (
+            "No test errors were found in the pytest output: No lines start with 'ERROR'."
+        )
+
+    report_content = REPORT_TEMPLATE.substitute(
+        summary_table=generate_summary_table(failures),
+        fail_messages=fail_messages,
+        error_messages=error_messages,
+        run_id=os.environ.get("GITHUB_RUN_ID", ""),
+    )
+
+    try:
+        with open(output_file, "w") as report:
+            report.write(report_content)
+    except Exception as e:
+        print(f"Error writing output file: {e}")
+        sys.exit(1)
+
+    print(f"Report generated successfully: {output_file}")
+    print(f"Found {len(failures)} outdated EIP references")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/check_eip_versions.yaml

@@ -0,0 +1,61 @@
+name: Check EIP Versions
+
+on:
+  repository_dispatch:
+  workflow_dispatch:
+  schedule:
+    - cron: "00 12 * * 1" # Run weekly on Mondays at 12:00 UTC
+
+jobs:
+  check_eip_versions:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write # required for peter-evans/create-issue-from-file
+      contents: read # needed for API access to GitHub
+    steps:
+      - name: Checkout ethereum/execution-spec-tests
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+
+      - name: Install uv ${{ vars.UV_VERSION }} and python ${{ matrix.python }}
+        uses: astral-sh/setup-uv@0c5e2b8115b80b4c7c5ddf6ffdd634974642d182
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+          version: ${{ vars.UV_VERSION }}
+          python-version: ${{ matrix.python }}
+
+      - name: Run EIP Version Checker
+        id: check-eip
+        continue-on-error: true
+        env:
+          # GitHub token provides API access for EIP version checking
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          mkdir -p ./reports
+          uv run check_eip_versions 2>&1 | tee ./reports/eip_check_output.txt
+          # Save the exit code but don't fail the workflow
+          exit_code=${PIPESTATUS[0]}
+          echo "exit_code=$exit_code" >> $GITHUB_OUTPUT
+          # Always return success to GitHub Actions
+          exit 0
+
+      - name: Generate report file
+        if: steps.check-eip.outputs.exit_code != 0
+        run: |
+          uv run python .github/scripts/generate_eip_report.py ./reports/eip_check_output.txt ./reports/outdated_eips.md
+
+      - name: Create Issue From File
+        if: steps.check-eip.outputs.exit_code != 0
+        uses: peter-evans/create-issue-from-file@e8ef132d6df98ed982188e460ebb3b5d4ef3a9cd
+        with:
+          title: "chore(tests): eip spec references outdated"
+          content-filepath: ./reports/outdated_eips.md
+          labels: report, automated issue, scope:tests, type:chore
+
+      - name: Upload test report as artifact
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: eip-check-report
+          path: ./reports/
+          retention-days: 30

docs/CHANGELOG.md

@@ -19,6 +19,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 ### 📋 Misc
 
 - ✨ Engine API updates for Osaka, add `get_blobs` rpc method ([#1510](https://github.com/ethereum/execution-spec-tests/pull/1510)).
+- ✨ The EIP Version checker has been moved from `fill` and `execute` to it's own command-line tool `check_eip_versions` that gets ran daily as a Github Action ([#1537](https://github.com/ethereum/execution-spec-tests/pull/1537)).
 
 ### 🧪 Test Cases
 

docs/gen_test_case_reference.py

@@ -48,7 +48,7 @@
     f"--until={GENERATE_UNTIL_FORK}",
     "--skip-index",
     "-m",
-    "(not blockchain_test_engine) and (not eip_version_check)",
+    "not blockchain_test_engine",
     "-s",
     test_arg,
 ]

docs/library/cli/index.md

@@ -1,4 +1,5 @@
 # EEST CLI Tools
 
+- [`check_eip_versions`](../../writing_tests/reference_specification.md) - A CLI tool to check the SHA values specified in EIP tests match the latest version from ethereum/EIPs.
 - [`eest`](eest.md) - A CLI tool that helps with routine tasks in ethereum/execution-spec-tests.
 - [`evm_bytes`](evm_bytes.md) - Convert the given EVM bytes from a binary file or a hex string to EEST's python opcodes.

docs/writing_tests/reference_specification.md

@@ -1,14 +1,21 @@
 # Referencing an EIP Spec Version
 
-An Ethereum Improvement Proposal ([ethereum/EIPs](https://github.com/ethereum/EIPs/tree/master/EIPS)) and its SHA digest can be directly referenced within a python test module in order to check whether the test implementation could be out-dated. The test framework automatically generates tests for every module that defines a spec version. If the spec is out-of-date because the SHA of the specified file in the remote repo changes, the corresponding `test_eip_spec_version()` test fails.
+Tests that implement features from an Ethereum Improvement Proposal ([ethereum/EIPs](https://github.com/ethereum/EIPs/tree/master/EIPS)) must define the EIP's markdown SHA digest within the test's Python module. This ensures our tests stay up-to-date with any changes to the EIP specifications.
+
+The `check_eip_versions` command-line utility automatically verifies that all EIP references in the codebase are current. It works by comparing the SHA specified in the test against the latest version in the ethereum/EIPs repository. This utility uses pytest to generate test cases for every module that includes "eip" in its path.
 
 <figure markdown>  <!-- markdownlint-disable MD033 (MD033=no-inline-html) -->
   ![Test framework summary for a failing EIP spec version test](./img/eip_reference_spec_console_output.png){ width=auto align=center}
-  `<-snip->`
-  ![EIP spec version test fail](./img/eip_reference_spec_console_output_fail.png){ width=auto align=center}
 </figure>
 
-!!! info ""
+!!! note "The SHA digest value is provided in the failure message of the corresponding test"
+
+    <figure markdown>  <!-- markdownlint-disable MD033 (MD033=no-inline-html) -->
+      ![EIP spec version test fail](./img/eip_reference_spec_console_output_fail.png){ width=auto align=center}
+    </figure>
+
+!!! info "Understanding and Retrieving the EIP Markdown's SHA Digest"
+
     The SHA value is the output from git's `hash-object` command, for example:
 
     ```console
@@ -18,7 +25,7 @@ An Ethereum Improvement Proposal ([ethereum/EIPs](https://github.com/ethereum/EI
     ```
 
     and can be retrieved from the remote repo via the Github API on the command-line as following:
-    
+
     ```console
     sudo apt install jq
     curl -s -H "Accept: application/vnd.github.v3+json" \
@@ -31,10 +38,34 @@ An Ethereum Improvement Proposal ([ethereum/EIPs](https://github.com/ethereum/EI
 
 This check accomplished by adding the following two global variables anywhere in the Python source file:
 
-| Variable Name               | Explanation                                                                                                                                                        |
-|-----------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `REFERENCE_SPEC_GIT_PATH`   | The relative path of the EIP markdown file in the [ethereum/EIPs](https://github.com/ethereum/EIPs/) repository, e.g. "`EIPS/eip-1234.md`"                         |
-| `REFERENCE_SPEC_VERSION`    | The SHA hash of the latest version of the file retrieved from the Github API:<br>`https://api.github.com/repos/ethereum/EIPs/contents/EIPS/eip-<EIP Number>.md`    |
+| Variable Name             | Explanation                                                                                                                                                     |
+| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `REFERENCE_SPEC_GIT_PATH` | The relative path of the EIP markdown file in the [ethereum/EIPs](https://github.com/ethereum/EIPs/) repository, e.g. "`EIPS/eip-1234.md`"                      |
+| `REFERENCE_SPEC_VERSION`  | The SHA hash of the latest version of the file retrieved from the Github API:<br>`https://api.github.com/repos/ethereum/EIPs/contents/EIPS/eip-<EIP Number>.md` |
+
+## Running the `check_eip_versions` Command Locally
+
+A Github Personal Access Token (PAT) is required to avoid rate-limiting issues when using the Github API. The token can be specified via an environment variable or via the command-line. For example, the Github CLI can be used to obtain a token:
+
+```bash
+uv run check_eip_versions --github-token=$(gh auth token)
+```
+
+or a PAT can be created at: https://github.com/settings/personal-access-tokens/new.
+
+By default, only tests up to and including the current fork under development will be checked. This is controlled by the `UNTIL_FORK` setting in the `src/config/check_eip_versions.py` configuration file. You can also pass a specific test path to limit the scope:
+
+```shell
+uv run check_eip_versions --github-token=$(gh auth token) tests/shanghai/eip3651_warm_coinbase/
+```
+
+This would only check EIP versions for the EIP-3651 tests in the `shanghai/eip3651_warm_coinbase` sub-directory.
+
+## Automated Checks via GitHub Actions
+
+The repository includes a [GitHub Actions workflow](https://github.com/ethereum/execution-spec-tests/actions/workflows/check_eip_versions.yaml) that automatically runs `check_eip_versions` on a daily schedule. If any outdated EIP references are detected, the workflow creates an issue in the repository with details about which references need to be updated.
+
+This workflow uses GitHub's built-in token for authentication, so there's no need to configure personal access tokens for the automated checks. The issue will include links to the relevant workflow run and details about which tests need updating.
 
 ## Example
 

pyproject.toml

@@ -95,6 +95,7 @@ phil = "cli.pytest_commands.fill:phil"
 execute = "cli.pytest_commands.execute:execute"
 attac = "cli.pytest_commands.execute:execute"
 checkfixtures = "cli.check_fixtures:check_fixtures"
+check_eip_versions = "cli.pytest_commands.check_eip_versions:check_eip_versions"
 consume = "cli.pytest_commands.consume:consume"
 protec = "cli.pytest_commands.consume:consume"
 genindex = "cli.gen_index:generate_fixtures_index_cli"

pytest-check-eip-versions.ini

@@ -0,0 +1,20 @@
+[pytest]
+console_output_style = count
+minversion = 7.0
+python_files = *.py
+testpaths = tests/
+markers =
+    slow
+    pre_alloc_modify
+    eip_version_check
+addopts =  
+    -p pytest_plugins.spec_version_checker.spec_version_checker
+    -p pytest_plugins.concurrency
+    -p pytest_plugins.filler.pre_alloc
+    -p pytest_plugins.solc.solc
+    -p pytest_plugins.filler.filler
+    -p pytest_plugins.shared.execute_fill
+    -p pytest_plugins.forks.forks
+    -p pytest_plugins.help.help
+    -m eip_version_check
+    --tb short

pytest-execute-hive.ini

@@ -15,10 +15,8 @@ addopts =
     -p pytest_plugins.execute.execute
     -p pytest_plugins.shared.execute_fill
     -p pytest_plugins.forks.forks
-    -p pytest_plugins.spec_version_checker.spec_version_checker
     -p pytest_plugins.pytest_hive.pytest_hive
     -p pytest_plugins.help.help
-    -m "not eip_version_check"
     --tb short
     --dist loadscope
     --ignore tests/cancun/eip4844_blobs/point_evaluation_vectors/

pytest-execute.ini

@@ -16,9 +16,7 @@ addopts =
     -p pytest_plugins.execute.rpc.remote_seed_sender
     -p pytest_plugins.execute.rpc.remote
     -p pytest_plugins.forks.forks
-    -p pytest_plugins.spec_version_checker.spec_version_checker
     -p pytest_plugins.help.help
-    -m "not eip_version_check"
     --tb short
     --dist loadscope
     --ignore tests/cancun/eip4844_blobs/point_evaluation_vectors/

pytest.ini

@@ -14,10 +14,8 @@ addopts =
     -p pytest_plugins.filler.static_filler
     -p pytest_plugins.shared.execute_fill
     -p pytest_plugins.forks.forks
-    -p pytest_plugins.spec_version_checker.spec_version_checker
     -p pytest_plugins.eels_resolver
     -p pytest_plugins.help.help
-    -m "not eip_version_check"
     --tb short
     --ignore tests/cancun/eip4844_blobs/point_evaluation_vectors/
 # these customizations require the pytest-custom-report plugin

src/cli/pytest_commands/check_eip_versions.py

@@ -0,0 +1,22 @@
+"""CLI entry point for the EIP version checker pytest-based command."""
+
+import sys
+from typing import List
+
+import click
+import pytest
+
+from config.check_eip_versions import CheckEipVersionsConfig
+
+from .common import common_click_options, handle_help_flags
+
+
+@click.command(context_settings={"ignore_unknown_options": True})
+@common_click_options
+def check_eip_versions(pytest_args: List[str], **kwargs) -> None:
+    """Run pytest with the `spec_version_checker` plugin."""
+    args = ["-c", "pytest-check-eip-versions.ini"]
+    args += ["--until", CheckEipVersionsConfig().UNTIL_FORK]
+    args += handle_help_flags(list(pytest_args), pytest_type="check-eip-versions")
+    result = pytest.main(args)
+    sys.exit(result)