From 1580c75a2cddbcddc4cb6639422b0d208f30bdac Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 16 Jul 2026 00:57:00 +0200 Subject: [PATCH 1/6] chore(test-forks): add release-information cache robustness tests Add tests for `get_release_information`'s caching behavior around the GitHub API rate limit: - A pinned version (immutable tag) that a stale cache already resolves must not trigger an API refresh. - A rate-limited refresh must fall back to the stale cache instead of deleting it and crashing. - A corrupt cache file must be re-downloaded, not crash every run. - `GITHUB_TOKEN`, when set, must authenticate API requests. The four tests covering the new behavior fail without the fix. --- .../plugins/consume/tests/test_releases.py | 264 +++++++++++++++++- 1 file changed, 263 insertions(+), 1 deletion(-) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py index cb347efc946..450c63e0af7 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py @@ -1,15 +1,23 @@ """Test release parsing given the github repository release JSON data.""" +import os +import shutil +import time from os.path import realpath from pathlib import Path -from typing import List +from typing import Any, Dict, List import pytest +import requests +from .. import releases from ..releases import ( SUPPORTED_REPOS, NoSuchReleaseError, ReleaseInformation, + download_release_information, + get_release_page_url, + get_release_url, get_release_url_from_release_information, is_release_url, parse_release_information_from_file, @@ -221,3 +229,257 @@ def test_supported_repos_contains_execution_specs() -> None: `tests-bal@v7.1.0` onward) and must be in `SUPPORTED_REPOS`. """ assert "ethereum/execution-specs" in SUPPORTED_REPOS + + +class FakeResponse: + """A minimal stand-in for `requests.Response`.""" + + def __init__( + self, payload: List[Dict], rate_limited: bool = False + ) -> None: + """Initialize with a JSON payload or a rate-limited failure.""" + self.payload = payload + self.rate_limited = rate_limited + self.headers: Dict[str, str] = {} + + def json(self) -> List[Dict]: + """Return the JSON payload.""" + return self.payload + + def raise_for_status(self) -> None: + """Raise an `HTTPError` if the response is rate-limited.""" + if self.rate_limited: + raise requests.exceptions.HTTPError( + "403 Client Error: rate limit exceeded" + ) + + +def fake_release(tag_name: str, asset_name: str) -> Dict: + """Build a minimal GitHub API release entry.""" + encoded_tag = tag_name.replace("@", "%40") + return { + "html_url": "https://github.com/ethereum/execution-specs/releases/" + f"tag/{encoded_tag}", + "id": 1, + "tag_name": tag_name, + "name": tag_name, + "created_at": "2026-07-15T00:00:00Z", + "published_at": "2026-07-15T00:00:00Z", + "assets": [ + { + "browser_download_url": "https://github.com/ethereum/" + f"execution-specs/releases/download/{encoded_tag}/" + f"{asset_name}", + "id": 1, + "name": asset_name, + "content_type": "application/gzip", + "size": 1, + } + ], + } + + +@pytest.fixture +def release_information_cache( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> Path: + """ + Point the release-information cache at a copy of the test manifest. + + Also disable the CI/Docker detection so the freshness check applies + (in CI, the cache never expires). + """ + cache_file = tmp_path / "release_information.json" + shutil.copyfile(CURRENT_FOLDER / "release_information.json", cache_file) + monkeypatch.setattr( + releases, "CACHED_RELEASE_INFORMATION_FILE", cache_file + ) + monkeypatch.setattr(releases, "is_docker_or_ci", lambda: False) + return cache_file + + +def make_stale(cache_file: Path) -> None: + """Age the cache file's mtime beyond the 4-hour freshness window.""" + stale_time = time.time() - 5 * 60 * 60 + os.utime(cache_file, (stale_time, stale_time)) + + +def block_api(monkeypatch: pytest.MonkeyPatch) -> None: + """Make any GitHub API request fail the test.""" + + def no_api(*args: Any, **kwargs: Any) -> None: + del args, kwargs + pytest.fail("The GitHub API must not be hit") + + monkeypatch.setattr(releases.requests, "get", no_api) + + +def rate_limited_get(*args: Any, **kwargs: Any) -> FakeResponse: + """Return a rate-limited (403) GitHub API response.""" + del args, kwargs + return FakeResponse([], rate_limited=True) + + +def new_release_get(*args: Any, **kwargs: Any) -> FakeResponse: + """Return a single-page response with a new `tests@v21.0.0` release.""" + del args, kwargs + return FakeResponse([fake_release("tests@v21.0.0", "fixtures.tar.gz")]) + + +def test_pinned_release_resolves_from_stale_cache_without_api( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + A pinned version already resolvable from the cache must not refresh. + + Release tags are immutable, so a cached entry for an exact version + cannot be outdated, no matter how old the cache file is. Regression + test for `consume --input=tests@vX.Y.Z` raising INTERNALERROR when + the unauthenticated GitHub API rate limit is exhausted, even though + the (stale) cache resolved the release. + """ + make_stale(release_information_cache) + block_api(monkeypatch) + assert get_release_url("tests@v20.0.0") == ( + "https://github.com/ethereum/execution-specs/releases/download/" + "tests%40v20.0.0/fixtures.tar.gz" + ) + assert get_release_page_url("tests@v20.0.0") == ( + "https://github.com/ethereum/execution-specs/releases/tag/" + "tests%40v20.0.0" + ) + assert release_information_cache.exists() + + +def test_fresh_cache_resolves_latest_without_api( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A fresh cache resolves unpinned lookups without an API request.""" + del release_information_cache + block_api(monkeypatch) + assert get_release_url("tests@latest").endswith( + "tests%40v20.0.0/fixtures.tar.gz" + ) + + +def test_rate_limited_refresh_falls_back_to_stale_cache( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + A failed refresh must fall back to the stale cache, not delete it. + + Previously the stale cache file was deleted before the download was + attempted, so a rate-limited refresh crashed the run and left no + cache at all, forcing every subsequent run onto the API. + """ + make_stale(release_information_cache) + monkeypatch.setattr(releases.requests, "get", rate_limited_get) + assert get_release_url("tests@latest").endswith( + "tests%40v20.0.0/fixtures.tar.gz" + ) + assert release_information_cache.exists() + + +def test_rate_limited_refresh_without_cache_raises( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Without a cache file, a failed refresh is a hard error.""" + monkeypatch.setattr( + releases, + "CACHED_RELEASE_INFORMATION_FILE", + tmp_path / "release_information.json", + ) + monkeypatch.setattr(releases, "is_docker_or_ci", lambda: False) + monkeypatch.setattr(releases.requests, "get", rate_limited_get) + with pytest.raises(requests.exceptions.HTTPError): + get_release_url("tests@latest") + + +def test_unpinned_release_refreshes_stale_cache( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + An unpinned lookup with a stale cache must refresh from the API. + + `latest` and bare feature names can resolve to a newer release at + any time, so the pinned-release fast path must not apply to them. + """ + make_stale(release_information_cache) + calls: List[str] = [] + + def fake_get(url: str, **kwargs: Any) -> FakeResponse: + del kwargs + calls.append(url) + return FakeResponse([fake_release("tests@v21.0.0", "fixtures.tar.gz")]) + + monkeypatch.setattr(releases.requests, "get", fake_get) + assert get_release_url("tests@latest").endswith( + "tests%40v21.0.0/fixtures.tar.gz" + ) + assert len(calls) == len(SUPPORTED_REPOS) + assert "tests@v21.0.0" in release_information_cache.read_text() + + +def test_pinned_release_not_in_stale_cache_refreshes( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + A pinned version missing from the stale cache must refresh. + + The pinned-release fast path only applies when the cache already + resolves the requested version. + """ + make_stale(release_information_cache) + monkeypatch.setattr(releases.requests, "get", new_release_get) + assert get_release_url("tests@v21.0.0").endswith( + "tests%40v21.0.0/fixtures.tar.gz" + ) + + +def test_corrupt_cache_file_is_refreshed( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + A corrupt cache file must be re-downloaded, not crash the run. + + A partially-written download (e.g. a killed process) must not wedge + every subsequent run until the file is manually deleted. + """ + release_information_cache.write_text("{ not json") + monkeypatch.setattr(releases.requests, "get", new_release_get) + assert get_release_url("tests@v21.0.0").endswith( + "tests%40v21.0.0/fixtures.tar.gz" + ) + + +@pytest.mark.parametrize("github_token", [None, "ghp_test_token"]) +def test_download_release_information_github_token( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + github_token: str | None, +) -> None: + """ + Authenticate GitHub API requests iff `GITHUB_TOKEN` is set. + + Authenticated requests get 5000 requests/hour instead of the + unauthenticated 60 requests/hour per IP. + """ + if github_token is None: + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + else: + monkeypatch.setenv("GITHUB_TOKEN", github_token) + seen_headers: List[Dict[str, str]] = [] + + def fake_get(url: str, **kwargs: Any) -> FakeResponse: + del url + seen_headers.append(kwargs.get("headers") or {}) + return FakeResponse([fake_release("tests@v21.0.0", "fixtures.tar.gz")]) + + monkeypatch.setattr(releases.requests, "get", fake_get) + download_release_information(tmp_path / "release_information.json") + expected_headers = ( + {} + if github_token is None + else {"Authorization": f"Bearer {github_token}"} + ) + assert seen_headers == [expected_headers] * len(SUPPORTED_REPOS) From cff68f9406dc070596a3b9290f597a5abcf1a609 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 16 Jul 2026 01:05:34 +0200 Subject: [PATCH 2/6] fix(consume): make release resolution robust to GitHub API rate limits Previously `get_release_information` deleted a stale (>4 hours old) release-information cache file before attempting to re-download it, so a failed refresh (e.g. the unauthenticated GitHub API rate limit of 60 requests/hour per IP was exhausted) crashed the run and left no cache behind, forcing every subsequent run onto the API. - Resolve pinned versions (e.g. `tests@v20.0.1`) from the cache regardless of its age and without any API requests: Release tags are immutable, so a cached entry cannot be outdated. - Keep the stale cache file as a fallback when the refresh fails, instead of deleting it upfront. - Re-download a corrupt cache file instead of crashing; write the cache via a temporary file so concurrent readers never see a partial write. - Authenticate API requests with `GITHUB_TOKEN` when set (5000 requests/hour instead of 60). - Pass the release spec instead of the asset URL to `get_release_page_url` so the release-page lookup also benefits from the pinned-version fast path. --- .../plugins/consume/consume.py | 2 +- .../plugins/consume/releases.py | 102 ++++++++++++++---- .../tests/test_fixtures_source_input_types.py | 9 +- 3 files changed, 88 insertions(+), 25 deletions(-) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py index 62ddb494040..7f4dc7f3840 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py @@ -265,7 +265,7 @@ def from_release_spec( if cache_folder is None: cache_folder = CACHED_DOWNLOADS_DIRECTORY url = get_release_url(spec) - release_page = get_release_page_url(url) + release_page = get_release_page_url(spec) destination_folder = extract_to or FixtureDownloader.get_cache_path( url, cache_folder diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py index f74f68b382e..9a16d79894b 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py @@ -1,6 +1,7 @@ """Procedures to consume fixtures from Github releases.""" import json +import logging import os import re from dataclasses import dataclass @@ -11,7 +12,9 @@ import platformdirs import requests -from pydantic import BaseModel, Field, RootModel +from pydantic import BaseModel, Field, RootModel, ValidationError + +logger = logging.getLogger(__name__) CACHED_RELEASE_INFORMATION_FILE = ( Path(platformdirs.user_cache_dir("ethereum-execution-spec-tests")) @@ -237,7 +240,15 @@ def download_release_information( pagination links up to `max_pages` pages, so resolution sees the 200 most recent releases per repo. Older releases fall outside this window and cannot be resolved. + + Authenticate with `GITHUB_TOKEN` when set: authenticated requests + get 5000 requests/hour instead of the unauthenticated 60/hour per + IP address. """ + headers = {} + github_token = os.environ.get("GITHUB_TOKEN") + if github_token: + headers["Authorization"] = f"Bearer {github_token}" all_releases = [] for repo in SUPPORTED_REPOS: current_url: str | None = ( @@ -246,7 +257,7 @@ def download_release_information( max_pages = 2 while current_url and max_pages > 0: max_pages -= 1 - response = requests.get(current_url) + response = requests.get(current_url, headers=headers) response.raise_for_status() all_releases.extend(response.json()) current_url = None @@ -260,8 +271,14 @@ def download_release_information( if destination_file: destination_file.parent.mkdir(parents=True, exist_ok=True) - with open(destination_file, "w") as file: + # Write via a temporary file so a concurrent reader never sees a + # partially-written cache. + temporary_file = destination_file.with_name( + destination_file.name + ".tmp" + ) + with open(temporary_file, "w") as file: json.dump(all_releases, file) + temporary_file.replace(destination_file) return parse_release_information(all_releases) @@ -310,6 +327,26 @@ def sort_key( return max(matches, key=sort_key) +def resolves_pinned_release( + release_string: str, + release_information: List[ReleaseInformation], +) -> bool: + """ + Check whether the release information resolves a pinned version. + + A release descriptor with an explicit version refers to an immutable + git tag: once it resolves, a refresh of the release information + cannot change the result. + """ + if ReleaseTag.from_string(release_string).version is None: + return False + try: + find_release(release_string, release_information) + except NoSuchReleaseError: + return False + return True + + def get_release_url_from_release_information( release_string: str, release_information: List[ReleaseInformation] ) -> str: @@ -329,7 +366,7 @@ def get_release_page_url(release_string: str) -> str: "https://github.com/ethereum/execution-specs/releases/ download/tests%40v20.0.0/fixtures.tar.gz"). """ - release_information = get_release_information() + release_information = get_release_information(release_string) # Case 1: If it's a direct GitHub Releases download link, find which # release in `release_information` has an asset with this exact URL. @@ -349,32 +386,57 @@ def get_release_page_url(release_string: str) -> str: return find_release(release_string, release_information).url -def get_release_information() -> List[ReleaseInformation]: +def get_release_information( + release_string: str | None = None, +) -> List[ReleaseInformation]: """ - Get the release information. - - First check if the cached release information file exists. If it does, but - it is older than 4 hours, delete the file, unless running inside a CI - environment or a Docker container. Then download the release information - from the Github API and save it to the cache file. + Get the release information, refreshing the cache file as needed. + + Return the cached release information if the cache file is fresh + (younger than 4 hours; any age when running inside a CI environment + or a Docker container). A stale cache is also used without + refreshing when `release_string` pins an exact version that the + cache already resolves: release tags are immutable, so the cached + entry cannot be outdated. Otherwise re-download the release + information, keeping the stale cache as a fallback in case the + GitHub API is unavailable (e.g. rate-limited). """ + cached_information: List[ReleaseInformation] | None = None if CACHED_RELEASE_INFORMATION_FILE.exists(): - last_modified = CACHED_RELEASE_INFORMATION_FILE.stat().st_mtime - if ( - datetime.now().timestamp() - last_modified - ) < 4 * 60 * 60 or is_docker_or_ci(): - return parse_release_information_from_file( + try: + cached_information = parse_release_information_from_file( CACHED_RELEASE_INFORMATION_FILE ) - CACHED_RELEASE_INFORMATION_FILE.unlink() - if not CACHED_RELEASE_INFORMATION_FILE.exists(): + except (json.JSONDecodeError, ValidationError): + logger.warning( + "Ignoring corrupt release information cache at " + f"{CACHED_RELEASE_INFORMATION_FILE}." + ) + else: + last_modified = CACHED_RELEASE_INFORMATION_FILE.stat().st_mtime + cache_age = datetime.now().timestamp() - last_modified + if cache_age < 4 * 60 * 60 or is_docker_or_ci(): + return cached_information + if release_string is not None and resolves_pinned_release( + release_string, cached_information + ): + return cached_information + try: return download_release_information(CACHED_RELEASE_INFORMATION_FILE) - return parse_release_information_from_file(CACHED_RELEASE_INFORMATION_FILE) + except requests.RequestException as error: + if cached_information is None: + raise + logger.warning( + f"Could not refresh release information from the GitHub API " + f"({error}); falling back to the stale cache at " + f"{CACHED_RELEASE_INFORMATION_FILE}." + ) + return cached_information def get_release_url(release_string: str) -> str: """Get the URL for a specific release.""" - release_information = get_release_information() + release_information = get_release_information(release_string) return get_release_url_from_release_information( release_string, release_information ) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py index 5a5418006d3..fa7cb7ffce4 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py @@ -58,11 +58,12 @@ def test_fixtures_source_from_release_spec_makes_api_calls(self) -> None: source = FixturesSource.from_release_spec(test_spec) - # Verify API calls were made and release page is set + # Verify API calls were made and release page is set. + # Both lookups receive the release spec, so pinned + # versions resolve from the release-information cache + # without querying the GitHub API. mock_get_url.assert_called_once_with(test_spec) - mock_get_page.assert_called_once_with( - "https://github.com/ethereum/execution-specs/releases/download/tests%40v20.0.0/fixtures.tar.gz" - ) + mock_get_page.assert_called_once_with(test_spec) assert ( source.release_page == "https://github.com/ethereum/execution-specs/releases/tag/tests%40v20.0.0" From ee38410752390e0602917ff734505fe49cd77078 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 16 Jul 2026 01:40:13 +0200 Subject: [PATCH 3/6] refactor(consume): resolve release descriptors once per run `FixturesSource.from_release_spec` resolved the same spec twice, once via `get_release_url` and once via `get_release_page_url`, so the cached release information was parsed twice and, when the cache was stale and the GitHub API unavailable, the doomed refresh was attempted twice (with a duplicate fallback warning). - Add `resolve_release` and derive both the asset download URL and the release page from a single resolution in `from_release_spec`; `get_release_url`/`get_release_page_url` remain as thin wrappers. - Drop `get_release_page_url`'s direct asset-URL branch: It became unreachable when its only caller switched to passing the release spec, and it duplicated `is_release_url`'s regex. - De-duplicate the canned API response and the cache-redirection monkeypatching in the release tests. --- .../plugins/consume/consume.py | 10 ++-- .../plugins/consume/releases.py | 44 +++++--------- .../tests/test_fixtures_source_input_types.py | 57 ++++++++++--------- .../plugins/consume/tests/test_releases.py | 29 +++++----- 4 files changed, 64 insertions(+), 76 deletions(-) diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py index 7f4dc7f3840..339904a4d14 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py @@ -43,10 +43,9 @@ from .releases import ( ReleaseTag, - get_release_page_url, - get_release_url, is_release_url, is_url, + resolve_release, ) CACHED_DOWNLOADS_DIRECTORY = ( @@ -264,8 +263,11 @@ def from_release_spec( """ if cache_folder is None: cache_folder = CACHED_DOWNLOADS_DIRECTORY - url = get_release_url(spec) - release_page = get_release_page_url(spec) + # Resolve the spec once; the download URL and the release page + # both derive from the same release information. + release = resolve_release(spec) + url = release.get_asset(ReleaseTag.from_string(spec)).url + release_page = release.url destination_folder = extract_to or FixtureDownloader.get_cache_path( url, cache_folder diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py index 9a16d79894b..b19cfe8e387 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py @@ -355,35 +355,21 @@ def get_release_url_from_release_information( return release.get_asset(ReleaseTag.from_string(release_string)).url -def get_release_page_url(release_string: str) -> str: +def resolve_release(release_string: str) -> ReleaseInformation: """ - Return the GitHub Release page URL for a specific release descriptor. - - This function can handle: - - A release string (e.g., "tests@latest" or "bal-devnet@v7.0.0") from - any repo in `SUPPORTED_REPOS`. - - A direct asset download link (e.g., - "https://github.com/ethereum/execution-specs/releases/ - download/tests%40v20.0.0/fixtures.tar.gz"). + Resolve a release descriptor string to its release information. + + Refresh the cached release information beforehand as needed (see + `get_release_information`). """ - release_information = get_release_information(release_string) + return find_release( + release_string, get_release_information(release_string) + ) - # Case 1: If it's a direct GitHub Releases download link, find which - # release in `release_information` has an asset with this exact URL. - repo_pattern = "|".join(re.escape(repo) for repo in SUPPORTED_REPOS) - regex_pattern = rf"https://github\.com/({repo_pattern})/releases/download/" - if re.match(regex_pattern, release_string): - for release in release_information: - for asset in release.assets.root: - if asset.url == release_string: - return release.url # The HTML page for this release - raise NoSuchReleaseError( - f"No release found for asset URL: {release_string}" - ) - # Case 2: Otherwise, treat it as a release descriptor (e.g., - # "tests@latest") - return find_release(release_string, release_information).url +def get_release_page_url(release_string: str) -> str: + """Get the GitHub release page URL for a release descriptor.""" + return resolve_release(release_string).url def get_release_information( @@ -435,8 +421,6 @@ def get_release_information( def get_release_url(release_string: str) -> str: - """Get the URL for a specific release.""" - release_information = get_release_information(release_string) - return get_release_url_from_release_information( - release_string, release_information - ) + """Get the asset download URL for a release descriptor.""" + release = resolve_release(release_string) + return release.get_asset(ReleaseTag.from_string(release_string)).url diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py index fa7cb7ffce4..97a229c49ef 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py @@ -39,35 +39,36 @@ def test_fixtures_source_from_release_spec_makes_api_calls(self) -> None: test_spec = "tests@latest" with patch( - "execution_testing.cli.pytest_commands.plugins.consume.consume.get_release_url" - ) as mock_get_url: - mock_get_url.return_value = "https://github.com/ethereum/execution-specs/releases/download/tests%40v20.0.0/fixtures.tar.gz" + "execution_testing.cli.pytest_commands.plugins.consume.consume.resolve_release" + ) as mock_resolve: + mock_release = MagicMock() + mock_release.url = "https://github.com/ethereum/execution-specs/releases/tag/tests%40v20.0.0" + mock_release.get_asset.return_value.url = "https://github.com/ethereum/execution-specs/releases/download/tests%40v20.0.0/fixtures.tar.gz" + mock_resolve.return_value = mock_release with patch( - "execution_testing.cli.pytest_commands.plugins.consume.consume.get_release_page_url" - ) as mock_get_page: - mock_get_page.return_value = "https://github.com/ethereum/execution-specs/releases/tag/tests%40v20.0.0" - with patch( - "execution_testing.cli.pytest_commands.plugins.consume.consume.FixtureDownloader" - ) as mock_downloader: - mock_instance = MagicMock() - mock_instance.download_and_extract.return_value = ( - False, - Path("/tmp/test"), - ) - mock_downloader.return_value = mock_instance - - source = FixturesSource.from_release_spec(test_spec) - - # Verify API calls were made and release page is set. - # Both lookups receive the release spec, so pinned - # versions resolve from the release-information cache - # without querying the GitHub API. - mock_get_url.assert_called_once_with(test_spec) - mock_get_page.assert_called_once_with(test_spec) - assert ( - source.release_page - == "https://github.com/ethereum/execution-specs/releases/tag/tests%40v20.0.0" - ) + "execution_testing.cli.pytest_commands.plugins.consume.consume.FixtureDownloader" + ) as mock_downloader: + mock_instance = MagicMock() + mock_instance.download_and_extract.return_value = ( + False, + Path("/tmp/test"), + ) + mock_downloader.return_value = mock_instance + + source = FixturesSource.from_release_spec(test_spec) + + # The spec is resolved exactly once; the download URL and + # the release page both derive from the same release + # information. + mock_resolve.assert_called_once_with(test_spec) + assert ( + source.url + == "https://github.com/ethereum/execution-specs/releases/download/tests%40v20.0.0/fixtures.tar.gz" + ) + assert ( + source.release_page + == "https://github.com/ethereum/execution-specs/releases/tag/tests%40v20.0.0" + ) def test_fixtures_source_from_regular_url_no_release_page(self) -> None: """Test that regular URLs (non-GitHub) don't have release page.""" diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py index 450c63e0af7..042421357cc 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py @@ -280,17 +280,16 @@ def fake_release(tag_name: str, asset_name: str) -> Dict: @pytest.fixture -def release_information_cache( +def release_cache_path( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> Path: """ - Point the release-information cache at a copy of the test manifest. + Redirect the release-information cache to a temporary path. Also disable the CI/Docker detection so the freshness check applies (in CI, the cache never expires). """ cache_file = tmp_path / "release_information.json" - shutil.copyfile(CURRENT_FOLDER / "release_information.json", cache_file) monkeypatch.setattr( releases, "CACHED_RELEASE_INFORMATION_FILE", cache_file ) @@ -298,6 +297,15 @@ def release_information_cache( return cache_file +@pytest.fixture +def release_information_cache(release_cache_path: Path) -> Path: + """Populate the redirected cache with a copy of the test manifest.""" + shutil.copyfile( + CURRENT_FOLDER / "release_information.json", release_cache_path + ) + return release_cache_path + + def make_stale(cache_file: Path) -> None: """Age the cache file's mtime beyond the 4-hour freshness window.""" stale_time = time.time() - 5 * 60 * 60 @@ -381,15 +389,10 @@ def test_rate_limited_refresh_falls_back_to_stale_cache( def test_rate_limited_refresh_without_cache_raises( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch + release_cache_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """Without a cache file, a failed refresh is a hard error.""" - monkeypatch.setattr( - releases, - "CACHED_RELEASE_INFORMATION_FILE", - tmp_path / "release_information.json", - ) - monkeypatch.setattr(releases, "is_docker_or_ci", lambda: False) + del release_cache_path monkeypatch.setattr(releases.requests, "get", rate_limited_get) with pytest.raises(requests.exceptions.HTTPError): get_release_url("tests@latest") @@ -408,9 +411,8 @@ def test_unpinned_release_refreshes_stale_cache( calls: List[str] = [] def fake_get(url: str, **kwargs: Any) -> FakeResponse: - del kwargs calls.append(url) - return FakeResponse([fake_release("tests@v21.0.0", "fixtures.tar.gz")]) + return new_release_get(url, **kwargs) monkeypatch.setattr(releases.requests, "get", fake_get) assert get_release_url("tests@latest").endswith( @@ -471,9 +473,8 @@ def test_download_release_information_github_token( seen_headers: List[Dict[str, str]] = [] def fake_get(url: str, **kwargs: Any) -> FakeResponse: - del url seen_headers.append(kwargs.get("headers") or {}) - return FakeResponse([fake_release("tests@v21.0.0", "fixtures.tar.gz")]) + return new_release_get(url, **kwargs) monkeypatch.setattr(releases.requests, "get", fake_get) download_release_information(tmp_path / "release_information.json") From a76614cb71645e4a23ab14409badda7693683709 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 16 Jul 2026 21:37:49 +0200 Subject: [PATCH 4/6] ci: retrigger to test runner provisioning (empty commit) From 8fe73136b2d7da533a922d7ac16eef5211dae3cc Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 16 Jul 2026 21:56:28 +0200 Subject: [PATCH 5/6] ci: retrigger to test runner provisioning (empty commit) From ce63a6ef14d24adec3b5f7e6da2ab8db76b90a39 Mon Sep 17 00:00:00 2001 From: danceratopz Date: Thu, 16 Jul 2026 22:16:05 +0200 Subject: [PATCH 6/6] ci: retrigger to test runner provisioning (empty commit)