diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py index 62ddb49404..339904a4d1 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/consume.py @@ -43,10 +43,9 @@ from .releases import ( ReleaseTag, - get_release_page_url, - get_release_url, is_release_url, is_url, + resolve_release, ) CACHED_DOWNLOADS_DIRECTORY = ( @@ -264,8 +263,11 @@ def from_release_spec( """ if cache_folder is None: cache_folder = CACHED_DOWNLOADS_DIRECTORY - url = get_release_url(spec) - release_page = get_release_page_url(url) + # Resolve the spec once; the download URL and the release page + # both derive from the same release information. + release = resolve_release(spec) + url = release.get_asset(ReleaseTag.from_string(spec)).url + release_page = release.url destination_folder = extract_to or FixtureDownloader.get_cache_path( url, cache_folder diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py index f74f68b382..b19cfe8e38 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/releases.py @@ -1,6 +1,7 @@ """Procedures to consume fixtures from Github releases.""" import json +import logging import os import re from dataclasses import dataclass @@ -11,7 +12,9 @@ import platformdirs import requests -from pydantic import BaseModel, Field, RootModel +from pydantic import BaseModel, Field, RootModel, ValidationError + +logger = logging.getLogger(__name__) CACHED_RELEASE_INFORMATION_FILE = ( Path(platformdirs.user_cache_dir("ethereum-execution-spec-tests")) @@ -237,7 +240,15 @@ def download_release_information( pagination links up to `max_pages` pages, so resolution sees the 200 most recent releases per repo. Older releases fall outside this window and cannot be resolved. + + Authenticate with `GITHUB_TOKEN` when set: authenticated requests + get 5000 requests/hour instead of the unauthenticated 60/hour per + IP address. """ + headers = {} + github_token = os.environ.get("GITHUB_TOKEN") + if github_token: + headers["Authorization"] = f"Bearer {github_token}" all_releases = [] for repo in SUPPORTED_REPOS: current_url: str | None = ( @@ -246,7 +257,7 @@ def download_release_information( max_pages = 2 while current_url and max_pages > 0: max_pages -= 1 - response = requests.get(current_url) + response = requests.get(current_url, headers=headers) response.raise_for_status() all_releases.extend(response.json()) current_url = None @@ -260,8 +271,14 @@ def download_release_information( if destination_file: destination_file.parent.mkdir(parents=True, exist_ok=True) - with open(destination_file, "w") as file: + # Write via a temporary file so a concurrent reader never sees a + # partially-written cache. + temporary_file = destination_file.with_name( + destination_file.name + ".tmp" + ) + with open(temporary_file, "w") as file: json.dump(all_releases, file) + temporary_file.replace(destination_file) return parse_release_information(all_releases) @@ -310,6 +327,26 @@ def sort_key( return max(matches, key=sort_key) +def resolves_pinned_release( + release_string: str, + release_information: List[ReleaseInformation], +) -> bool: + """ + Check whether the release information resolves a pinned version. + + A release descriptor with an explicit version refers to an immutable + git tag: once it resolves, a refresh of the release information + cannot change the result. + """ + if ReleaseTag.from_string(release_string).version is None: + return False + try: + find_release(release_string, release_information) + except NoSuchReleaseError: + return False + return True + + def get_release_url_from_release_information( release_string: str, release_information: List[ReleaseInformation] ) -> str: @@ -318,63 +355,72 @@ def get_release_url_from_release_information( return release.get_asset(ReleaseTag.from_string(release_string)).url -def get_release_page_url(release_string: str) -> str: +def resolve_release(release_string: str) -> ReleaseInformation: """ - Return the GitHub Release page URL for a specific release descriptor. - - This function can handle: - - A release string (e.g., "tests@latest" or "bal-devnet@v7.0.0") from - any repo in `SUPPORTED_REPOS`. - - A direct asset download link (e.g., - "https://github.com/ethereum/execution-specs/releases/ - download/tests%40v20.0.0/fixtures.tar.gz"). + Resolve a release descriptor string to its release information. + + Refresh the cached release information beforehand as needed (see + `get_release_information`). """ - release_information = get_release_information() + return find_release( + release_string, get_release_information(release_string) + ) - # Case 1: If it's a direct GitHub Releases download link, find which - # release in `release_information` has an asset with this exact URL. - repo_pattern = "|".join(re.escape(repo) for repo in SUPPORTED_REPOS) - regex_pattern = rf"https://github\.com/({repo_pattern})/releases/download/" - if re.match(regex_pattern, release_string): - for release in release_information: - for asset in release.assets.root: - if asset.url == release_string: - return release.url # The HTML page for this release - raise NoSuchReleaseError( - f"No release found for asset URL: {release_string}" - ) - # Case 2: Otherwise, treat it as a release descriptor (e.g., - # "tests@latest") - return find_release(release_string, release_information).url +def get_release_page_url(release_string: str) -> str: + """Get the GitHub release page URL for a release descriptor.""" + return resolve_release(release_string).url -def get_release_information() -> List[ReleaseInformation]: +def get_release_information( + release_string: str | None = None, +) -> List[ReleaseInformation]: """ - Get the release information. - - First check if the cached release information file exists. If it does, but - it is older than 4 hours, delete the file, unless running inside a CI - environment or a Docker container. Then download the release information - from the Github API and save it to the cache file. + Get the release information, refreshing the cache file as needed. + + Return the cached release information if the cache file is fresh + (younger than 4 hours; any age when running inside a CI environment + or a Docker container). A stale cache is also used without + refreshing when `release_string` pins an exact version that the + cache already resolves: release tags are immutable, so the cached + entry cannot be outdated. Otherwise re-download the release + information, keeping the stale cache as a fallback in case the + GitHub API is unavailable (e.g. rate-limited). """ + cached_information: List[ReleaseInformation] | None = None if CACHED_RELEASE_INFORMATION_FILE.exists(): - last_modified = CACHED_RELEASE_INFORMATION_FILE.stat().st_mtime - if ( - datetime.now().timestamp() - last_modified - ) < 4 * 60 * 60 or is_docker_or_ci(): - return parse_release_information_from_file( + try: + cached_information = parse_release_information_from_file( CACHED_RELEASE_INFORMATION_FILE ) - CACHED_RELEASE_INFORMATION_FILE.unlink() - if not CACHED_RELEASE_INFORMATION_FILE.exists(): + except (json.JSONDecodeError, ValidationError): + logger.warning( + "Ignoring corrupt release information cache at " + f"{CACHED_RELEASE_INFORMATION_FILE}." + ) + else: + last_modified = CACHED_RELEASE_INFORMATION_FILE.stat().st_mtime + cache_age = datetime.now().timestamp() - last_modified + if cache_age < 4 * 60 * 60 or is_docker_or_ci(): + return cached_information + if release_string is not None and resolves_pinned_release( + release_string, cached_information + ): + return cached_information + try: return download_release_information(CACHED_RELEASE_INFORMATION_FILE) - return parse_release_information_from_file(CACHED_RELEASE_INFORMATION_FILE) + except requests.RequestException as error: + if cached_information is None: + raise + logger.warning( + f"Could not refresh release information from the GitHub API " + f"({error}); falling back to the stale cache at " + f"{CACHED_RELEASE_INFORMATION_FILE}." + ) + return cached_information def get_release_url(release_string: str) -> str: - """Get the URL for a specific release.""" - release_information = get_release_information() - return get_release_url_from_release_information( - release_string, release_information - ) + """Get the asset download URL for a release descriptor.""" + release = resolve_release(release_string) + return release.get_asset(ReleaseTag.from_string(release_string)).url diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py index 5a5418006d..97a229c49e 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_fixtures_source_input_types.py @@ -39,34 +39,36 @@ def test_fixtures_source_from_release_spec_makes_api_calls(self) -> None: test_spec = "tests@latest" with patch( - "execution_testing.cli.pytest_commands.plugins.consume.consume.get_release_url" - ) as mock_get_url: - mock_get_url.return_value = "https://github.com/ethereum/execution-specs/releases/download/tests%40v20.0.0/fixtures.tar.gz" + "execution_testing.cli.pytest_commands.plugins.consume.consume.resolve_release" + ) as mock_resolve: + mock_release = MagicMock() + mock_release.url = "https://github.com/ethereum/execution-specs/releases/tag/tests%40v20.0.0" + mock_release.get_asset.return_value.url = "https://github.com/ethereum/execution-specs/releases/download/tests%40v20.0.0/fixtures.tar.gz" + mock_resolve.return_value = mock_release with patch( - "execution_testing.cli.pytest_commands.plugins.consume.consume.get_release_page_url" - ) as mock_get_page: - mock_get_page.return_value = "https://github.com/ethereum/execution-specs/releases/tag/tests%40v20.0.0" - with patch( - "execution_testing.cli.pytest_commands.plugins.consume.consume.FixtureDownloader" - ) as mock_downloader: - mock_instance = MagicMock() - mock_instance.download_and_extract.return_value = ( - False, - Path("/tmp/test"), - ) - mock_downloader.return_value = mock_instance - - source = FixturesSource.from_release_spec(test_spec) - - # Verify API calls were made and release page is set - mock_get_url.assert_called_once_with(test_spec) - mock_get_page.assert_called_once_with( - "https://github.com/ethereum/execution-specs/releases/download/tests%40v20.0.0/fixtures.tar.gz" - ) - assert ( - source.release_page - == "https://github.com/ethereum/execution-specs/releases/tag/tests%40v20.0.0" - ) + "execution_testing.cli.pytest_commands.plugins.consume.consume.FixtureDownloader" + ) as mock_downloader: + mock_instance = MagicMock() + mock_instance.download_and_extract.return_value = ( + False, + Path("/tmp/test"), + ) + mock_downloader.return_value = mock_instance + + source = FixturesSource.from_release_spec(test_spec) + + # The spec is resolved exactly once; the download URL and + # the release page both derive from the same release + # information. + mock_resolve.assert_called_once_with(test_spec) + assert ( + source.url + == "https://github.com/ethereum/execution-specs/releases/download/tests%40v20.0.0/fixtures.tar.gz" + ) + assert ( + source.release_page + == "https://github.com/ethereum/execution-specs/releases/tag/tests%40v20.0.0" + ) def test_fixtures_source_from_regular_url_no_release_page(self) -> None: """Test that regular URLs (non-GitHub) don't have release page.""" diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py index cb347efc94..042421357c 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/consume/tests/test_releases.py @@ -1,15 +1,23 @@ """Test release parsing given the github repository release JSON data.""" +import os +import shutil +import time from os.path import realpath from pathlib import Path -from typing import List +from typing import Any, Dict, List import pytest +import requests +from .. import releases from ..releases import ( SUPPORTED_REPOS, NoSuchReleaseError, ReleaseInformation, + download_release_information, + get_release_page_url, + get_release_url, get_release_url_from_release_information, is_release_url, parse_release_information_from_file, @@ -221,3 +229,258 @@ def test_supported_repos_contains_execution_specs() -> None: `tests-bal@v7.1.0` onward) and must be in `SUPPORTED_REPOS`. """ assert "ethereum/execution-specs" in SUPPORTED_REPOS + + +class FakeResponse: + """A minimal stand-in for `requests.Response`.""" + + def __init__( + self, payload: List[Dict], rate_limited: bool = False + ) -> None: + """Initialize with a JSON payload or a rate-limited failure.""" + self.payload = payload + self.rate_limited = rate_limited + self.headers: Dict[str, str] = {} + + def json(self) -> List[Dict]: + """Return the JSON payload.""" + return self.payload + + def raise_for_status(self) -> None: + """Raise an `HTTPError` if the response is rate-limited.""" + if self.rate_limited: + raise requests.exceptions.HTTPError( + "403 Client Error: rate limit exceeded" + ) + + +def fake_release(tag_name: str, asset_name: str) -> Dict: + """Build a minimal GitHub API release entry.""" + encoded_tag = tag_name.replace("@", "%40") + return { + "html_url": "https://github.com/ethereum/execution-specs/releases/" + f"tag/{encoded_tag}", + "id": 1, + "tag_name": tag_name, + "name": tag_name, + "created_at": "2026-07-15T00:00:00Z", + "published_at": "2026-07-15T00:00:00Z", + "assets": [ + { + "browser_download_url": "https://github.com/ethereum/" + f"execution-specs/releases/download/{encoded_tag}/" + f"{asset_name}", + "id": 1, + "name": asset_name, + "content_type": "application/gzip", + "size": 1, + } + ], + } + + +@pytest.fixture +def release_cache_path( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> Path: + """ + Redirect the release-information cache to a temporary path. + + Also disable the CI/Docker detection so the freshness check applies + (in CI, the cache never expires). + """ + cache_file = tmp_path / "release_information.json" + monkeypatch.setattr( + releases, "CACHED_RELEASE_INFORMATION_FILE", cache_file + ) + monkeypatch.setattr(releases, "is_docker_or_ci", lambda: False) + return cache_file + + +@pytest.fixture +def release_information_cache(release_cache_path: Path) -> Path: + """Populate the redirected cache with a copy of the test manifest.""" + shutil.copyfile( + CURRENT_FOLDER / "release_information.json", release_cache_path + ) + return release_cache_path + + +def make_stale(cache_file: Path) -> None: + """Age the cache file's mtime beyond the 4-hour freshness window.""" + stale_time = time.time() - 5 * 60 * 60 + os.utime(cache_file, (stale_time, stale_time)) + + +def block_api(monkeypatch: pytest.MonkeyPatch) -> None: + """Make any GitHub API request fail the test.""" + + def no_api(*args: Any, **kwargs: Any) -> None: + del args, kwargs + pytest.fail("The GitHub API must not be hit") + + monkeypatch.setattr(releases.requests, "get", no_api) + + +def rate_limited_get(*args: Any, **kwargs: Any) -> FakeResponse: + """Return a rate-limited (403) GitHub API response.""" + del args, kwargs + return FakeResponse([], rate_limited=True) + + +def new_release_get(*args: Any, **kwargs: Any) -> FakeResponse: + """Return a single-page response with a new `tests@v21.0.0` release.""" + del args, kwargs + return FakeResponse([fake_release("tests@v21.0.0", "fixtures.tar.gz")]) + + +def test_pinned_release_resolves_from_stale_cache_without_api( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + A pinned version already resolvable from the cache must not refresh. + + Release tags are immutable, so a cached entry for an exact version + cannot be outdated, no matter how old the cache file is. Regression + test for `consume --input=tests@vX.Y.Z` raising INTERNALERROR when + the unauthenticated GitHub API rate limit is exhausted, even though + the (stale) cache resolved the release. + """ + make_stale(release_information_cache) + block_api(monkeypatch) + assert get_release_url("tests@v20.0.0") == ( + "https://github.com/ethereum/execution-specs/releases/download/" + "tests%40v20.0.0/fixtures.tar.gz" + ) + assert get_release_page_url("tests@v20.0.0") == ( + "https://github.com/ethereum/execution-specs/releases/tag/" + "tests%40v20.0.0" + ) + assert release_information_cache.exists() + + +def test_fresh_cache_resolves_latest_without_api( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A fresh cache resolves unpinned lookups without an API request.""" + del release_information_cache + block_api(monkeypatch) + assert get_release_url("tests@latest").endswith( + "tests%40v20.0.0/fixtures.tar.gz" + ) + + +def test_rate_limited_refresh_falls_back_to_stale_cache( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + A failed refresh must fall back to the stale cache, not delete it. + + Previously the stale cache file was deleted before the download was + attempted, so a rate-limited refresh crashed the run and left no + cache at all, forcing every subsequent run onto the API. + """ + make_stale(release_information_cache) + monkeypatch.setattr(releases.requests, "get", rate_limited_get) + assert get_release_url("tests@latest").endswith( + "tests%40v20.0.0/fixtures.tar.gz" + ) + assert release_information_cache.exists() + + +def test_rate_limited_refresh_without_cache_raises( + release_cache_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Without a cache file, a failed refresh is a hard error.""" + del release_cache_path + monkeypatch.setattr(releases.requests, "get", rate_limited_get) + with pytest.raises(requests.exceptions.HTTPError): + get_release_url("tests@latest") + + +def test_unpinned_release_refreshes_stale_cache( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + An unpinned lookup with a stale cache must refresh from the API. + + `latest` and bare feature names can resolve to a newer release at + any time, so the pinned-release fast path must not apply to them. + """ + make_stale(release_information_cache) + calls: List[str] = [] + + def fake_get(url: str, **kwargs: Any) -> FakeResponse: + calls.append(url) + return new_release_get(url, **kwargs) + + monkeypatch.setattr(releases.requests, "get", fake_get) + assert get_release_url("tests@latest").endswith( + "tests%40v21.0.0/fixtures.tar.gz" + ) + assert len(calls) == len(SUPPORTED_REPOS) + assert "tests@v21.0.0" in release_information_cache.read_text() + + +def test_pinned_release_not_in_stale_cache_refreshes( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + A pinned version missing from the stale cache must refresh. + + The pinned-release fast path only applies when the cache already + resolves the requested version. + """ + make_stale(release_information_cache) + monkeypatch.setattr(releases.requests, "get", new_release_get) + assert get_release_url("tests@v21.0.0").endswith( + "tests%40v21.0.0/fixtures.tar.gz" + ) + + +def test_corrupt_cache_file_is_refreshed( + release_information_cache: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + A corrupt cache file must be re-downloaded, not crash the run. + + A partially-written download (e.g. a killed process) must not wedge + every subsequent run until the file is manually deleted. + """ + release_information_cache.write_text("{ not json") + monkeypatch.setattr(releases.requests, "get", new_release_get) + assert get_release_url("tests@v21.0.0").endswith( + "tests%40v21.0.0/fixtures.tar.gz" + ) + + +@pytest.mark.parametrize("github_token", [None, "ghp_test_token"]) +def test_download_release_information_github_token( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + github_token: str | None, +) -> None: + """ + Authenticate GitHub API requests iff `GITHUB_TOKEN` is set. + + Authenticated requests get 5000 requests/hour instead of the + unauthenticated 60 requests/hour per IP. + """ + if github_token is None: + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + else: + monkeypatch.setenv("GITHUB_TOKEN", github_token) + seen_headers: List[Dict[str, str]] = [] + + def fake_get(url: str, **kwargs: Any) -> FakeResponse: + seen_headers.append(kwargs.get("headers") or {}) + return new_release_get(url, **kwargs) + + monkeypatch.setattr(releases.requests, "get", fake_get) + download_release_information(tmp_path / "release_information.json") + expected_headers = ( + {} + if github_token is None + else {"Authorization": f"Bearer {github_token}"} + ) + assert seen_headers == [expected_headers] * len(SUPPORTED_REPOS)