diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index 1c54c9ce..62db3b7d 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -12,7 +12,7 @@ from packaging.requirements import Requirement from packaging.version import Version -from . import resolver, sources, wheels +from . import finders, resolver, sources, wheels from .dependency_graph import DependencyGraph from .requirements_file import RequirementType @@ -40,15 +40,24 @@ def __init__( self, ctx: context.WorkContext, prev_graph: DependencyGraph | None = None, + multiple_versions: bool = False, + cache_wheel_server_url: str = "", ) -> None: """Initialize requirement resolver. Args: ctx: Work context with constraints and settings prev_graph: Optional previous dependency graph for caching + multiple_versions: If ``True``, age filtering returns an empty + list instead of falling back to all candidates, letting the + caller decide how to handle the case. + cache_wheel_server_url: URL of the remote wheel cache server. + Used as a fallback when age filtering produces no candidates. """ self.ctx = ctx self.prev_graph = prev_graph + self.multiple_versions = multiple_versions + self.cache_wheel_server_url = cache_wheel_server_url # Session-level resolution cache to avoid re-resolving same requirements # Key: (requirement_string, pre_built) to distinguish source vs prebuilt # Value: tuple of (url, version) tuples sorted by version (highest first) @@ -72,6 +81,8 @@ def resolve( 1. Session cache (if previously resolved) 2. Previous dependency graph 3. PyPI resolution (source or prebuilt based on package build info) + 4. Remote wheel cache server (multi-version mode only, when age + filtering produced no candidates) Args: req: Package requirement @@ -113,8 +124,12 @@ def resolve( # Resolve using strategies results = self._resolve(req, req_type, parent_req, pre_built) - # Cache the result - self.cache_resolution(req, pre_built, results) + # Only cache non-empty results. + if results: + self.cache_resolution(req, pre_built, results) + + if not results: + return [] return results if return_all_versions else [results[0]] def _resolve( @@ -129,6 +144,8 @@ def _resolve( Tries resolution strategies in order: 1. Previous dependency graph 2. PyPI resolution (source or prebuilt) + 3. Remote wheel cache server (multi-version mode only, when age + filtering produced no source candidates) Args: req: Package requirement @@ -167,18 +184,70 @@ def _resolve( wheel_server_urls=wheel_server_urls, req_type=req_type, ) - else: - # Resolve source (sdist) - provider = sources.get_source_provider( - ctx=self.ctx, - req=req, - sdist_server_url=resolver.PYPI_SERVER_URL, - req_type=req_type, + + # Resolve source (sdist) + provider = sources.get_source_provider( + ctx=self.ctx, + req=req, + sdist_server_url=resolver.PYPI_SERVER_URL, + req_type=req_type, + ) + max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx) + results = resolver.find_all_matching_from_provider( + provider, + req, + max_age_cutoff=max_age_cutoff, + fallback_on_empty_age_filter=not self.multiple_versions, + ) + + if not results and self.multiple_versions and self.cache_wheel_server_url: + results = self._resolve_from_cache_server(req) + + if not results: + logger.warning( + "%s: could not find any versions (all filtered by " + "max-release-age and no cached wheel on server)", + req.name, + ) + + return results + + def _resolve_from_cache_server(self, req: Requirement) -> list[tuple[str, Version]]: + """Fall back to the remote wheel cache server for a cached version. + + When age filtering removes all candidates in multi-version mode, + queries the remote cache server for the newest available wheel. + Returns at most one version so that transitive dependencies are + re-processed without rebuilding every old version. + """ + logger.info( + "%s: all versions filtered by max-release-age, " + "checking cache server %s for existing wheel", + req.name, + self.cache_wheel_server_url, + ) + try: + provider = finders.PyPICacheProvider( + cache_server_url=self.cache_wheel_server_url, + constraints=self.ctx.constraints, ) - max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx) - return resolver.find_all_matching_from_provider( - provider, req, max_age_cutoff=max_age_cutoff + results = resolver.find_all_matching_from_provider(provider, req) + if results: + url, version = results[0] + logger.info( + "%s: found version %s on cache server", + req.name, + version, + ) + return [(url, version)] + except Exception as err: + logger.warning( + "%s: error checking cache server %s: %s", + req.name, + self.cache_wheel_server_url, + err, ) + return [] def get_cached_resolution( self, diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index f635497c..10514cd0 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -173,6 +173,8 @@ def __init__( self._resolver = bootstrap_requirement_resolver.BootstrapRequirementResolver( ctx=ctx, prev_graph=prev_graph, + multiple_versions=multiple_versions, + cache_wheel_server_url=self.cache_wheel_server_url, ) # Push items onto the stack as we start to resolve their # dependencies so at the end we have a list of items that need to diff --git a/src/fromager/resolver.py b/src/fromager/resolver.py index 0ac959f8..bc670841 100644 --- a/src/fromager/resolver.py +++ b/src/fromager/resolver.py @@ -243,6 +243,7 @@ def find_all_matching_from_provider( provider: BaseProvider, req: Requirement, max_age_cutoff: datetime.datetime | None = None, + fallback_on_empty_age_filter: bool = True, ) -> list[tuple[str, Version]]: """Find all matching candidates from provider without full dependency resolution. @@ -255,6 +256,10 @@ def find_all_matching_from_provider( max_age_cutoff: If set, reject candidates published before this time. If all candidates are older than the cutoff, all are kept and a warning is emitted to avoid empty resolution. + fallback_on_empty_age_filter: If ``True`` (default), keep all + candidates when age filtering would produce an empty result. + If ``False``, return an empty list instead, allowing the + caller to implement its own fallback strategy. Returns list of (url, version) tuples sorted by version (highest first). @@ -315,7 +320,7 @@ def find_all_matching_from_provider( ) if filtered: candidates_list = filtered - else: + elif fallback_on_empty_age_filter: logger.warning( "%s: all %d candidate(s) of %s are older than %d days, " "keeping all to avoid empty resolution", @@ -324,6 +329,15 @@ def find_all_matching_from_provider( req, max_age_days, ) + else: + logger.info( + "%s: all %d candidate(s) of %s are older than %d days", + req.name, + len(candidates_list), + req, + max_age_days, + ) + candidates_list = [] # Convert candidates to list of (url, version) tuples # Candidates are sorted by version (highest first) by BaseProvider.find_matches() diff --git a/tests/test_bootstrap_requirement_resolver.py b/tests/test_bootstrap_requirement_resolver.py index 710dca71..cd85d7fd 100644 --- a/tests/test_bootstrap_requirement_resolver.py +++ b/tests/test_bootstrap_requirement_resolver.py @@ -694,3 +694,170 @@ def test_resolve_prebuilt_after_source_uses_separate_cache( url2, version2 = results2[0] assert url2 == "https://files.pythonhosted.org/testpkg-1.5-py3-none-any.whl" assert version2 == Version("1.5") + + +class TestResolveFromCacheServer: + """Tests for the cache server fallback in _resolve_from_cache_server.""" + + def test_returns_newest_version_from_cache(self, tmp_context: WorkContext) -> None: + """Falls back to cache server and returns only the newest version.""" + brr = BootstrapRequirementResolver( + tmp_context, + multiple_versions=True, + cache_wheel_server_url="http://cache.test/simple", + ) + req = Requirement("testpkg") + + with patch( + "fromager.bootstrap_requirement_resolver.resolver" + ".find_all_matching_from_provider", + return_value=[ + ("http://cache.test/testpkg-3.0.whl", Version("3.0")), + ("http://cache.test/testpkg-2.0.whl", Version("2.0")), + ], + ): + result = brr._resolve_from_cache_server(req) + + assert len(result) == 1 + assert result[0][1] == Version("3.0") + + def test_returns_empty_when_cache_has_no_match( + self, tmp_context: WorkContext + ) -> None: + """Returns empty list when cache server has nothing.""" + brr = BootstrapRequirementResolver( + tmp_context, + multiple_versions=True, + cache_wheel_server_url="http://cache.test/simple", + ) + req = Requirement("testpkg") + + with patch( + "fromager.bootstrap_requirement_resolver.resolver" + ".find_all_matching_from_provider", + return_value=[], + ): + result = brr._resolve_from_cache_server(req) + + assert result == [] + + def test_returns_empty_on_exception(self, tmp_context: WorkContext) -> None: + """Returns empty list when cache server query fails.""" + brr = BootstrapRequirementResolver( + tmp_context, + multiple_versions=True, + cache_wheel_server_url="http://cache.test/simple", + ) + req = Requirement("testpkg") + + with patch( + "fromager.bootstrap_requirement_resolver.resolver" + ".find_all_matching_from_provider", + side_effect=RuntimeError("connection refused"), + ): + result = brr._resolve_from_cache_server(req) + + assert result == [] + + def test_resolve_uses_cache_fallback_when_age_filter_empties( + self, tmp_context: WorkContext + ) -> None: + """_resolve falls back to cache server when age filter produces empty result.""" + brr = BootstrapRequirementResolver( + tmp_context, + multiple_versions=True, + cache_wheel_server_url="http://cache.test/simple", + ) + req = Requirement("testpkg") + + with ( + patch.object(brr, "_resolve_from_graph", return_value=None), + patch( + "fromager.bootstrap_requirement_resolver.sources.get_source_provider", + ), + patch( + "fromager.bootstrap_requirement_resolver.resolver" + ".find_all_matching_from_provider", + return_value=[], + ) as mock_pypi, + patch.object( + brr, + "_resolve_from_cache_server", + return_value=[("http://cache.test/testpkg-1.0.whl", Version("1.0"))], + ) as mock_cache, + ): + result = brr._resolve( + req, + RequirementType.INSTALL, + parent_req=None, + pre_built=False, + ) + + mock_pypi.assert_called_once() + mock_cache.assert_called_once_with(req) + assert len(result) == 1 + assert result[0][1] == Version("1.0") + + def test_resolve_skips_cache_fallback_in_single_version_mode( + self, tmp_context: WorkContext + ) -> None: + """_resolve does not fall back to cache server in single-version mode.""" + brr = BootstrapRequirementResolver( + tmp_context, + multiple_versions=False, + cache_wheel_server_url="http://cache.test/simple", + ) + req = Requirement("testpkg") + + with ( + patch.object(brr, "_resolve_from_graph", return_value=None), + patch( + "fromager.bootstrap_requirement_resolver.sources.get_source_provider", + ), + patch( + "fromager.bootstrap_requirement_resolver.resolver" + ".find_all_matching_from_provider", + return_value=[("url", Version("1.0"))], + ), + patch.object(brr, "_resolve_from_cache_server") as mock_cache, + ): + brr._resolve( + req, + RequirementType.INSTALL, + parent_req=None, + pre_built=False, + ) + + mock_cache.assert_not_called() + + def test_resolve_skips_cache_fallback_when_no_server_url( + self, tmp_context: WorkContext + ) -> None: + """_resolve does not fall back when no cache_wheel_server_url is set.""" + brr = BootstrapRequirementResolver( + tmp_context, + multiple_versions=True, + cache_wheel_server_url="", + ) + req = Requirement("testpkg") + + with ( + patch.object(brr, "_resolve_from_graph", return_value=None), + patch( + "fromager.bootstrap_requirement_resolver.sources.get_source_provider", + ), + patch( + "fromager.bootstrap_requirement_resolver.resolver" + ".find_all_matching_from_provider", + return_value=[], + ), + patch.object(brr, "_resolve_from_cache_server") as mock_cache, + ): + brr._resolve( + req, + RequirementType.INSTALL, + parent_req=None, + pre_built=False, + ) + + mock_cache.assert_not_called() diff --git a/tests/test_bootstrapper_iterative.py b/tests/test_bootstrapper_iterative.py index 1bf16de9..dbbbbf08 100644 --- a/tests/test_bootstrapper_iterative.py +++ b/tests/test_bootstrapper_iterative.py @@ -353,6 +353,20 @@ def test_no_filtering_in_single_version_mode( assert len(result) == 1 mock_cache.assert_not_called() + def test_empty_resolution_raises_runtime_error( + self, tmp_context: WorkContext + ) -> None: + """Empty resolution raises RuntimeError regardless of mode.""" + for multi in (False, True): + bt = bootstrapper.Bootstrapper(tmp_context, multiple_versions=multi) + item = _make_resolve_item() + + with ( + patch.object(bt, "resolve_versions", return_value=[]), + pytest.raises(RuntimeError, match="Could not resolve"), + ): + bt._phase_resolve(item) + class TestPhaseStart: def test_new_item_advances_to_prepare_source( diff --git a/tests/test_cooldown.py b/tests/test_cooldown.py index 098081a3..0d47cc5f 100644 --- a/tests/test_cooldown.py +++ b/tests/test_cooldown.py @@ -796,6 +796,30 @@ def test_max_release_age_all_too_old_keeps_all( assert "keeping all to avoid empty resolution" in caplog.text +def test_max_release_age_all_too_old_returns_empty_when_fallback_disabled( + caplog: pytest.LogCaptureFixture, +) -> None: + """When fallback is disabled and all versions are too old, return empty list.""" + max_age_cutoff = _BOOTSTRAP_TIME + datetime.timedelta(days=1) + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True) + with caplog.at_level(logging.INFO, logger="fromager.resolver"): + results = resolver.find_all_matching_from_provider( + provider, + Requirement("test-pkg"), + max_age_cutoff=max_age_cutoff, + fallback_on_empty_age_filter=False, + ) + assert results == [] + assert "all 3 candidate(s)" in caplog.text + assert "keeping all to avoid empty resolution" not in caplog.text + + def test_max_release_age_candidates_without_upload_time_pass_through() -> None: """Candidates without upload_time are not filtered out by max-release-age.""" no_timestamp_response = {