feat: write facts about what we find on SimpleAPI to MODULE.bazel.lock

Author: aignas

python/private/pypi/extension.bzl

@@ -27,7 +27,7 @@ load(":parse_whl_name.bzl", "parse_whl_name")
 load(":pep508_env.bzl", "env")
 load(":pip_repository_attrs.bzl", "ATTRS")
 load(":platform.bzl", _plat = "platform")
-load(":simpleapi_download.bzl", "simpleapi_download")
+load(":simpleapi_download.bzl", "simpleapi_download", _simpleapi_cache = "simpleapi_cache")
 load(":whl_library.bzl", "whl_library")
 
 def _whl_mods_impl(whl_mods_dict):
@@ -224,7 +224,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
     # dict[str repo, HubBuilder]
     # See `hub_builder.bzl%hub_builder()` for `HubBuilder`
     pip_hub_map = {}
-    simpleapi_cache = {}
+    simpleapi_cache = _simpleapi_cache(module_ctx)
 
     for mod in module_ctx.modules:
         for pip_attr in mod.tags.parse:
@@ -296,6 +296,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
         hub_whl_map = hub_whl_map,
         whl_libraries = whl_libraries,
         whl_mods = whl_mods,
+        facts = simpleapi_cache.get_facts(),
         platform_config_settings = {
             hub_name: {
                 platform_name: sorted([str(Label(cv)) for cv in p.config_settings])
@@ -393,9 +394,11 @@ def _pip_impl(module_ctx):
             groups = mods.hub_group_map.get(hub_name),
         )
 
-    return module_ctx.extension_metadata(
-        reproducible = True,
-    )
+    kwargs = {"reproducible": True}
+    if mods.facts:
+        kwargs["facts"] = mods.facts
+
+    return module_ctx.extension_metadata(**kwargs)
 
 _default_attrs = {
     "arch_name": attr.string(

python/private/pypi/hub_builder.bzl

@@ -395,11 +395,11 @@ def _set_get_index_urls(self, pip_attr):
             index_url = pip_attr.experimental_index_url,
             extra_index_urls = pip_attr.experimental_extra_index_urls or [],
             index_url_overrides = pip_attr.experimental_index_url_overrides or {},
-            sources = [
-                d
-                for d in distributions
+            sources = {
+                d: versions
+                for d, versions in distributions.items()
                 if _use_downloader(self, python_version, d)
-            ],
+            },
             envsubst = pip_attr.envsubst,
             # Auth related info
             netrc = pip_attr.netrc,

python/private/pypi/parse_requirements.bzl

@@ -170,16 +170,15 @@ def parse_requirements(
 
     index_urls = {}
     if get_index_urls:
-        index_urls = get_index_urls(
-            ctx,
-            # Use list({}) as a way to have a set
-            list({
-                req.distribution: None
-                for reqs in requirements_by_platform.values()
-                for req in reqs.values()
-                if not req.srcs.url
-            }),
-        )
+        distributions = {}
+        for reqs in requirements_by_platform.values():
+            for req in reqs.values():
+                if req.srcs.url:
+                    continue
+
+                distributions.setdefault(req.distribution, []).append(req.srcs.version)
+
+        index_urls = get_index_urls(ctx, distributions)
 
     ret = []
     for name, reqs in sorted(requirements_by_platform.items()):

python/private/pypi/parse_simpleapi_html.bzl

@@ -16,11 +16,11 @@
 Parse SimpleAPI HTML in Starlark.
 """
 
-def parse_simpleapi_html(*, url, content):
+def parse_simpleapi_html(*, content, distribution):
     """Get the package URLs for given shas by parsing the Simple API HTML.
 
     Args:
-        url(str): The URL that the HTML content can be downloaded from.
+        distribution(str): Distribution name for which we are parsing the HTML.
         content(str): The Simple API HTML content.
 
     Returns:
@@ -55,16 +55,14 @@ def parse_simpleapi_html(*, url, content):
     sha256s_by_version = {}
     for line in lines[1:]:
         dist_url, _, tail = line.partition("#sha256=")
-        dist_url = _absolute_url(url, dist_url)
-
         sha256, _, tail = tail.partition("\"")
 
         # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api
         yanked = "data-yanked" in line
 
         head, _, _ = tail.rpartition("</a>")
         maybe_metadata, _, filename = head.rpartition(">")
-        version = _version(filename)
+        version = pkg_version(filename, distribution)
         sha256s_by_version.setdefault(version, []).append(sha256)
 
         metadata_sha256 = ""
@@ -79,13 +77,14 @@ def parse_simpleapi_html(*, url, content):
                 break
 
         if filename.endswith(".whl"):
+            metadata_url = metadata_url or ""
             whls[sha256] = struct(
                 filename = filename,
                 version = version,
                 url = dist_url,
                 sha256 = sha256,
                 metadata_sha256 = metadata_sha256,
-                metadata_url = _absolute_url(url, metadata_url) if metadata_url else "",
+                metadata_url = metadata_url,
                 yanked = yanked,
             )
         else:
@@ -110,18 +109,36 @@ _SDIST_EXTS = [
     ".zip",
 ]
 
-def _version(filename):
+def pkg_version(filename, distribution = None):
+    """pkg_version extracts the version from the filename.
+
+    TODO: move this to a different location
+
+    Args:
+        filename: TODO
+        distribution: TODO
+
+    Returns:
+        version string
+    """
     # See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#binary-distribution-format
 
-    _, _, tail = filename.partition("-")
-    version, _, _ = tail.partition("-")
-    if version != tail:
-        # The format is {name}-{version}-{whl_specifiers}.whl
-        return version
+    if filename.endswith(".whl"):
+        _, _, tail = filename.partition("-")
+        version, _, _ = tail.partition("-")
+        if version != tail:
+            # The format is {name}-{version}-{whl_specifiers}.whl
+            return version
+
+    if not distribution:
+        fail("for parsing sdists passing 'distribution' is mandatory")
 
     # NOTE @aignas 2025-03-29: most of the files are wheels, so this is not the common path
 
     # {name}-{version}.{ext}
+    # TODO @aignas 2026-01-20: test for handling dashes in names, can't think of any other way to
+    # get the version from the filename but to pass in the distribution name to this function.
+    version = filename[len(distribution) + 1:]
     for ext in _SDIST_EXTS:
         version, _, _ = version.partition(ext)  # build or name
 
@@ -147,26 +164,35 @@ def _is_downloadable(url):
     """
     return url.startswith("http://") or url.startswith("https://") or url.startswith("file://")
 
-def _absolute_url(index_url, candidate):
-    if candidate == "":
-        return candidate
+def absolute_url(*, index_url, url):
+    """Return an absolute URL in case the url is not absolute.
+
+    Args:
+        index_url: {type}`str` The index_url.
+        url: {type}`str` The url of the artifact.
+
+    Returns:
+        `url` if it is absolute, or absolute URL based on the `index_url`.
+    """
+    if url == "":
+        return url
 
-    if _is_downloadable(candidate):
-        return candidate
+    if _is_downloadable(url):
+        return url
 
-    if candidate.startswith("/"):
+    if url.startswith("/"):
         # absolute path
         root_directory = _get_root_directory(index_url)
-        return "{}{}".format(root_directory, candidate)
+        return "{}{}".format(root_directory, url)
 
-    if candidate.startswith(".."):
+    if url.startswith(".."):
         # relative path with up references
-        candidate_parts = candidate.split("..")
+        candidate_parts = url.split("..")
         last = candidate_parts[-1]
         for _ in range(len(candidate_parts) - 1):
             index_url, _, _ = index_url.rstrip("/").rpartition("/")
 
         return "{}/{}".format(index_url, last.strip("/"))
 
     # relative path without up-references
-    return "{}/{}".format(index_url.rstrip("/"), candidate)
+    return "{}/{}".format(index_url.rstrip("/"), url)