artisanal inteligence spike

Author: aignas

python/private/pypi/extension.bzl

@@ -22,6 +22,7 @@ load("//python/private:auth.bzl", "AUTH_ATTRS")
 load("//python/private:normalize_name.bzl", "normalize_name")
 load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_requires_python")
 load("//python/private:repo_utils.bzl", "repo_utils")
+load("//python/private:text_util.bzl", "render")
 load(":hub_builder.bzl", "hub_builder")
 load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
 load(":parse_whl_name.bzl", "parse_whl_name")
@@ -464,8 +465,26 @@ You cannot use both the additive_build_content and additive_build_content_file a
         out = hub.build()
 
         for whl_name, lib in out.whl_libraries.items():
+            # NOTE @aignas 2026-07-04: if the same wheel is downloaded from multiple
+            # indexes, this will fail, forcing the user to actually download the wheel
+            # from the same and deterministic location. This is usually the case for
+            # public wheels and users should setup the defaults.index_url to correct
+            # fall-back in rules_python we should handle the default index to substitute
+            # any index-url in requirements pointing to the public PyPI mirrors.
             if whl_name in whl_libraries:
-                print("'{}' already in created".format(whl_name))
+                existing = whl_libraries[whl_name]
+
+                # TODO @aignas 2026-07-04: stop ignoring the index_url
+                diff = _diff_dict(existing, lib, ignore_keys = {"index_url": True})
+                if diff:
+                    fail("'{}' already in created:\n{}".format(
+                        whl_name,
+                        "\n".join([
+                            "    {}: {}".format(key, render.indent(render.dict(value)).lstrip())
+                            for key, value in diff.items()
+                            if value
+                        ]),
+                    ))
 
             whl_libraries[whl_name] = lib
 
@@ -1223,3 +1242,45 @@ This rule creates json files based on the whl_mods attribute.
         ),
     },
 )
+
+# TODO dedupe code
+
+def _diff_dict(first, second, *, ignore_keys = {}):
+    """A simple utility to shallow compare dictionaries.
+
+    Args:
+        first: The first dictionary to compare.
+        second: The second dictionary to compare.
+
+    Returns:
+        A dictionary containing the differences, with keys "common", "different",
+        "extra", and "missing", or None if the dictionaries are identical.
+    """
+    missing = {}
+    extra = {
+        key: value
+        for key, value in second.items()
+        if key not in first and key not in ignore_keys
+    }
+    common = {}
+    different = {}
+
+    for key, value in first.items():
+        if key in ignore_keys:
+            continue
+        elif key not in second:
+            missing[key] = value
+        elif value == second[key]:
+            common[key] = value
+        else:
+            different[key] = (value, second[key])
+
+    if missing or extra or different:
+        return {
+            "common": common,
+            "different": different,
+            "extra": extra,
+            "missing": missing,
+        }
+    else:
+        return None

python/private/pypi/generate_whl_library_build_bazel.bzl

@@ -15,6 +15,7 @@
 """Generate the BUILD.bazel contents for a repo defined by a whl_library."""
 
 load("//python/private:text_util.bzl", "render")
+load(":labels.bzl", "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", "PACKAGE_METADATA_LABEL")
 
 # These are functions on how to render particular args, should be reused across all rendering
 # invocations to make things easier.
@@ -31,10 +32,11 @@ _RENDER_FNS = {
     "srcs_exclude": render.list,
     "tags": render.list,
 }
+
 def _render(**kwargs):
     return {
         arg: _RENDER_FNS.get(arg, repr)(value)
-        for arg, value in kwargs.items
+        for arg, value in kwargs.items()
     }
 
 # NOTE @aignas 2024-10-25: We have to keep this so that files in
@@ -50,11 +52,26 @@ package(default_visibility = ["//visibility:public"])
 
 def generate_whl_library_build_bazel(
         *,
-        # TODO @aignas 2026-07-04: add extra args that are used in this function
+        name,
         annotation = None,
         config_load,
+        copy_executables = {},
+        copy_files = {},
+        data_exclude = [],
+        dep_template,
+        enable_implicit_namespace_pkgs = False,
+        entry_points = {},
+        extras = [],
+        group_deps = [],
+        group_name = None,
+        metadata_name,
+        metadata_version,
+        namespace_package_files = {},
         purl = None,
         requires_dist = [],
+        sdist_filename = None,
+        srcs_exclude = [],
+        visibility = ["//visibility:public"],
         **kwargs):
     """Generate a BUILD file for an unzipped Wheel
 
@@ -72,7 +89,7 @@ def generate_whl_library_build_bazel(
 
     loads = [
         """load("@package_metadata//rules:package_metadata.bzl", "package_metadata")""",
-        """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist")"""
+        """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist")""",
     ]
 
     srcs_kwargs = dict(
@@ -90,13 +107,11 @@ def generate_whl_library_build_bazel(
         copy_files = copy_files,
         copy_executables = copy_executables,
         namespace_package_files = namespace_package_files,
-        data = [],
         visibility = visibility,
     )
     from_requires_kwargs = dict(
-        name = name,
-        metadata_name = metadata_name,
-        metadata_version = metadata_version,
+        name = metadata_name,
+        version = metadata_version,
         requires_dist = requires_dist,
         extras = extras,
         group_deps = group_deps,
@@ -121,15 +136,15 @@ def generate_whl_library_build_bazel(
         render.call(
             "package_metadata",
             **_render(
-                name = "package_metadata",
+                name = PACKAGE_METADATA_LABEL,
                 purl = purl,
                 visibility = ["//:__subpackages__"],
-            ),
+            )
         ),
         render.call(
             "whl_library_srcs",
             **_render(**srcs_kwargs)
-        )
+        ),
     ]
 
     if config_load:
@@ -138,7 +153,7 @@ def generate_whl_library_build_bazel(
 
     macro_parts.append(render.call(
         "whl_library_from_requires_dist",
-        **_render(**from_requires_kwargs),
+        **_render(**from_requires_kwargs)
     ))
 
     contents = "\n".join(
@@ -155,7 +170,15 @@ def generate_whl_library_build_bazel(
 
 def generate_whl_library_deps_build_bazel(
         *,
-        # TODO @aignas 2026-07-04: add extra args that are used in this function
+        name,
+        version,
+        config_load,
+        dep_template,
+        extras,
+        group_deps,
+        group_name,
+        requires_dist,
+        whl_library,
         **kwargs):
     """Generate a BUILD file for an unzipped Wheel
 
@@ -165,32 +188,32 @@ def generate_whl_library_deps_build_bazel(
     """
 
     loads = [
-        """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_from_requires_dist")"""
+        """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_from_requires_dist")""",
     ]
 
     from_requires_kwargs = dict(
         name = name,
-        metadata_name = metadata_name,
-        metadata_version = metadata_version,
+        version = version,
         requires_dist = requires_dist,
         extras = extras,
         group_deps = group_deps,
         dep_template = dep_template,
         group_name = group_name,
+        src_pkg = str(whl_library),
     )
 
     macro_parts = [
         render.call(
             "alias",
-            name=target,
-            actual=whl_library.same_package_label(target),
+            **_render(
+                name = target,
+                actual = str(whl_library.same_package_label(target)),
+            )
         )
         for target in [
-            # TODO @aignas 2026-07-04: use ./labels.bzl for the following
-            "package_metadata",
-            "data",
-            "dist_info",
-            "extracted_whl_files",
+            DATA_LABEL,
+            DIST_INFO_LABEL,
+            EXTRACTED_WHEEL_FILES,
         ]
     ]
 
@@ -200,7 +223,7 @@ def generate_whl_library_deps_build_bazel(
 
     macro_parts.append(render.call(
         "whl_library_from_requires_dist",
-        **_render(**from_requires_kwargs),
+        **_render(**from_requires_kwargs)
     ))
 
     contents = _TEMPLATE.format(

python/private/pypi/hub_builder.bzl

@@ -316,7 +316,7 @@ def _add_whl_library(self, *, python_version, whl, repo):
         # are more platforms defined than there are wheels for and users
         # disallow building from sdist.
         return
-    
+
     forbidden_args = {
         "annotation": True,
         "extra_pip_args": True,
@@ -332,13 +332,14 @@ def _add_whl_library(self, *, python_version, whl, repo):
         deps_args = repo.args
     else:
         whl_repo_name = "whl_{}".format(repo.whl_repo_name)
-
-        self._whl_libraries[whl_repo_name] = {
-            k: v for k, v in repo.args.items()
+        _add_library(self, repos_dict = self._whl_libraries, name = whl_repo_name, args = {
+            k: v
+            for k, v in repo.args.items()
             if k not in forbidden_args | {
-                "config_load": True
+                "config_load": None,
+                "dep_template": None,
             }
-        }
+        })
 
         args = repo.args
 
@@ -351,21 +352,7 @@ def _add_whl_library(self, *, python_version, whl, repo):
         repos_dict = self._whl_library_deps
 
     repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name)
-    if repo_name in repos_dict:
-        diff = _diff_dict(repos_dict[repo_name], deps_args)
-        if diff:
-            self._logger.fail(lambda: (
-                "Attempting to create a duplicate library {repo_name} for {whl_name} with different arguments. Already existing declaration has:\n".format(
-                    repo_name = repo_name,
-                    whl_name = whl.name,
-                ) + "\n".join([
-                    "    {}: {}".format(key, render.indent(render.dict(value)).lstrip())
-                    for key, value in diff.items()
-                    if value
-                ])
-            ))
-            return
-    repos_dict[repo_name] = deps_args
+    _add_library(self, repos_dict = repos_dict, name = repo_name, args = deps_args)
 
     mapping = self._whl_map.setdefault(whl.name, {})
     if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name:
@@ -381,6 +368,22 @@ def _add_whl_library(self, *, python_version, whl, repo):
 
 ### end of setters, below we have various functions to implement the public methods
 
+def _add_library(self, *, repos_dict, name, args):
+    if name in repos_dict:
+        diff = _diff_dict(repos_dict[name], args)
+        if diff:
+            self._logger.fail(lambda: (
+                "Attempting to create a duplicate library {name} with different arguments. Already existing declaration has:\n".format(
+                    repo_name = name,
+                ) + "\n".join([
+                    "    {}: {}".format(key, render.indent(render.dict(value)).lstrip())
+                    for key, value in diff.items()
+                    if value
+                ])
+            ))
+            return
+    repos_dict[name] = args
+
 def _set_get_index_urls(self, mctx, pip_attr):
     # Resolve the index URL through envsubst so the ``$VAR`` / ``${VAR:-default}``
     # form is honored when deciding whether the experimental index-url mode is

python/private/pypi/labels.bzl

@@ -21,6 +21,7 @@ PY_LIBRARY_PUBLIC_LABEL = "pkg"
 PY_LIBRARY_IMPL_LABEL = "_pkg"
 DATA_LABEL = "data"
 DIST_INFO_LABEL = "dist_info"
+PACKAGE_METADATA_LABEL = "package_metadata"
 NODEPS_LABEL = "no_deps"
 NODEPS_WHL_FILE_LABEL = "_whl_file"
 NODEPS_PY_LIBRARY_LABEL = "_srcs"

python/private/pypi/whl_library.bzl

@@ -637,15 +637,15 @@ def _whl_library_deps_impl(rctx):
         return
 
     build_file_contents = generate_whl_library_deps_build_bazel(
-        dep_template = rctx.attr.dep_template,
+        name = metadata.name,
+        version = metadata.version,
         config_load = rctx.attr.config_load,
-        metadata_name = metadata.name,
-        metadata_version = metadata.version,
-        requires_dist = metadata.requires_dist,
+        dep_template = rctx.attr.dep_template,
+        extras = rctx.attr.extras,
         group_deps = rctx.attr.group_deps,
         group_name = rctx.attr.group_name,
-        extras = requirement(rctx.attr.requirement).extras,
-        whl_library=rctx.attr.whl_library,
+        requires_dist = metadata.requires_dist,
+        whl_library = rctx.attr.whl_library,
     )
 
     rctx.file("WORKSPACE")