feat(pypi): add pip.dep to declare abstract pypi dependencies (bazel-contrib#3850)

Introduce the pip.dep tag class to allow modules to declare abstract
PyPI dependencies. These declarations ensure the target structure for
a PyPI package is created, but provide a no-op implementation that
passes analysis time, but fails at execution time.

Along the way, create an agent rule for `*.bzl` files to help guide it
in creating better bzl files.

Author: rickeylev

.agents/rules/bzl.md

@@ -0,0 +1,10 @@
+---
+trigger: glob
+description: Starlark / Bazel .bzl file coding style rules
+globs: *.bzl
+---
+
+# Starlark Rules
+
+* Use triple-quoted strings for multi-line rule doc args.
+* Don't use backslash line continuation in rule doc args.

docs/pypi/download.md

@@ -121,6 +121,41 @@ Shared library targets can simply depend on the unified hub (e.g.,
 `@pypi//numpy`), and the dependency will automatically resolve to the correct
 wheel version from the active hub during the build.
 
+### Declaring Abstract Dependencies (pip.dep)
+
+:::{versionadded} VERSION_NEXT_FEATURE
+Declaring abstract PyPI dependencies via `pip.dep` tags.
+:::
+
+Sometimes a shared library target or a ruleset needs to depend on a PyPI
+package (e.g., `@pypi//numpy`), but does not want to force a specific package
+version or a concrete `requirements.txt` lock file on its consumers.
+
+Instead of calling `pip.parse()`, the module can declare its dependency using
+the `pip.dep` tag:
+
+```starlark
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+
+# Declare an abstract dependency on 'numpy' and specify extra targets that
+# are expected to be available in the package.
+pip.dep(
+    name = "numpy",
+    extra_targets = ["extra-alias"],
+)
+```
+
+This ensures that the target structure `@pypi//numpy` (and
+`@pypi//numpy:extra-alias`) exists in the unified `@pypi` hub repository, so the
+declaring module can compile and analyze successfully without needing any local
+requirements file.
+
+The actual concrete implementation and version of the package must be provided
+by a downstream module calling `pip.parse`.
+
+If a downstream module attempts to build a target that depends on an abstract
+dependency, but has not provided a concrete implementation for it via any
+`pip.parse` call, the build will fail at execution time.
 
 
 As with any repository rule or extension, if you would like to ensure that `pip_parse` is

news/pip-dep-tag-class.added.md

@@ -0,0 +1,4 @@
+(pypi) Added a `dep` tag class to the `pip` bzlmod extension. This allows
+modules to declare abstract PyPI dependencies, ensuring target structures
+exist in the unified hub, while allowing other modules to provide the
+concrete implementation via `pip.parse`.

python/private/pypi/extension.bzl

@@ -424,6 +424,15 @@ You cannot use both the additive_build_content and additive_build_content_file a
                 pip_attr = pip_attr,
             )
 
+    # dict[str package, dict[str, None] extra_targets]
+    declared_deps = {}
+    for mod in module_ctx.modules:
+        for dep_attr in mod.tags.dep:
+            name = normalize_name(dep_attr.name)
+            targets = declared_deps.setdefault(name, {})
+            for target in dep_attr.extra_targets:
+                targets[target] = None
+
     # Keeps track of all the hub's whl repos across the different versions.
     # dict[hub, dict[whl, dict[version, str pip]]]
     # Where hub, whl, and pip are the repo names
@@ -448,6 +457,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
 
     return struct(
         config = config,
+        declared_deps = declared_deps,
         default_hub = config.default_hub or renamed_default_hub,
         exposed_packages = exposed_packages,
         extra_aliases = extra_aliases,
@@ -492,6 +502,15 @@ def _create_unified_hub_repo(mods):
                 if hub_name not in extra_aliases[qual_alias]:
                     extra_aliases[qual_alias].append(hub_name)
 
+    for norm_pkg, extra_targets in mods.declared_deps.items():
+        if norm_pkg not in packages:
+            packages[norm_pkg] = []
+
+        for target_name in extra_targets:
+            qual_alias = "%s:%s" % (norm_pkg, target_name)
+            if qual_alias not in extra_aliases:
+                extra_aliases[qual_alias] = []
+
     unified_hub_repo(
         name = "pypi",
         default_hub = mods.default_hub or (hubs[0] if hubs else ""),
@@ -1015,6 +1034,31 @@ Apply any overrides (e.g. patches) to a given Python distribution defined by
 other tags in this extension.""",
 )
 
+_dep_tag = tag_class(
+    attrs = {
+        "extra_targets": attr.string_list(
+            doc = """\
+A list of extra target names in the package that are expected to be available.
+See {obj}`pip.parse.extra_hub_aliases`.
+""",
+            default = [],
+        ),
+        "name": attr.string(
+            doc = "The name of a pypi package. Note that the name is normalized.",
+            mandatory = True,
+        ),
+    },
+    doc = """\
+Declare an abstract PyPI dependency to ensure its target structure exists in the unified hub.
+
+This is useful for targets or rules that need to depend on a package (e.g., `@pypi//numpy`)
+but do not want to force a specific version or concrete requirements lock file on their
+consumers. The concrete version and implementation must be provided by downstreams calling
+`pip.parse`. If they are not, the target will still be defined, but it will result in an
+execution-phase error when built.
+""",
+)
+
 pypi = module_extension(
     environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"],
     doc = """\
@@ -1065,6 +1109,7 @@ terms used in this extension.
 :::
 """,
         ),
+        "dep": _dep_tag,
         "override": _override_tag,
         "parse": tag_class(
             attrs = _pip_parse_ext_attrs(),

python/private/pypi/missing_package.bzl

@@ -6,12 +6,19 @@ load("//python/private:reexports.bzl", "BuiltinPyInfo")
 def _missing_package_error_impl(ctx):
     out = ctx.actions.declare_file(ctx.label.name + ".error")
 
+    if ctx.attr.hub_name:
+        hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format(
+            hub = ctx.attr.hub_name,
+        )
+    else:
+        hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv'
+
     # Register an action that fails when Bazel attempts to stage/build this file
     ctx.actions.run_shell(
         outputs = [out],
         command = "echo 'ERROR: PyPI package \"{pkg}\" is not available{hub_clause}.' >&2 && exit 1".format(
             pkg = ctx.attr.package_name,
-            hub_clause = (' when building under PyPI hub "%s"' % ctx.attr.hub_name) if ctx.attr.hub_name else " because no PyPI hub or default hub is requested",
+            hub_clause = hub_clause,
         ),
     )
 

tests/integration/unified_pypi/BUILD.bazel

@@ -46,3 +46,16 @@ py_binary(
     },
     deps = ["@pypi//six"],
 )
+
+py_binary(
+    name = "bin_declared_only",
+    srcs = ["bin_declared_only.py"],
+    deps = ["@pypi//declared_only_pkg"],
+)
+
+py_binary(
+    name = "bin_declared_only_alias",
+    srcs = ["bin_declared_only.py"],
+    main = "bin_declared_only.py",
+    deps = ["@pypi//declared_only_pkg:declared-only-alias"],
+)

tests/integration/unified_pypi/MODULE.bazel

@@ -45,4 +45,8 @@ pip.parse(
 use_repo(pip, "pypi_b")
 
 pip.default(default_hub = "pypi_b")
+pip.dep(
+    name = "declared-only-pkg",
+    extra_targets = ["declared-only-alias"],
+)
 use_repo(pip, "pypi")

tests/integration/unified_pypi/bin_declared_only.py

@@ -0,0 +1,2 @@
+# Dummy file for integration test
+print("declared_only")

tests/integration/unified_pypi_test.py

@@ -30,7 +30,7 @@ def test_disjoint_package_cquery_succeeds_but_build_fails(self):
         )
         self.assert_result_matches(
             result,
-            'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_a".',
+            'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_a"\\. Try adding it to the requirements of this hub',
         )
 
     def test_sibling_extra_alias_cquery_succeeds_but_build_fails(self):
@@ -43,7 +43,7 @@ def test_sibling_extra_alias_cquery_succeeds_but_build_fails(self):
         )
         self.assert_result_matches(
             result,
-            'ERROR: PyPI package "colorama:my_colorama" is not available when building under PyPI hub "pypi_b".',
+            'ERROR: PyPI package "colorama:my_colorama" is not available when building under PyPI hub "pypi_b"\\. Try adding it to the requirements of this hub',
         )
 
     @contextlib.contextmanager
@@ -74,6 +74,27 @@ def test_invalid_default_hub_fails_evaluation(self):
                 "default_hub 'invalid_hub' is not a defined PyPI hub",
             )
 
+    def test_unimplemented_declared_dep_fails_build(self):
+        # Even though cquery succeeds:
+        self.run_bazel("cquery", "//:bin_declared_only")
+
+        # Build must fail because the package is not implemented by any concrete hub
+        result = self.run_bazel("build", "//:bin_declared_only", check=False)
+        self.assertNotEqual(result.exit_code, 0)
+        self.assert_result_matches(
+            result,
+            'ERROR: PyPI package "declared_only_pkg" is not available when building under PyPI hub "pypi_b"\\. Try adding it to the requirements of this hub',
+        )
+
+    def test_unimplemented_declared_dep_alias_fails_build(self):
+        # Build must fail for alias too
+        result = self.run_bazel("build", "//:bin_declared_only_alias", check=False)
+        self.assertNotEqual(result.exit_code, 0)
+        self.assert_result_matches(
+            result,
+            'ERROR: PyPI package "declared_only_pkg:declared-only-alias" is not available when building under PyPI hub "pypi_b"\\. Try adding it to the requirements of this hub',
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/pypi/extension/extension_tests.bzl

@@ -90,14 +90,21 @@ _default_tags_default = [
     }.items()
 ]
 
-def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], is_root = True):
+def _dep(*, name, extra_targets = []):
+    return struct(
+        name = name,
+        extra_targets = extra_targets,
+    )
+
+def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True):
     return struct(
         name = name,
         tags = struct(
             parse = parse,
             override = override,
             whl_mods = whl_mods,
             default = default,
+            dep = dep,
         ),
         is_root = is_root,
     )
@@ -106,6 +113,7 @@ def _parse_modules(env, **kwargs):
     return env.expect.that_struct(
         parse_modules(**kwargs),
         attrs = dict(
+            declared_deps = subjects.dict,
             default_hub = subjects.str,
             exposed_packages = subjects.dict,
             hub_group_map = subjects.dict,
@@ -435,6 +443,89 @@ def _test_default_hub_precedence(env):
 
 _tests.append(_test_default_hub_precedence)
 
+def _test_extension_dep(env):
+    pypi = _parse_modules(
+        env,
+        module_ctx = _pypi_mock_mctx(
+            _mod(
+                name = "my_module",
+                dep = [
+                    _dep(
+                        name = "declared-pkg",
+                        extra_targets = ["declared-alias"],
+                    ),
+                ],
+            ),
+            os_name = "linux",
+            arch_name = "x86_64",
+        ),
+        available_interpreters = {},
+        minor_mapping = {},
+    )
+
+    pypi.declared_deps().contains_exactly({"declared_pkg": {"declared-alias": None}})
+    pypi.exposed_packages().contains_exactly({})
+    pypi.hub_group_map().contains_exactly({})
+    pypi.hub_whl_map().contains_exactly({})
+    pypi.whl_libraries().contains_exactly({})
+    pypi.whl_mods().contains_exactly({})
+
+_tests.append(_test_extension_dep)
+
+def _test_extension_dep_coexists_with_concrete_hub(env):
+    pypi = _parse_modules(
+        env,
+        module_ctx = _pypi_mock_mctx(
+            _mod(
+                name = "my_module",
+                parse = [
+                    _parse(
+                        hub_name = "pypi_a",
+                        python_version = "3.15",
+                        simpleapi_skip = ["simple"],
+                        requirements_lock = "requirements.txt",
+                    ),
+                ],
+                dep = [
+                    _dep(
+                        name = "simple",
+                        extra_targets = ["extra-target"],
+                    ),
+                ],
+            ),
+            os_name = "linux",
+            arch_name = "x86_64",
+        ),
+        available_interpreters = {
+            "python_3_15_host": "unit_test_interpreter_target",
+        },
+        minor_mapping = {"3.15": "3.15.19"},
+    )
+
+    pypi.declared_deps().contains_exactly({"simple": {"extra-target": None}})
+    pypi.exposed_packages().contains_exactly({"pypi_a": ["simple"]})
+    pypi.hub_group_map().contains_exactly({"pypi_a": {}})
+    pypi.hub_whl_map().contains_exactly({"pypi_a": {
+        "simple": {
+            "pypi_a_315_simple": [
+                whl_config_setting(
+                    version = "3.15",
+                ),
+            ],
+        },
+    }})
+    pypi.whl_libraries().contains_exactly({
+        "pypi_a_315_simple": {
+            "config_load": "@pypi_a//:config.bzl",
+            "dep_template": "@pypi_a//{name}:{target}",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf",
+        },
+    })
+    pypi.whl_mods().contains_exactly({})
+
+_tests.append(_test_extension_dep_coexists_with_concrete_hub)
+
 def extension_test_suite(name):
     """Create the test suite.