feat: enable pyproject.toml as single source of truth for Python version

Author: golithe

.gitattributes

@@ -5,3 +5,4 @@ python/private/runtimes_manifest_workspace.bzl text eol=lf
 python/private/runtimes_manifest.txt text eol=lf
 
 *.bat text eol=crlf
+requirements_lock.txt linguist-generated=true

.gitignore

@@ -48,8 +48,10 @@ user.bazelrc
 # CLion
 .clwb
 
-# Python cache
+# Python artifacts
 **/__pycache__/
+*.egg
+*.egg-info
 
 # MODULE.bazel.lock is ignored for now as per recommendation from upstream.
 # See https://github.com/bazelbuild/bazel/issues/20369

CHANGELOG.md

@@ -145,6 +145,8 @@ and [#1975](https://github.com/bazel-contrib/rules_python/issues/1975).
 * (toolchains) Support dynamically fetching and registering Python runtimes
   from a python-build-standalone manifest file using
   `python.override(add_runtime_manifest_urls = ..., runtime_manifest_sha = ...)`.
+* (pip,python) Added `pyproject_toml` attribute to `pip.default()` and `python.defaults()`
+  to read Python version from pyproject.toml `requires-python` field (must be `==X.Y.Z` format).
 * (toolchain) Added {obj}`python.override.toolchain_target_settings` to allow
   adding `config_setting` labels to all registered toolchains.
 * (windows) Full venv support for Windows is available. Set

python/private/BUILD.bazel

@@ -709,6 +709,8 @@ bzl_library(
         ":full_version",
         ":pbs_manifest",
         ":platform_info",
+        ":pyproject_repo",
+        ":pyproject_utils",
         ":python_register_toolchains",
         ":pythons_hub",
         ":repo_utils",
@@ -941,6 +943,16 @@ bzl_library(
     srcs = ["py_runtime_info.bzl"],
 )
 
+bzl_library(
+    name = "pyproject_repo",
+    srcs = ["pyproject_repo.bzl"],
+)
+
+bzl_library(
+    name = "pyproject_utils",
+    srcs = ["pyproject_utils.bzl"],
+)
+
 bzl_library(
     name = "repo_utils",
     srcs = ["repo_utils.bzl"],

python/private/pypi/BUILD.bazel

@@ -125,6 +125,7 @@ bzl_library(
         ":whl_library",
         "//python/private:auth",
         "//python/private:normalize_name",
+        "//python/private:pyproject_utils",
         "//python/private:repo_utils",
         "@pythons_hub//:interpreters",
         "@pythons_hub//:versions",

python/private/pypi/extension.bzl

@@ -20,6 +20,7 @@ load("@rules_python_internal//:rules_python_config.bzl", rp_config = "config")
 load("@toml.bzl", "toml")
 load("//python/private:auth.bzl", "AUTH_ATTRS")
 load("//python/private:normalize_name.bzl", "normalize_name")
+load("//python/private:pyproject_utils.bzl", "read_pyproject_version")
 load("//python/private:repo_utils.bzl", "repo_utils")
 load(":hub_builder.bzl", "hub_builder")
 load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json")
@@ -208,6 +209,7 @@ def build_config(
     default_hub = None
     defaults = {
         "platforms": default_platforms(),
+        "python_version": None,
     }
     for mod in module_ctx.modules:
         if not (mod.is_root or mod.name == "rules_python"):
@@ -219,6 +221,15 @@ def build_config(
                     if default_hub:
                         fail("Duplicate pip.default tag: only one explicit default PyPI hub is allowed.")
                     default_hub = tag.default_hub
+            pyproject_toml = tag.pyproject_toml
+            if pyproject_toml:
+                pyproject_version = read_pyproject_version(
+                    module_ctx,
+                    pyproject_toml,
+                    logger = None,
+                )
+                if pyproject_version:
+                    defaults["python_version"] = pyproject_version
 
             platform = tag.platform
             if platform:
@@ -256,6 +267,7 @@ def build_config(
         default_hub = default_hub,
         index_url = defaults.get("index_url", "https://pypi.org/simple").rstrip("/"),
         netrc = defaults.get("netrc", None),
+        python_version = defaults.get("python_version", None),
         platforms = {
             name: _plat(**values)
             for name, values in defaults["platforms"].items()
@@ -359,6 +371,10 @@ You cannot use both the additive_build_content and additive_build_content_file a
 
     for mod in module_ctx.modules:
         for pip_attr in mod.tags.parse:
+            python_version = pip_attr.python_version or config.python_version
+            if not python_version:
+                _fail("pip.parse() requires either python_version attribute or pip.default(pyproject_toml=...) to be set")
+
             hub_name = pip_attr.hub_name
             if hub_name == "pypi":
                 if is_pypi_hub_reserved:
@@ -422,6 +438,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
             builder.pip_parse(
                 module_ctx,
                 pip_attr = pip_attr,
+                python_version = python_version,
             )
 
     # Keeps track of all the hub's whl repos across the different versions.
@@ -699,6 +716,21 @@ If you are defining custom platforms in your project and don't want things to cl
 [isolation] feature.
 
 [isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate
+""",
+    ),
+    "pyproject_toml": attr.label(
+        mandatory = False,
+        doc = """\
+Label pointing to pyproject.toml file to read the default Python version from.
+When specified, reads the `requires-python` field from pyproject.toml and uses
+it as the default python_version for all `pip.parse()` calls that don't
+explicitly specify one.
+
+The version must be specified as `==X.Y.Z` (exact version with full semver).
+This is designed to work with dependency management tools like Renovate.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
 """,
     ),
     "whl_abi_tags": attr.string_list(
@@ -870,14 +902,18 @@ find in case extra indexes are specified.
             default = True,
         ),
         "python_version": attr.string(
-            mandatory = True,
+            mandatory = False,
             doc = """
 The Python version the dependencies are targetting, in Major.Minor format
 (e.g., "3.11") or patch level granularity (e.g. "3.11.1").
 
 If an interpreter isn't explicitly provided (using `python_interpreter` or
 `python_interpreter_target`), then the version specified here must have
 a corresponding `python.toolchain()` configured.
+
+:::{seealso}
+The {obj}`pyproject_toml` attribute for getting the version from a project file.
+:::
 """,
         ),
         "simpleapi_skip": attr.string_list(

python/private/pypi/hub_builder.bzl

@@ -144,8 +144,8 @@ def _build(self):
         whl_libraries = self._whl_libraries,
     )
 
-def _pip_parse(self, module_ctx, pip_attr):
-    python_version = pip_attr.python_version
+def _pip_parse(self, module_ctx, pip_attr, python_version = None):
+    python_version = python_version or pip_attr.python_version
     if python_version in self._platforms:
         fail((
             "Duplicate pip python version '{version}' for hub " +
@@ -191,7 +191,8 @@ def _pip_parse(self, module_ctx, pip_attr):
         self,
         module_ctx,
         pip_attr = pip_attr,
-        enable_pipstar_extract = bool(self._config.enable_pipstar_extract or self._get_index_urls.get(pip_attr.python_version)),
+        python_version = python_version,
+        enable_pipstar_extract = bool(self._config.enable_pipstar_extract or self._get_index_urls.get(python_version)),
     )
 
 ### end of PUBLIC methods
@@ -393,11 +394,11 @@ def _set_get_index_urls(self, mctx, pip_attr):
     )
     return True
 
-def _detect_interpreter(self, pip_attr):
+def _detect_interpreter(self, pip_attr, python_version):
     python_interpreter_target = pip_attr.python_interpreter_target
     if python_interpreter_target == None and not pip_attr.python_interpreter:
         python_name = "python_{}_host".format(
-            pip_attr.python_version.replace(".", "_"),
+            python_version.replace(".", "_"),
         )
         if python_name not in self._available_interpreters:
             fail((
@@ -407,7 +408,7 @@ def _detect_interpreter(self, pip_attr):
                 "Expected to find {python_name} among registered versions:\n  {labels}"
             ).format(
                 hub_name = self.name,
-                version = pip_attr.python_version,
+                version = python_version,
                 python_name = python_name,
                 labels = "  \n".join(self._available_interpreters),
             ))
@@ -476,17 +477,19 @@ def _create_whl_repos(
         module_ctx,
         *,
         pip_attr,
+        python_version,
         enable_pipstar_extract = False):
     """create all of the whl repositories
 
     Args:
         self: the builder.
         module_ctx: {type}`module_ctx`.
         pip_attr: {type}`struct` - the struct that comes from the tag class iteration.
+        python_version: {type}`str` - the resolved python version for this pip.parse call.
         enable_pipstar_extract: {type}`bool` - enable the pipstar extraction or not.
     """
     logger = self._logger
-    platforms = self._platforms[pip_attr.python_version]
+    platforms = self._platforms[python_version]
     requirements_by_platform = parse_requirements(
         module_ctx,
         requirements_by_platform = requirements_files_by_platform(
@@ -498,7 +501,7 @@ def _create_whl_repos(
             extra_pip_args = pip_attr.extra_pip_args,
             platforms = sorted(platforms),  # here we only need keys
             python_version = full_version(
-                version = pip_attr.python_version,
+                version = python_version,
                 minor_mapping = self._minor_mapping,
             ),
             logger = logger,
@@ -528,7 +531,7 @@ def _create_whl_repos(
         pip_attr = pip_attr,
     )
 
-    interpreter = _detect_interpreter(self, pip_attr)
+    interpreter = _detect_interpreter(self, pip_attr, python_version)
 
     for whl in requirements_by_platform:
         whl_library_args = common_args | _whl_library_args(
@@ -543,16 +546,16 @@ def _create_whl_repos(
                 whl_library_args = whl_library_args,
                 download_only = pip_attr.download_only,
                 netrc = self._config.netrc or pip_attr.netrc,
-                use_downloader = src.url and _use_downloader(self, pip_attr.python_version, whl.name),
+                use_downloader = src.url and _use_downloader(self, python_version, whl.name),
                 auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns,
-                python_version = _major_minor_version(pip_attr.python_version),
+                python_version = _major_minor_version(python_version),
                 is_multiple_versions = whl.is_multiple_versions,
                 interpreter = interpreter,
                 enable_pipstar_extract = enable_pipstar_extract,
             )
             _add_whl_library(
                 self,
-                python_version = pip_attr.python_version,
+                python_version = python_version,
                 whl = whl,
                 repo = repo,
             )

python/private/pyproject_repo.bzl

@@ -0,0 +1,80 @@
+"""Repository rule to expose Python version from pyproject.toml."""
+
+_TOML2JSON = Label("//tools/private/toml2json:toml2json.py")
+
+def _parse_requires_python(requires_python):
+    """Parse and validate the requires-python field."""
+    if not requires_python.startswith("=="):
+        fail("requires-python must use '==' for exact version, got: {}".format(requires_python))
+
+    bare_version = requires_python[2:].strip()
+    parts = bare_version.split(".")
+    if len(parts) != 3:
+        fail("requires-python must be in X.Y.Z format, got: {}".format(bare_version))
+    for part in parts:
+        if not part.isdigit():
+            fail("requires-python must be in X.Y.Z format, got: {}".format(bare_version))
+
+    return bare_version
+
+def _pyproject_version_repo_impl(rctx):
+    """Create a repository that exports PYTHON_VERSION from pyproject.toml."""
+    pyproject_path = rctx.path(rctx.attr.pyproject_toml)
+    rctx.read(pyproject_path, watch = "yes")
+
+    toml2json = rctx.path(_TOML2JSON)
+    result = rctx.execute([
+        "python3",
+        str(toml2json),
+        str(pyproject_path),
+    ])
+
+    if result.return_code != 0:
+        fail("Failed to parse pyproject.toml: " + result.stderr)
+
+    data = json.decode(result.stdout)
+    requires_python = data.get("project", {}).get("requires-python")
+    if not requires_python:
+        fail("pyproject.toml must contain [project] requires-python field")
+
+    version = _parse_requires_python(requires_python)
+
+    rctx.file("version.bzl", """\
+\"\"\"Python version from pyproject.toml.
+
+This file is automatically generated. Do not edit.
+\"\"\"
+
+PYTHON_VERSION = "{version}"
+""".format(version = version))
+
+    rctx.file("BUILD.bazel", """\
+# Automatically generated from pyproject.toml
+exports_files(["version.bzl"])
+""")
+
+pyproject_version_repo = repository_rule(
+    implementation = _pyproject_version_repo_impl,
+    attrs = {
+        "pyproject_toml": attr.label(
+            mandatory = True,
+            doc = "Label pointing to pyproject.toml file.",
+        ),
+    },
+    doc = """Repository rule that reads Python version from pyproject.toml.
+
+This rule creates a repository with a `version.bzl` file that exports
+`PYTHON_VERSION` constant.
+
+Example:
+```python
+    load("@python_version_from_pyproject//:version.bzl", "PYTHON_VERSION")
+
+    compile_pip_requirements(
+        name = "requirements",
+        python_version = PYTHON_VERSION,
+        requirements_txt = "requirements.txt",
+    )
+```
+""",
+)