feat(venv): support data, include, and scripts schemes (#3726)

Currently, the files from the data, headers, and scripts portions of a
wheel don't end up in the proper sub-directories of the venv. This
means the full files of a distribution aren't available at the typical
location in the venv, making it harder to integrate with standard
tools.

To fix, simply map the directories to paths in the venv and give them
similar treatment as site-packages.

The spec says certain first-level directories of the `.data` directory
map to specific scheme paths, which `whl_library` already handles.
Here's a listing of the
`wheel data directory -> install scheme path key -> whl_library
directory`
relationships

* purelib -> purelib -> site-packages
* platlib -> platlib -> site-packages
* headers -> include -> include
* scripts -> scripts -> bin
* data -> data -> data

Relevant reading:
* Packaging specification:

https://packaging.python.org/en/latest/specifications/binary-distribution-format
* Posix install scheme paths:
  https://docs.python.org/3/library/sysconfig.html#posix-prefix
* Windows install scheme paths:
  https://docs.python.org/3/library/sysconfig.html#nt

The whl_library rule uses posix names for extracting. When
materialized into a binary's venv, platform specific names are used:

* bin -> (posix) bin; (Windows) Scripts
* include -> (posix) include; (Windows) Include (capital i)
* data -> venv root directory

Along the way ...

* The data files (files under the "data" scheme of a whl) are now
  always included as part of depending on the library. They would be
  in included in venv_site_packages=yes mode, so this better aligns
  behavior of the two modes.
* Rename `is_venv_site_packages` to `_is_venv_site_packages_yes` to
  better represent the purpose and visibility of it.
* Make whl_from_dir support Windows. Testing of venvs relies on
  using it for testing various special cases.

Author: rickeylev

CHANGELOG.md

@@ -60,6 +60,8 @@ END_UNRELEASED_TEMPLATE
 ### Changed
 * (gazelle) WORKSPACE's bazel-gazelle dependency bumped from 0.36.0 to 0.47.0.
   The go version was also bumped from 1.21.13 to 1.22.9.
+* (pypi) The data files of a wheel (bin, includes, etc) are now always included
+  as a library's data dependencies.
 
 {#v0-0-0-fixed}
 ### Fixed
@@ -74,6 +76,8 @@ END_UNRELEASED_TEMPLATE
   adding `config_setting` labels to all registered toolchains.
 * (windows) Full venv support for Windows is available. Set
   {obj}`--venvs_site_packages=yes` to enable.
+* (test/binaries) When {obj}`--venv_site_packages=yes` is enabled,
+  wheel `data`, `bin`, and `include` files are populated into the venv.
 * (runfiles) Added a pathlib-compatible API: {obj}`Runfiles.root()`
   Fixes [#3296](https://github.com/bazel-contrib/rules_python/issues/3296).
 * (toolchains) `3.13.12`, `3.14.3` Python toolchain from [20260325] release.

MODULE.bazel

@@ -246,6 +246,8 @@ use_repo(
     "somepkg_with_build_files",
     "whl_library_extras_direct_dep",
     "whl_with_build_files",
+    "whl_with_data1",
+    "whl_with_data2",
 )
 
 dev_rules_python_config = use_extension(

examples/pip_parse/pip_parse_test.py

@@ -52,16 +52,16 @@ def test_data(self):
         self.assertIsNotNone(actual)
         actual = self._remove_leading_dirs(actual.split(" "))
 
-        self.assertListEqual(
-            actual,
-            [
-                "data/share/doc/packages/s3cmd/INSTALL.md",
-                "data/share/doc/packages/s3cmd/LICENSE",
-                "data/share/doc/packages/s3cmd/NEWS",
-                "data/share/doc/packages/s3cmd/README.md",
-                "data/share/man/man1/s3cmd.1",
-            ],
-        )
+        expected = [
+            "bin/s3cmd",
+            "data/share/doc/packages/s3cmd/INSTALL.md",
+            "data/share/doc/packages/s3cmd/LICENSE",
+            "data/share/doc/packages/s3cmd/NEWS",
+            "data/share/doc/packages/s3cmd/README.md",
+            "data/share/man/man1/s3cmd.1",
+        ]
+
+        self.assertListEqual(actual, expected)
 
     def test_dist_info(self):
         actual = os.environ.get("WHEEL_DIST_INFO_CONTENTS")

internal_dev_setup.bzl

@@ -26,6 +26,8 @@ load("//python:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS")
 load("//python/private:pythons_hub.bzl", "hub_repo")  # buildifier: disable=bzl-visibility
 load("//python/private:runtime_env_repo.bzl", "runtime_env_repo")  # buildifier: disable=bzl-visibility
 load("//python/private/pypi:deps.bzl", "pypi_deps")  # buildifier: disable=bzl-visibility
+load("//python/private/pypi:whl_library.bzl", "whl_library")  # buildifier: disable=bzl-visibility
+load("//tests/support/whl_from_dir:whl_from_dir_repo.bzl", "whl_from_dir_repo")  # buildifier: disable=bzl-visibility
 
 def rules_python_internal_setup():
     """Setup for development and testing of rules_python itself."""
@@ -59,3 +61,25 @@ def rules_python_internal_setup():
     bazel_features_deps()
     rules_shell_dependencies()
     rules_shell_toolchains()
+
+    whl_from_dir_repo(
+        name = "whl_with_data1_whl",
+        root = "//tests/repos/whl_with_data1:BUILD.bazel",
+        output = "whl_with_data1-1.0-any-none-any.whl",
+    )
+    whl_library(
+        name = "whl_with_data1",
+        whl_file = "@whl_with_data1_whl//:whl_with_data1-1.0-any-none-any.whl",
+        requirement = "whl-with-data1",
+    )
+
+    whl_from_dir_repo(
+        name = "whl_with_data2_whl",
+        root = "//tests/repos/whl_with_data2:BUILD.bazel",
+        output = "whl_with_data2-1.0-any-none-any.whl",
+    )
+    whl_library(
+        name = "whl_with_data2",
+        whl_file = "@whl_with_data2_whl//:whl_with_data2-1.0-any-none-any.whl",
+        requirement = "whl-with-data2",
+    )

python/config_settings/BUILD.bazel

@@ -229,7 +229,7 @@ string_flag(
 )
 
 config_setting(
-    name = "is_venvs_site_packages",
+    name = "_is_venvs_site_packages_yes",
     flag_values = {
         ":venvs_site_packages": VenvsSitePackages.YES,
     },

python/private/internal_dev_deps.bzl

@@ -91,6 +91,28 @@ def _internal_dev_deps_impl(mctx):
         enable_implicit_namespace_pkgs = False,
     )
 
+    whl_from_dir_repo(
+        name = "whl_with_data1_whl",
+        root = "//tests/repos/whl_with_data1:BUILD.bazel",
+        output = "whl_with_data1-1.0-any-none-any.whl",
+    )
+    whl_library(
+        name = "whl_with_data1",
+        whl_file = "@whl_with_data1_whl//:whl_with_data1-1.0-any-none-any.whl",
+        requirement = "whl-with-data1",
+    )
+
+    whl_from_dir_repo(
+        name = "whl_with_data2_whl",
+        root = "//tests/repos/whl_with_data2:BUILD.bazel",
+        output = "whl_with_data2-1.0-any-none-any.whl",
+    )
+    whl_library(
+        name = "whl_with_data2",
+        whl_file = "@whl_with_data2_whl//:whl_with_data2-1.0-any-none-any.whl",
+        requirement = "whl-with-data2",
+    )
+
     _whl_library_from_dir(
         name = "whl_library_extras_direct_dep",
         root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel",

python/private/py_executable.bzl

@@ -575,9 +575,13 @@ def _create_venv(ctx, output_prefix, imports, runtime_details, add_runfiles_root
         computed_substitutions = computed_subs,
     )
 
+    # See https://docs.python.org/3/library/sysconfig.html#posix-prefix
+    # for how schemes map under the venv.
     venv_dir_map = {
-        VenvSymlinkKind.BIN: venv_details.bin_dir,
+        VenvSymlinkKind.BIN: "{}/{}".format(venv_ctx_rel_root, venv_details.bin_dir),
         VenvSymlinkKind.LIB: site_packages,
+        VenvSymlinkKind.INCLUDE: "{}/{}".format(venv_ctx_rel_root, venv_details.include_dir),
+        VenvSymlinkKind.DATA: venv_ctx_rel_root,
     }
     venv_app_files = create_venv_app_files(
         ctx,
@@ -659,7 +663,7 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
 
     recreate_venv_at_runtime = False
 
-    bin_dir = "{}/bin".format(venv_ctx_rel_root)
+    venv_bin_ctx_rel_path = "{}/bin".format(venv_ctx_rel_root)
     if create_full_venv:
         # Some wrappers around the interpreter (e.g. pyenv) use the program
         # name to decide what to do, so preserve the name.
@@ -671,15 +675,15 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
             # When the venv symlinks are disabled, the $venv/bin/python3 file isn't
             # needed or used at runtime. However, the zip code uses the interpreter
             # File object to figure out some paths.
-            interpreter = ctx.actions.declare_file("{}/{}".format(bin_dir, py_exe_basename))
+            interpreter = ctx.actions.declare_file("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename))
             ctx.actions.write(interpreter, "actual:{}".format(interpreter_actual_path))
 
         elif runtime.interpreter:
             # Even though ctx.actions.symlink() is used, using
             # declare_symlink() is required to ensure that the resulting file
             # in runfiles is always a symlink. An RBE implementation, for example,
             # may choose to write what symlink() points to instead.
-            interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename))
+            interpreter = ctx.actions.declare_symlink("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename))
             interpreter_runfiles.add(interpreter)
 
             rel_path = relative_path(
@@ -690,7 +694,7 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
             )
             ctx.actions.symlink(output = interpreter, target_path = rel_path)
         else:
-            interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename))
+            interpreter = ctx.actions.declare_symlink("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename))
             interpreter_runfiles.add(interpreter)
             ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path)
     else:
@@ -715,7 +719,8 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
         interpreter = interpreter,
         pyvenv_cfg = pyvenv_cfg,
         site_packages = site_packages,
-        bin_dir = bin_dir,
+        bin_dir = "bin",
+        include_dir = "include",
         recreate_venv_at_runtime = recreate_venv_at_runtime,
         interpreter_runfiles = interpreter_runfiles.build(ctx),
         interpreter_symlinks = depset(),
@@ -777,7 +782,8 @@ def _create_venv_windows(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_
         interpreter = interpreter,
         pyvenv_cfg = None,
         site_packages = site_packages,
-        bin_dir = venv_bin_ctx_rel_path,
+        bin_dir = venv_bin_rel_path,
+        include_dir = "Include",
         recreate_venv_at_runtime = True,
         interpreter_runfiles = interpreter_runfiles.build(ctx),
         interpreter_symlinks = interpreter_symlinks.build(),
@@ -789,6 +795,7 @@ def _venv_details(
         pyvenv_cfg,
         site_packages,
         bin_dir,
+        include_dir,
         recreate_venv_at_runtime,
         interpreter_runfiles,
         interpreter_symlinks):
@@ -801,8 +808,10 @@ def _venv_details(
         pyvenv_cfg = pyvenv_cfg,
         # str; venv-relative path to the site-packages directory
         site_packages = site_packages,
-        # str; ctx-relative path to the venv's bin directory.
+        # str; venv-relative path to the venv's bin directory.
         bin_dir = bin_dir,
+        # str; venv-relative-path to the venv's include directory.
+        include_dir = include_dir,
         # bool; True if the venv needs to be recreated at runtime (because the
         # build-time construction isn't sufficient). False if the build-time
         # constructed venv is sufficient.

python/private/py_info.bzl

@@ -37,6 +37,12 @@ def _VenvSymlinkKind_typedef():
 
     Indicates to create paths under the venv's include directory.
     :::
+
+    :::{field} DATA
+    :type: object
+
+    Indicates to create paths under the venv's data directory.
+    :::
     """
 
 # buildifier: disable=name-conventions
@@ -45,6 +51,7 @@ VenvSymlinkKind = struct(
     BIN = "BIN",
     LIB = "LIB",
     INCLUDE = "INCLUDE",
+    DATA = "DATA",
 )
 
 def _VenvSymlinkEntry_init(**kwargs):

python/private/pypi/whl_library_targets.bzl

@@ -42,6 +42,8 @@ _BAZEL_REPO_FILE_GLOBS = [
     "WORKSPACE.bazel",
 ]
 
+_IS_VENV_SITE_PACKAGES_YES = Label("//python/config_settings:_is_venvs_site_packages_yes")
+
 def whl_library_targets_from_requires(
         *,
         name,
@@ -191,7 +193,7 @@ def whl_library_targets(
                 include = ["site-packages/*.dist-info/**"],
             ),
             DATA_LABEL: dict(
-                include = ["data/**"],
+                include = ["data/**", "bin/**", "include/**"],
             ),
         }
 
@@ -351,7 +353,7 @@ def whl_library_targets(
 
         if not enable_implicit_namespace_pkgs:
             generated_namespace_package_files = select({
-                Label("//python/config_settings:is_venvs_site_packages"): [],
+                _IS_VENV_SITE_PACKAGES_YES: [],
                 "//conditions:default": rules.create_inits(
                     srcs = srcs + data + pyi_srcs,
                     ignored_dirnames = [],  # If you need to ignore certain folders, you can patch rules_python here to do so.
@@ -361,6 +363,10 @@ def whl_library_targets(
             namespace_package_files += generated_namespace_package_files
             srcs = srcs + generated_namespace_package_files
 
+        # This is done after create_inits() is called so that the data scheme
+        # files don't have such files created in their directories.
+        data = data + [DATA_LABEL]
+
         rules.py_library(
             name = py_library_label,
             srcs = srcs,