From 12eac29b5d09d94e819f93af40e6eb661c4f2dc5 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 27 Apr 2026 00:50:48 -0700 Subject: [PATCH] feat(venv): support data, include, and scripts schemes (#3726) Currently, the files from the data, headers, and scripts portions of a wheel don't end up in the proper sub-directories of the venv. This means the full files of a distribution aren't available at the typical location in the venv, making it harder to integrate with standard tools. To fix, simply map the directories to paths in the venv and give them similar treatment as site-packages. The spec says certain first-level directories of the `.data` directory map to specific scheme paths, which `whl_library` already handles. Here's a listing of the `wheel data directory -> install scheme path key -> whl_library directory` relationships * purelib -> purelib -> site-packages * platlib -> platlib -> site-packages * headers -> include -> include * scripts -> scripts -> bin * data -> data -> data Relevant reading: * Packaging specification: https://packaging.python.org/en/latest/specifications/binary-distribution-format * Posix install scheme paths: https://docs.python.org/3/library/sysconfig.html#posix-prefix * Windows install scheme paths: https://docs.python.org/3/library/sysconfig.html#nt The whl_library rule uses posix names for extracting. When materialized into a binary's venv, platform specific names are used: * bin -> (posix) bin; (Windows) Scripts * include -> (posix) include; (Windows) Include (capital i) * data -> venv root directory Along the way ... * The data files (files under the "data" scheme of a whl) are now always included as part of depending on the library. They would be in included in venv_site_packages=yes mode, so this better aligns behavior of the two modes. * Rename `is_venv_site_packages` to `_is_venv_site_packages_yes` to better represent the purpose and visibility of it. * Make whl_from_dir support Windows. Testing of venvs relies on using it for testing various special cases. --- CHANGELOG.md | 4 + MODULE.bazel | 2 + examples/pip_parse/pip_parse_test.py | 20 ++-- internal_dev_setup.bzl | 24 +++++ python/config_settings/BUILD.bazel | 2 +- python/private/internal_dev_deps.bzl | 22 +++++ python/private/py_executable.bzl | 25 +++-- python/private/py_info.bzl | 7 ++ python/private/pypi/whl_library_targets.bzl | 10 +- python/private/venv_runfiles.bzl | 72 ++++++++++++-- .../whl_library_targets_tests.bzl | 20 ++-- tests/repos/whl_with_data1/BUILD.bazel | 1 + .../data/bin/data_overlap.sh | 1 + .../data/include/data_overlap.h | 1 + .../data/overlap/both.txt | 1 + .../data/overlap/data1.txt | 1 + .../data/site-packages/data_overlap.py | 1 + .../data/whl_with_data1/data_data_file.txt | 1 + .../headers/data_overlap.h | 1 + .../headers/overlap/both.h | 1 + .../headers/overlap/header1.h | 1 + .../headers/whl_with_data1/header_file.h | 1 + .../platlib/whl_with_data1/platlib_file.txt | 1 + .../purelib/data_overlap.py | 1 + .../purelib/whl_with_data1/__init__.py | 0 .../purelib/whl_with_data1/data_file.txt | 1 + .../scripts/data_overlap.sh | 1 + .../scripts/overlap/both.sh | 1 + .../scripts/overlap/script1.sh | 1 + .../scripts/whl_script.sh | 1 + .../whl_with_data1-1.0.dist-info/METADATA | 3 + .../whl_with_data1-1.0.dist-info/RECORD | 18 ++++ .../whl_with_data1-1.0.dist-info/WHEEL | 1 + tests/repos/whl_with_data2/BUILD.bazel | 1 + .../data/overlap/both.txt | 1 + .../data/overlap/data2.txt | 1 + .../data/whl_with_data2/data_data_file.txt | 1 + .../headers/overlap/both.h | 1 + .../headers/overlap/header2.h | 1 + .../headers/whl_with_data2/header_file.h | 1 + .../platlib/whl_with_data2/platlib_file.txt | 1 + .../purelib/whl_with_data2/__init__.py | 0 .../purelib/whl_with_data2/data_file.txt | 1 + .../scripts/overlap/both.sh | 1 + .../scripts/overlap/script2.sh | 1 + .../scripts/whl_script.sh | 1 + .../whl_with_data2-1.0.dist-info/METADATA | 3 + .../whl_with_data2-1.0.dist-info/RECORD | 12 +++ .../whl_with_data2-1.0.dist-info/WHEEL | 1 + tests/venv_site_packages_libs/BUILD.bazel | 2 + tests/venv_site_packages_libs/bin.py | 93 +++++++++++++++---- 51 files changed, 318 insertions(+), 53 deletions(-) create mode 100644 tests/repos/whl_with_data1/BUILD.bazel create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/bin/data_overlap.sh create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/include/data_overlap.h create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/overlap/both.txt create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/overlap/data1.txt create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/site-packages/data_overlap.py create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/data_overlap.h create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/overlap/both.h create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/overlap/header1.h create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/whl_with_data1/header_file.h create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/purelib/data_overlap.py create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/purelib/whl_with_data1/__init__.py create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/purelib/whl_with_data1/data_file.txt create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/data_overlap.sh create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/overlap/both.sh create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/overlap/script1.sh create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/whl_script.sh create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/METADATA create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD create mode 100644 tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/WHEEL create mode 100644 tests/repos/whl_with_data2/BUILD.bazel create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/overlap/both.txt create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/overlap/data2.txt create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/whl_with_data2/data_data_file.txt create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/overlap/both.h create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/overlap/header2.h create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/whl_with_data2/header_file.h create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/platlib/whl_with_data2/platlib_file.txt create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/purelib/whl_with_data2/__init__.py create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/purelib/whl_with_data2/data_file.txt create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/overlap/both.sh create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/overlap/script2.sh create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/whl_script.sh create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/METADATA create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD create mode 100644 tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/WHEEL diff --git a/CHANGELOG.md b/CHANGELOG.md index 225a1c1c00..7c19d1176e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,8 @@ END_UNRELEASED_TEMPLATE ### Changed * (gazelle) WORKSPACE's bazel-gazelle dependency bumped from 0.36.0 to 0.47.0. The go version was also bumped from 1.21.13 to 1.22.9. +* (pypi) The data files of a wheel (bin, includes, etc) are now always included + as a library's data dependencies. {#v0-0-0-fixed} ### Fixed @@ -74,6 +76,8 @@ END_UNRELEASED_TEMPLATE adding `config_setting` labels to all registered toolchains. * (windows) Full venv support for Windows is available. Set {obj}`--venvs_site_packages=yes` to enable. +* (test/binaries) When {obj}`--venv_site_packages=yes` is enabled, + wheel `data`, `bin`, and `include` files are populated into the venv. * (runfiles) Added a pathlib-compatible API: {obj}`Runfiles.root()` Fixes [#3296](https://github.com/bazel-contrib/rules_python/issues/3296). * (toolchains) `3.13.12`, `3.14.3` Python toolchain from [20260325] release. diff --git a/MODULE.bazel b/MODULE.bazel index 95d6b9e3a9..b5f67c204e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -246,6 +246,8 @@ use_repo( "somepkg_with_build_files", "whl_library_extras_direct_dep", "whl_with_build_files", + "whl_with_data1", + "whl_with_data2", ) dev_rules_python_config = use_extension( diff --git a/examples/pip_parse/pip_parse_test.py b/examples/pip_parse/pip_parse_test.py index 2fdd45477e..c532dff564 100644 --- a/examples/pip_parse/pip_parse_test.py +++ b/examples/pip_parse/pip_parse_test.py @@ -52,16 +52,16 @@ def test_data(self): self.assertIsNotNone(actual) actual = self._remove_leading_dirs(actual.split(" ")) - self.assertListEqual( - actual, - [ - "data/share/doc/packages/s3cmd/INSTALL.md", - "data/share/doc/packages/s3cmd/LICENSE", - "data/share/doc/packages/s3cmd/NEWS", - "data/share/doc/packages/s3cmd/README.md", - "data/share/man/man1/s3cmd.1", - ], - ) + expected = [ + "bin/s3cmd", + "data/share/doc/packages/s3cmd/INSTALL.md", + "data/share/doc/packages/s3cmd/LICENSE", + "data/share/doc/packages/s3cmd/NEWS", + "data/share/doc/packages/s3cmd/README.md", + "data/share/man/man1/s3cmd.1", + ] + + self.assertListEqual(actual, expected) def test_dist_info(self): actual = os.environ.get("WHEEL_DIST_INFO_CONTENTS") diff --git a/internal_dev_setup.bzl b/internal_dev_setup.bzl index c37c59a5da..0bbcd97748 100644 --- a/internal_dev_setup.bzl +++ b/internal_dev_setup.bzl @@ -26,6 +26,8 @@ load("//python:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS") load("//python/private:pythons_hub.bzl", "hub_repo") # buildifier: disable=bzl-visibility load("//python/private:runtime_env_repo.bzl", "runtime_env_repo") # buildifier: disable=bzl-visibility load("//python/private/pypi:deps.bzl", "pypi_deps") # buildifier: disable=bzl-visibility +load("//python/private/pypi:whl_library.bzl", "whl_library") # buildifier: disable=bzl-visibility +load("//tests/support/whl_from_dir:whl_from_dir_repo.bzl", "whl_from_dir_repo") # buildifier: disable=bzl-visibility def rules_python_internal_setup(): """Setup for development and testing of rules_python itself.""" @@ -59,3 +61,25 @@ def rules_python_internal_setup(): bazel_features_deps() rules_shell_dependencies() rules_shell_toolchains() + + whl_from_dir_repo( + name = "whl_with_data1_whl", + root = "//tests/repos/whl_with_data1:BUILD.bazel", + output = "whl_with_data1-1.0-any-none-any.whl", + ) + whl_library( + name = "whl_with_data1", + whl_file = "@whl_with_data1_whl//:whl_with_data1-1.0-any-none-any.whl", + requirement = "whl-with-data1", + ) + + whl_from_dir_repo( + name = "whl_with_data2_whl", + root = "//tests/repos/whl_with_data2:BUILD.bazel", + output = "whl_with_data2-1.0-any-none-any.whl", + ) + whl_library( + name = "whl_with_data2", + whl_file = "@whl_with_data2_whl//:whl_with_data2-1.0-any-none-any.whl", + requirement = "whl-with-data2", + ) diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index 7060d50b26..fc0ac51451 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -229,7 +229,7 @@ string_flag( ) config_setting( - name = "is_venvs_site_packages", + name = "_is_venvs_site_packages_yes", flag_values = { ":venvs_site_packages": VenvsSitePackages.YES, }, diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index fbdd5711b1..11b020e59f 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -91,6 +91,28 @@ def _internal_dev_deps_impl(mctx): enable_implicit_namespace_pkgs = False, ) + whl_from_dir_repo( + name = "whl_with_data1_whl", + root = "//tests/repos/whl_with_data1:BUILD.bazel", + output = "whl_with_data1-1.0-any-none-any.whl", + ) + whl_library( + name = "whl_with_data1", + whl_file = "@whl_with_data1_whl//:whl_with_data1-1.0-any-none-any.whl", + requirement = "whl-with-data1", + ) + + whl_from_dir_repo( + name = "whl_with_data2_whl", + root = "//tests/repos/whl_with_data2:BUILD.bazel", + output = "whl_with_data2-1.0-any-none-any.whl", + ) + whl_library( + name = "whl_with_data2", + whl_file = "@whl_with_data2_whl//:whl_with_data2-1.0-any-none-any.whl", + requirement = "whl-with-data2", + ) + _whl_library_from_dir( name = "whl_library_extras_direct_dep", root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel", diff --git a/python/private/py_executable.bzl b/python/private/py_executable.bzl index 375030ce91..6197c0c789 100644 --- a/python/private/py_executable.bzl +++ b/python/private/py_executable.bzl @@ -575,9 +575,13 @@ def _create_venv(ctx, output_prefix, imports, runtime_details, add_runfiles_root computed_substitutions = computed_subs, ) + # See https://docs.python.org/3/library/sysconfig.html#posix-prefix + # for how schemes map under the venv. venv_dir_map = { - VenvSymlinkKind.BIN: venv_details.bin_dir, + VenvSymlinkKind.BIN: "{}/{}".format(venv_ctx_rel_root, venv_details.bin_dir), VenvSymlinkKind.LIB: site_packages, + VenvSymlinkKind.INCLUDE: "{}/{}".format(venv_ctx_rel_root, venv_details.include_dir), + VenvSymlinkKind.DATA: venv_ctx_rel_root, } venv_app_files = create_venv_app_files( ctx, @@ -659,7 +663,7 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa recreate_venv_at_runtime = False - bin_dir = "{}/bin".format(venv_ctx_rel_root) + venv_bin_ctx_rel_path = "{}/bin".format(venv_ctx_rel_root) if create_full_venv: # Some wrappers around the interpreter (e.g. pyenv) use the program # name to decide what to do, so preserve the name. @@ -671,7 +675,7 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa # When the venv symlinks are disabled, the $venv/bin/python3 file isn't # needed or used at runtime. However, the zip code uses the interpreter # File object to figure out some paths. - interpreter = ctx.actions.declare_file("{}/{}".format(bin_dir, py_exe_basename)) + interpreter = ctx.actions.declare_file("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename)) ctx.actions.write(interpreter, "actual:{}".format(interpreter_actual_path)) elif runtime.interpreter: @@ -679,7 +683,7 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa # declare_symlink() is required to ensure that the resulting file # in runfiles is always a symlink. An RBE implementation, for example, # may choose to write what symlink() points to instead. - interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename)) + interpreter = ctx.actions.declare_symlink("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename)) interpreter_runfiles.add(interpreter) rel_path = relative_path( @@ -690,7 +694,7 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa ) ctx.actions.symlink(output = interpreter, target_path = rel_path) else: - interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename)) + interpreter = ctx.actions.declare_symlink("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename)) interpreter_runfiles.add(interpreter) ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path) else: @@ -715,7 +719,8 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa interpreter = interpreter, pyvenv_cfg = pyvenv_cfg, site_packages = site_packages, - bin_dir = bin_dir, + bin_dir = "bin", + include_dir = "include", recreate_venv_at_runtime = recreate_venv_at_runtime, interpreter_runfiles = interpreter_runfiles.build(ctx), interpreter_symlinks = depset(), @@ -777,7 +782,8 @@ def _create_venv_windows(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_ interpreter = interpreter, pyvenv_cfg = None, site_packages = site_packages, - bin_dir = venv_bin_ctx_rel_path, + bin_dir = venv_bin_rel_path, + include_dir = "Include", recreate_venv_at_runtime = True, interpreter_runfiles = interpreter_runfiles.build(ctx), interpreter_symlinks = interpreter_symlinks.build(), @@ -789,6 +795,7 @@ def _venv_details( pyvenv_cfg, site_packages, bin_dir, + include_dir, recreate_venv_at_runtime, interpreter_runfiles, interpreter_symlinks): @@ -801,8 +808,10 @@ def _venv_details( pyvenv_cfg = pyvenv_cfg, # str; venv-relative path to the site-packages directory site_packages = site_packages, - # str; ctx-relative path to the venv's bin directory. + # str; venv-relative path to the venv's bin directory. bin_dir = bin_dir, + # str; venv-relative-path to the venv's include directory. + include_dir = include_dir, # bool; True if the venv needs to be recreated at runtime (because the # build-time construction isn't sufficient). False if the build-time # constructed venv is sufficient. diff --git a/python/private/py_info.bzl b/python/private/py_info.bzl index 8868b9d3b4..551e0bb48c 100644 --- a/python/private/py_info.bzl +++ b/python/private/py_info.bzl @@ -37,6 +37,12 @@ def _VenvSymlinkKind_typedef(): Indicates to create paths under the venv's include directory. ::: + + :::{field} DATA + :type: object + + Indicates to create paths under the venv's data directory. + ::: """ # buildifier: disable=name-conventions @@ -45,6 +51,7 @@ VenvSymlinkKind = struct( BIN = "BIN", LIB = "LIB", INCLUDE = "INCLUDE", + DATA = "DATA", ) def _VenvSymlinkEntry_init(**kwargs): diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index dde5f815eb..4ed66cdddc 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -42,6 +42,8 @@ _BAZEL_REPO_FILE_GLOBS = [ "WORKSPACE.bazel", ] +_IS_VENV_SITE_PACKAGES_YES = Label("//python/config_settings:_is_venvs_site_packages_yes") + def whl_library_targets_from_requires( *, name, @@ -191,7 +193,7 @@ def whl_library_targets( include = ["site-packages/*.dist-info/**"], ), DATA_LABEL: dict( - include = ["data/**"], + include = ["data/**", "bin/**", "include/**"], ), } @@ -351,7 +353,7 @@ def whl_library_targets( if not enable_implicit_namespace_pkgs: generated_namespace_package_files = select({ - Label("//python/config_settings:is_venvs_site_packages"): [], + _IS_VENV_SITE_PACKAGES_YES: [], "//conditions:default": rules.create_inits( srcs = srcs + data + pyi_srcs, ignored_dirnames = [], # If you need to ignore certain folders, you can patch rules_python here to do so. @@ -361,6 +363,10 @@ def whl_library_targets( namespace_package_files += generated_namespace_package_files srcs = srcs + generated_namespace_package_files + # This is done after create_inits() is called so that the data scheme + # files don't have such files created in their directories. + data = data + [DATA_LABEL] + rules.py_library( name = py_library_label, srcs = srcs, diff --git a/python/private/venv_runfiles.bzl b/python/private/venv_runfiles.bzl index 6daf0d4e5c..a94f29f71c 100644 --- a/python/private/venv_runfiles.bzl +++ b/python/private/venv_runfiles.bzl @@ -69,10 +69,16 @@ def create_venv_app_files(ctx, deps, venv_dir_map): ctx.label.package, ) + seen_bin_venv_paths = {} + for kind, kind_map in link_map.items(): base = venv_dir_map[kind] for venv_path, link_to in kind_map.items(): bin_venv_path = paths.join(base, venv_path) + if bin_venv_path in seen_bin_venv_paths: + continue + seen_bin_venv_paths[bin_venv_path] = True + if is_file(link_to): # use paths.join to handle ctx.label.package = "" # runfile_prefix should be prepended as we use runfiles.root_symlinks @@ -80,6 +86,18 @@ def create_venv_app_files(ctx, deps, venv_dir_map): symlink_from = paths.join(runfile_prefix, ctx.label.package, bin_venv_path) runfiles_symlinks[symlink_from] = link_to + + # On Windows, we need to explicitly create the symlink in the venv + # because the bootstrap script won't otherwise know about it. + if is_windows: + rf_path = paths.join(ctx_rf_path, bin_venv_path) + _, _, venv_path = bin_venv_path.partition(".venv/") + explicit_symlinks.append(ExplicitSymlink( + runfiles_path = rf_path, + venv_path = venv_path, + link_to_path = runfiles_root_path(ctx, link_to.short_path), + files = depset([link_to]), + )) elif not is_windows: venv_link = ctx.actions.declare_symlink(bin_venv_path) venv_link_rf_path = runfiles_root_path(ctx, venv_link.short_path) @@ -275,10 +293,16 @@ def _get_file_venv_path(ctx, f, site_packages_root): Returns: A tuple `(venv_path, rf_root_path)` if the file is under - `site_packages_root`, otherwise `(None, None)`. + `site_packages_root` or data/, bin/, include/ otherwise `(None, None)`. """ rf_root_path = runfiles_root_path(ctx, f.short_path) _, _, repo_rel_path = rf_root_path.partition("/") + + # Check for wheel data/bin/include folders first + for prefix in ["data/", "bin/", "include/"]: + if repo_rel_path.startswith(prefix): + return (repo_rel_path, rf_root_path) + head, found_sp_root, venv_path = repo_rel_path.partition(site_packages_root) if head or not found_sp_root: # If head is set, then the path didn't start with site_packages_root @@ -324,6 +348,25 @@ def get_venv_symlinks( all_files = sorted(files, key = lambda f: f.short_path) + cannot_be_linked_directly = {} + for dirname in [ + # The venv directories that bin, include, and data get put into are + # shared across wheels, so we cannot link them directly + "bin", + "include", + "data", + # The data scheme is overlaid on the venv root, so the files under it + # could, in theory, get installed into e.g. bin/ or similar. Explicitly + # mark them as non-directly linkable to avoid issues. + "data/bin", + "data/include", + "data/lib", + "data/Scripts", + "data/Include", + "data/Lib", + ]: + cannot_be_linked_directly[dirname] = True + # dict[str venv-relative dirname, bool is_namespace_package] namespace_package_dirs = { ns: True @@ -331,10 +374,10 @@ def get_venv_symlinks( } # venv paths that cannot be directly linked. Dict acting as set. - cannot_be_linked_directly = { + cannot_be_linked_directly.update({ dirname: True for dirname in namespace_package_dirs.keys() - } + }) for f in namespace_package_files: venv_path, _ = _get_file_venv_path(ctx, f, site_packages_root) if venv_path == None: @@ -452,19 +495,36 @@ def get_venv_symlinks( # Finally, for each group, we create the VenvSymlinkEntry objects for venv_path, files in optimized_groups.items(): + if venv_path.startswith("data/"): + out_venv_path = venv_path[len("data/"):] + kind = VenvSymlinkKind.DATA + prefix = "" + elif venv_path.startswith("include/"): + out_venv_path = venv_path[len("include/"):] + kind = VenvSymlinkKind.INCLUDE + prefix = "" + elif venv_path.startswith("bin/"): + out_venv_path = venv_path[len("bin/"):] + kind = VenvSymlinkKind.BIN + prefix = "" + else: + out_venv_path = venv_path + kind = VenvSymlinkKind.LIB + prefix = site_packages_root + link_to_path = ( _get_label_runfiles_repo(ctx, files[0].owner) + "/" + - site_packages_root + + prefix + venv_path ) venv_symlinks[venv_path] = VenvSymlinkEntry( - kind = VenvSymlinkKind.LIB, + kind = kind, link_to_path = link_to_path, link_to_file = None, package = package, version = version_str, - venv_path = venv_path, + venv_path = out_venv_path, files = depset(files), ) diff --git a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl index 28765770f4..60e1f3f3dd 100644 --- a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl +++ b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl @@ -50,7 +50,7 @@ def _test_filegroups(env): }, { "name": "data", - "srcs": ["data/**"], + "srcs": ["data/**", "bin/**", "include/**"], "visibility": ["//visibility:public"], }, { @@ -216,11 +216,11 @@ def _test_whl_and_library_deps_from_requires(env): env.expect.that_dict(py_library_call).contains_exactly({ "name": "pkg", "srcs": ["site-packages/foo/SRCS.py"] + select({ - Label("//python/config_settings:is_venvs_site_packages"): [], + Label("//python/config_settings:_is_venvs_site_packages_yes"): [], "//conditions:default": ["_create_inits_target"], }), "pyi_srcs": ["site-packages/foo/PYI.pyi"], - "data": ["site-packages/foo/DATA.txt"], + "data": ["site-packages/foo/DATA.txt", "data"], "imports": ["site-packages"], "deps": ["@pypi//bar:pkg"] + select({ ":is_include_bar_baz_true": ["@pypi//bar_baz:pkg"], @@ -230,7 +230,7 @@ def _test_whl_and_library_deps_from_requires(env): "visibility": ["//visibility:public"], "experimental_venvs_site_packages": Label("//python/config_settings:venvs_site_packages"), "namespace_package_files": [] + select({ - Label("//python/config_settings:is_venvs_site_packages"): [], + Label("//python/config_settings:_is_venvs_site_packages_yes"): [], "//conditions:default": ["_create_inits_target"], }), }) # buildifier: @unsorted-dict-items @@ -332,11 +332,11 @@ def _test_whl_and_library_deps(env): env.expect.that_dict(py_library_calls[0]).contains_exactly({ "name": "pkg", "srcs": ["site-packages/foo/SRCS.py"] + select({ - Label("//python/config_settings:is_venvs_site_packages"): [], + Label("//python/config_settings:_is_venvs_site_packages_yes"): [], "//conditions:default": ["_create_inits_target"], }), "pyi_srcs": ["site-packages/foo/PYI.pyi"], - "data": ["site-packages/foo/DATA.txt"], + "data": ["site-packages/foo/DATA.txt", "data"], "imports": ["site-packages"], "deps": [ "@pypi_bar_baz//:pkg", @@ -357,7 +357,7 @@ def _test_whl_and_library_deps(env): "visibility": ["//visibility:public"], "experimental_venvs_site_packages": Label("//python/config_settings:venvs_site_packages"), "namespace_package_files": [] + select({ - Label("//python/config_settings:is_venvs_site_packages"): [], + Label("//python/config_settings:_is_venvs_site_packages_yes"): [], "//conditions:default": ["_create_inits_target"], }), }) # buildifier: @unsorted-dict-items @@ -414,11 +414,11 @@ def _test_group(env): ).contains_exactly({ "name": "_pkg", "srcs": ["site-packages/foo/srcs.py"] + select({ - Label("//python/config_settings:is_venvs_site_packages"): [], + Label("//python/config_settings:_is_venvs_site_packages_yes"): [], "//conditions:default": ["_create_inits_target"], }), "pyi_srcs": ["site-packages/foo/pyi.pyi"], - "data": ["site-packages/foo/data.txt"], + "data": ["site-packages/foo/data.txt", "data"], "imports": ["site-packages"], "deps": ["@pypi_bar_baz//:pkg"] + select({ "@platforms//os:linux": ["@pypi_box//:pkg"], @@ -429,7 +429,7 @@ def _test_group(env): "visibility": ["@pypi__config//_groups:__pkg__"], "experimental_venvs_site_packages": Label("//python/config_settings:venvs_site_packages"), "namespace_package_files": [] + select({ - Label("//python/config_settings:is_venvs_site_packages"): [], + Label("//python/config_settings:_is_venvs_site_packages_yes"): [], "//conditions:default": ["_create_inits_target"], }), }) # buildifier: @unsorted-dict-items diff --git a/tests/repos/whl_with_data1/BUILD.bazel b/tests/repos/whl_with_data1/BUILD.bazel new file mode 100644 index 0000000000..af49d1ebbf --- /dev/null +++ b/tests/repos/whl_with_data1/BUILD.bazel @@ -0,0 +1 @@ +exports_files(glob(["*"])) diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/bin/data_overlap.sh b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/bin/data_overlap.sh new file mode 100644 index 0000000000..b47ce4e9f3 --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/bin/data_overlap.sh @@ -0,0 +1 @@ +echo data_bin diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/include/data_overlap.h b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/include/data_overlap.h new file mode 100644 index 0000000000..299c39d0a7 --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/include/data_overlap.h @@ -0,0 +1 @@ +/* data_include */ diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/overlap/both.txt b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/overlap/both.txt new file mode 100644 index 0000000000..771c76ed7b --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/overlap/both.txt @@ -0,0 +1 @@ +both1 diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/overlap/data1.txt b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/overlap/data1.txt new file mode 100644 index 0000000000..d760283f59 --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/overlap/data1.txt @@ -0,0 +1 @@ +data1 diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/site-packages/data_overlap.py b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/site-packages/data_overlap.py new file mode 100644 index 0000000000..d3ee4d8a3f --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/site-packages/data_overlap.py @@ -0,0 +1 @@ +# data_site_packages diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt new file mode 100644 index 0000000000..39ec676600 --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt @@ -0,0 +1 @@ +from .data/data diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/data_overlap.h b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/data_overlap.h new file mode 100644 index 0000000000..ffd49d0cee --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/data_overlap.h @@ -0,0 +1 @@ +/* headers */ diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/overlap/both.h b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/overlap/both.h new file mode 100644 index 0000000000..49f33a8c6e --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/overlap/both.h @@ -0,0 +1 @@ +both diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/overlap/header1.h b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/overlap/header1.h new file mode 100644 index 0000000000..412e9ed7df --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/overlap/header1.h @@ -0,0 +1 @@ +header1 diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/whl_with_data1/header_file.h b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/whl_with_data1/header_file.h new file mode 100644 index 0000000000..59c9bf78c2 --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/headers/whl_with_data1/header_file.h @@ -0,0 +1 @@ +from .data/headers diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt new file mode 100644 index 0000000000..b27295614f --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt @@ -0,0 +1 @@ +from .data/platlib diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/purelib/data_overlap.py b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/purelib/data_overlap.py new file mode 100644 index 0000000000..f82e46670f --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/purelib/data_overlap.py @@ -0,0 +1 @@ +# purelib diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/purelib/whl_with_data1/__init__.py b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/purelib/whl_with_data1/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/purelib/whl_with_data1/data_file.txt b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/purelib/whl_with_data1/data_file.txt new file mode 100644 index 0000000000..e547fe48ed --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/purelib/whl_with_data1/data_file.txt @@ -0,0 +1 @@ +from .data diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/data_overlap.sh b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/data_overlap.sh new file mode 100644 index 0000000000..d6eb28dc3d --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/data_overlap.sh @@ -0,0 +1 @@ +echo scripts diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/overlap/both.sh b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/overlap/both.sh new file mode 100644 index 0000000000..49f33a8c6e --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/overlap/both.sh @@ -0,0 +1 @@ +both diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/overlap/script1.sh b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/overlap/script1.sh new file mode 100644 index 0000000000..4d68a2e3e0 --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/overlap/script1.sh @@ -0,0 +1 @@ +script1 diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/whl_script.sh b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/whl_script.sh new file mode 100644 index 0000000000..1a2485251c --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.data/scripts/whl_script.sh @@ -0,0 +1 @@ +#!/bin/sh diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/METADATA b/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/METADATA new file mode 100644 index 0000000000..f403970d7a --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/METADATA @@ -0,0 +1,3 @@ +Metadata-Version: 2.1 +Name: whl-with-data1 +Version: 1.0 diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD b/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD new file mode 100644 index 0000000000..a39e9ed7ad --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/RECORD @@ -0,0 +1,18 @@ +whl_with_data1-1.0.data/platlib/whl_with_data1/platlib_file.txt,sha256=123,123 +whl_with_data1-1.0.data/scripts/whl_script.sh,sha256=123,123 +whl_with_data1-1.0.data/headers/whl_with_data1/header_file.h,sha256=123,123 +whl_with_data1-1.0.data/purelib/whl_with_data1/data_file.txt,sha256=123,123 +whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt,sha256=123,123 +whl_with_data1-1.0.data/data/whl_with_data1/data_data_file.txt,sha256=123,123 +whl_with_data1-1.0.data/data/overlap/both.txt,sha256=123,123 +whl_with_data1-1.0.data/data/overlap/data1.txt,sha256=123,123 +whl_with_data1-1.0.data/scripts/overlap/both.sh,sha256=123,123 +whl_with_data1-1.0.data/scripts/overlap/script1.sh,sha256=123,123 +whl_with_data1-1.0.data/headers/overlap/both.h,sha256=123,123 +whl_with_data1-1.0.data/headers/overlap/header1.h,sha256=123,123 +whl_with_data1-1.0.data/scripts/data_overlap.sh,sha256=123,123 +whl_with_data1-1.0.data/data/bin/data_overlap.sh,sha256=123,123 +whl_with_data1-1.0.data/headers/data_overlap.h,sha256=123,123 +whl_with_data1-1.0.data/data/include/data_overlap.h,sha256=123,123 +whl_with_data1-1.0.data/purelib/data_overlap.py,sha256=123,123 +whl_with_data1-1.0.data/data/site-packages/data_overlap.py,sha256=123,123 diff --git a/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/WHEEL b/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/WHEEL new file mode 100644 index 0000000000..a64521a1cc --- /dev/null +++ b/tests/repos/whl_with_data1/whl_with_data1-1.0.dist-info/WHEEL @@ -0,0 +1 @@ +Wheel-Version: 1.0 diff --git a/tests/repos/whl_with_data2/BUILD.bazel b/tests/repos/whl_with_data2/BUILD.bazel new file mode 100644 index 0000000000..af49d1ebbf --- /dev/null +++ b/tests/repos/whl_with_data2/BUILD.bazel @@ -0,0 +1 @@ +exports_files(glob(["*"])) diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/overlap/both.txt b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/overlap/both.txt new file mode 100644 index 0000000000..1a8aa8b533 --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/overlap/both.txt @@ -0,0 +1 @@ +both2 diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/overlap/data2.txt b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/overlap/data2.txt new file mode 100644 index 0000000000..98d81a2ec6 --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/overlap/data2.txt @@ -0,0 +1 @@ +data2 diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/whl_with_data2/data_data_file.txt b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/whl_with_data2/data_data_file.txt new file mode 100644 index 0000000000..39ec676600 --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/data/whl_with_data2/data_data_file.txt @@ -0,0 +1 @@ +from .data/data diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/overlap/both.h b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/overlap/both.h new file mode 100644 index 0000000000..49f33a8c6e --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/overlap/both.h @@ -0,0 +1 @@ +both diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/overlap/header2.h b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/overlap/header2.h new file mode 100644 index 0000000000..da0a719745 --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/overlap/header2.h @@ -0,0 +1 @@ +header2 diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/whl_with_data2/header_file.h b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/whl_with_data2/header_file.h new file mode 100644 index 0000000000..59c9bf78c2 --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/headers/whl_with_data2/header_file.h @@ -0,0 +1 @@ +from .data/headers diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/platlib/whl_with_data2/platlib_file.txt b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/platlib/whl_with_data2/platlib_file.txt new file mode 100644 index 0000000000..b27295614f --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/platlib/whl_with_data2/platlib_file.txt @@ -0,0 +1 @@ +from .data/platlib diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/purelib/whl_with_data2/__init__.py b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/purelib/whl_with_data2/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/purelib/whl_with_data2/data_file.txt b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/purelib/whl_with_data2/data_file.txt new file mode 100644 index 0000000000..e547fe48ed --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/purelib/whl_with_data2/data_file.txt @@ -0,0 +1 @@ +from .data diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/overlap/both.sh b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/overlap/both.sh new file mode 100644 index 0000000000..49f33a8c6e --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/overlap/both.sh @@ -0,0 +1 @@ +both diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/overlap/script2.sh b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/overlap/script2.sh new file mode 100644 index 0000000000..026ed8b62a --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/overlap/script2.sh @@ -0,0 +1 @@ +script2 diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/whl_script.sh b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/whl_script.sh new file mode 100644 index 0000000000..1a2485251c --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.data/scripts/whl_script.sh @@ -0,0 +1 @@ +#!/bin/sh diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/METADATA b/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/METADATA new file mode 100644 index 0000000000..c762d184fb --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/METADATA @@ -0,0 +1,3 @@ +Metadata-Version: 2.1 +Name: whl-with-data2 +Version: 1.0 diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD b/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD new file mode 100644 index 0000000000..5eeb915ba7 --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/RECORD @@ -0,0 +1,12 @@ +whl_with_data2-1.0.data/platlib/whl_with_data2/platlib_file.txt,sha256=123,123 +whl_with_data2-1.0.data/scripts/whl_script.sh,sha256=123,123 +whl_with_data2-1.0.data/headers/whl_with_data2/header_file.h,sha256=123,123 +whl_with_data2-1.0.data/purelib/whl_with_data2/data_file.txt,sha256=123,123 +whl_with_data2-1.0.data/data/whl_with_data2/data_data_file.txt,sha256=123,123 +whl_with_data2-1.0.data/data/whl_with_data2/data_data_file.txt,sha256=123,123 +whl_with_data2-1.0.data/data/overlap/both.txt,sha256=123,123 +whl_with_data2-1.0.data/data/overlap/data2.txt,sha256=123,123 +whl_with_data2-1.0.data/scripts/overlap/both.sh,sha256=123,123 +whl_with_data2-1.0.data/scripts/overlap/script2.sh,sha256=123,123 +whl_with_data2-1.0.data/headers/overlap/both.h,sha256=123,123 +whl_with_data2-1.0.data/headers/overlap/header2.h,sha256=123,123 diff --git a/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/WHEEL b/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/WHEEL new file mode 100644 index 0000000000..a64521a1cc --- /dev/null +++ b/tests/repos/whl_with_data2/whl_with_data2-1.0.dist-info/WHEEL @@ -0,0 +1 @@ +Wheel-Version: 1.0 diff --git a/tests/venv_site_packages_libs/BUILD.bazel b/tests/venv_site_packages_libs/BUILD.bazel index 56f0eb0909..c99426b375 100644 --- a/tests/venv_site_packages_libs/BUILD.bazel +++ b/tests/venv_site_packages_libs/BUILD.bazel @@ -45,6 +45,8 @@ py_reconfig_test( "@other//nspkg_gamma", "@other//nspkg_single", "@other//with_external_data", + "@whl_with_data1//:pkg", + "@whl_with_data2//:pkg", ], ) diff --git a/tests/venv_site_packages_libs/bin.py b/tests/venv_site_packages_libs/bin.py index 439a964906..368251e75b 100644 --- a/tests/venv_site_packages_libs/bin.py +++ b/tests/venv_site_packages_libs/bin.py @@ -1,5 +1,7 @@ import importlib +import os import sys +import sysconfig import unittest from pathlib import Path @@ -9,7 +11,23 @@ def setUp(self): super().setUp() if sys.prefix == sys.base_prefix: raise AssertionError("Not running under a venv") - self.venv = sys.prefix + self.venv = Path(sys.prefix) + self.site_packages = Path(sysconfig.get_paths()["purelib"]) + + is_windows = sys.platform == "win32" + if is_windows: + self.bin_dir_name = Path("Scripts") + self.include_dir_name = Path("Include") + else: + self.bin_dir_name = Path("bin") + self.include_dir_name = Path("include") + + def assert_venv_path_exists(self, rel_path): + path = self.venv / rel_path + self.assertTrue( + path.exists(), + f"Expected {path} to exist. {path.parent.name} contents: {list(path.parent.iterdir()) if path.parent.exists() else 'N/A'}", + ) def assert_imported_from_venv(self, module_name): module = importlib.import_module(module_name) @@ -20,7 +38,7 @@ def assert_imported_from_venv(self, module_name): + f"__file__ set, but got None. {module=}", ) self.assertTrue( - module.__file__.startswith(self.venv), + module.__file__.startswith(str(self.venv)), f"\n{module_name} was imported, but not from the venv.\n" + f" venv: {self.venv}\n" + f"module file: {module.__file__}\n" @@ -48,12 +66,8 @@ def test_imported_from_venv(self): def test_data_is_included(self): self.assert_imported_from_venv("simple") module = importlib.import_module("simple") - module_path = Path(module.__file__) - - site_packages = module_path.parent.parent - # Ensure that packages from simple v1 are not present - files = [p.name for p in site_packages.glob("*")] + files = [p.name for p in self.site_packages.glob("*")] self.assertIn("simple_v1_extras", files) def test_override_pkg(self): @@ -67,30 +81,77 @@ def test_override_pkg(self): def test_dirs_from_replaced_package_are_not_present(self): self.assert_imported_from_venv("simple") module = importlib.import_module("simple") - module_path = Path(module.__file__) - - site_packages = module_path.parent.parent - dist_info_dirs = [p.name for p in site_packages.glob("*.dist-info")] + dist_info_dirs = [p.name for p in self.site_packages.glob("simple*.dist-info")] self.assertEqual( ["simple-1.0.0.dist-info"], dist_info_dirs, ) # Ensure that packages from simple v1 are not present - files = [p.name for p in site_packages.glob("*")] + files = [p.name for p in self.site_packages.glob("*")] self.assertNotIn("simple.libs", files) def test_data_from_another_pkg_is_included_via_copy_file(self): self.assert_imported_from_venv("simple") module = importlib.import_module("simple") - module_path = Path(module.__file__) - - site_packages = module_path.parent.parent # Ensure that packages from simple v1 are not present - d = site_packages / "external_data" + d = self.site_packages / "external_data" files = [p.name for p in d.glob("*")] self.assertIn("another_module_data.txt", files) + def test_whl_with_data1_included(self): + module = self.assert_imported_from_venv("whl_with_data1") + site_packages_rel = self.site_packages.relative_to(self.venv) + # purelib + self.assert_venv_path_exists(site_packages_rel / "whl_with_data1/data_file.txt") + + # platlib + self.assert_venv_path_exists( + site_packages_rel / "whl_with_data1/platlib_file.txt" + ) + + venv_root = self.venv + + # data + self.assert_venv_path_exists("whl_with_data1/data_data_file.txt") + + # scripts + self.assert_venv_path_exists(self.bin_dir_name / "whl_script.sh") + + # headers + self.assert_venv_path_exists( + self.include_dir_name / "whl_with_data1/header_file.h" + ) + + def test_whl_with_data2_included(self): + module = self.assert_imported_from_venv("whl_with_data2") + + site_packages_rel = self.site_packages.relative_to(self.venv) + self.assert_venv_path_exists(site_packages_rel / "whl_with_data2/data_file.txt") + + self.assert_venv_path_exists(self.bin_dir_name / "whl_script.sh") + + # Ensure that `data` files are unpacked in `venv/root/` + # and then linked as `venv/whl_with_data1/data_data_file.txt`. + self.assert_venv_path_exists("whl_with_data2/data_data_file.txt") + + self.assert_venv_path_exists( + self.include_dir_name / "whl_with_data2/header_file.h" + ) + + def test_whl_with_data_overlap(self): + self.assert_venv_path_exists("overlap/both.txt") + self.assert_venv_path_exists("overlap/data1.txt") + self.assert_venv_path_exists("overlap/data2.txt") + + self.assert_venv_path_exists(self.bin_dir_name / "overlap/both.sh") + self.assert_venv_path_exists(self.bin_dir_name / "overlap/script1.sh") + self.assert_venv_path_exists(self.bin_dir_name / "overlap/script2.sh") + + self.assert_venv_path_exists(self.include_dir_name / "overlap/both.h") + self.assert_venv_path_exists(self.include_dir_name / "overlap/header1.h") + self.assert_venv_path_exists(self.include_dir_name / "overlap/header2.h") + if __name__ == "__main__": unittest.main()