Skip to content

Commit b4e6496

Browse files
rickeylevrickeylev
committed
basic impl
ref https://packaging.python.org/en/latest/specifications/binary-distribution-format/#installing-a-wheel-distribution-1-0-py32-none-any-whl for generic data scheme: these are usually installed to the venv root, but that seems ill-advised. So we install into a data sub-directory of the venv.
1 parent 547521e commit b4e6496

20 files changed

Lines changed: 118 additions & 23 deletions

File tree

‎MODULE.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,7 @@ use_repo(
246246
"somepkg_with_build_files",
247247
"whl_library_extras_direct_dep",
248248
"whl_with_build_files",
249+
"whl_with_data",
249250
)
250251

251252
dev_rules_python_config = use_extension(

‎python/config_settings/BUILD.bazel‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ string_flag(
229229
)
230230

231231
config_setting(
232-
name = "is_venvs_site_packages",
232+
name = "_is_venvs_site_packages_yes",
233233
flag_values = {
234234
":venvs_site_packages": VenvsSitePackages.YES,
235235
},

‎python/private/internal_dev_deps.bzl‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,17 @@ def _internal_dev_deps_impl(mctx):
9191
enable_implicit_namespace_pkgs = False,
9292
)
9393

94+
whl_from_dir_repo(
95+
name = "whl_with_data_whl",
96+
root = "//tests/repos/whl_with_data:BUILD.bazel",
97+
output = "whl_with_data-1.0-any-none-any.whl",
98+
)
99+
whl_library(
100+
name = "whl_with_data",
101+
whl_file = "@whl_with_data_whl//:whl_with_data-1.0-any-none-any.whl",
102+
requirement = "whl-with-data",
103+
)
104+
94105
_whl_library_from_dir(
95106
name = "whl_library_extras_direct_dep",
96107
root = "//tests/pypi/whl_library/testdata/pkg:BUILD.bazel",

‎python/private/py_executable.bzl‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -576,8 +576,10 @@ def _create_venv(ctx, output_prefix, imports, runtime_details, add_runfiles_root
576576
)
577577

578578
venv_dir_map = {
579-
VenvSymlinkKind.BIN: venv_details.bin_dir,
579+
VenvSymlinkKind.BIN: "{}/{}".format(venv_ctx_rel_root, venv_details.bin_dir),
580580
VenvSymlinkKind.LIB: site_packages,
581+
VenvSymlinkKind.INCLUDE: "{}/{}".format(venv_ctx_rel_root, venv_details.include_dir),
582+
VenvSymlinkKind.DATA: "{}/data".format(venv_ctx_rel_root),
581583
}
582584
venv_app_files = create_venv_app_files(
583585
ctx,
@@ -659,7 +661,7 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
659661

660662
recreate_venv_at_runtime = False
661663

662-
bin_dir = "{}/bin".format(venv_ctx_rel_root)
664+
venv_bin_ctx_rel_path = "{}/bin".format(venv_ctx_rel_root)
663665
if create_full_venv:
664666
# Some wrappers around the interpreter (e.g. pyenv) use the program
665667
# name to decide what to do, so preserve the name.
@@ -671,15 +673,15 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
671673
# When the venv symlinks are disabled, the $venv/bin/python3 file isn't
672674
# needed or used at runtime. However, the zip code uses the interpreter
673675
# File object to figure out some paths.
674-
interpreter = ctx.actions.declare_file("{}/{}".format(bin_dir, py_exe_basename))
676+
interpreter = ctx.actions.declare_file("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename))
675677
ctx.actions.write(interpreter, "actual:{}".format(interpreter_actual_path))
676678

677679
elif runtime.interpreter:
678680
# Even though ctx.actions.symlink() is used, using
679681
# declare_symlink() is required to ensure that the resulting file
680682
# in runfiles is always a symlink. An RBE implementation, for example,
681683
# may choose to write what symlink() points to instead.
682-
interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename))
684+
interpreter = ctx.actions.declare_symlink("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename))
683685
interpreter_runfiles.add(interpreter)
684686

685687
rel_path = relative_path(
@@ -690,7 +692,7 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
690692
)
691693
ctx.actions.symlink(output = interpreter, target_path = rel_path)
692694
else:
693-
interpreter = ctx.actions.declare_symlink("{}/{}".format(bin_dir, py_exe_basename))
695+
interpreter = ctx.actions.declare_symlink("{}/{}".format(venv_bin_ctx_rel_path, py_exe_basename))
694696
interpreter_runfiles.add(interpreter)
695697
ctx.actions.symlink(output = interpreter, target_path = runtime.interpreter_path)
696698
else:
@@ -715,7 +717,8 @@ def _create_venv_unixy(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_pa
715717
interpreter = interpreter,
716718
pyvenv_cfg = pyvenv_cfg,
717719
site_packages = site_packages,
718-
bin_dir = bin_dir,
720+
bin_dir = "bin",
721+
include_dir = "include",
719722
recreate_venv_at_runtime = recreate_venv_at_runtime,
720723
interpreter_runfiles = interpreter_runfiles.build(ctx),
721724
interpreter_symlinks = depset(),
@@ -777,7 +780,8 @@ def _create_venv_windows(ctx, *, venv_ctx_rel_root, runtime, interpreter_actual_
777780
interpreter = interpreter,
778781
pyvenv_cfg = None,
779782
site_packages = site_packages,
780-
bin_dir = venv_bin_ctx_rel_path,
783+
bin_dir = venv_bin_rel_path,
784+
include_dir = "Include",
781785
recreate_venv_at_runtime = True,
782786
interpreter_runfiles = interpreter_runfiles.build(ctx),
783787
interpreter_symlinks = interpreter_symlinks.build(),
@@ -789,6 +793,7 @@ def _venv_details(
789793
pyvenv_cfg,
790794
site_packages,
791795
bin_dir,
796+
include_dir,
792797
recreate_venv_at_runtime,
793798
interpreter_runfiles,
794799
interpreter_symlinks):
@@ -801,8 +806,10 @@ def _venv_details(
801806
pyvenv_cfg = pyvenv_cfg,
802807
# str; venv-relative path to the site-packages directory
803808
site_packages = site_packages,
804-
# str; ctx-relative path to the venv's bin directory.
809+
# str; venv-relative path to the venv's bin directory.
805810
bin_dir = bin_dir,
811+
# str; venv-relative-path to the venv's include directory.
812+
include_dir = include_dir,
806813
# bool; True if the venv needs to be recreated at runtime (because the
807814
# build-time construction isn't sufficient). False if the build-time
808815
# constructed venv is sufficient.

‎python/private/py_info.bzl‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ VenvSymlinkKind = struct(
4545
BIN = "BIN",
4646
LIB = "LIB",
4747
INCLUDE = "INCLUDE",
48+
DATA = "DATA",
4849
)
4950

5051
def _VenvSymlinkEntry_init(**kwargs):

‎python/private/pypi/whl_library_targets.bzl‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ _BAZEL_REPO_FILE_GLOBS = [
4343
"WORKSPACE.bazel",
4444
]
4545

46+
_IS_VENV_SITE_PACKAGES_YES = Label("//python/config_settings:_is_venvs_site_packages_yes")
47+
4648
def whl_library_targets_from_requires(
4749
*,
4850
name,
@@ -194,8 +196,10 @@ def whl_library_targets(
194196
DIST_INFO_LABEL: dict(
195197
include = ["site-packages/*.dist-info/**"],
196198
),
199+
200+
## TO CHECK: should bin/ and include/ be part of the data target?
197201
DATA_LABEL: dict(
198-
include = ["data/**"],
202+
include = ["data/**", "bin/**", "include/**"],
199203
),
200204
}
201205

@@ -356,11 +360,15 @@ def whl_library_targets(
356360
if item not in _data_exclude:
357361
_data_exclude.append(item)
358362

359-
data = data + native.glob(
363+
data = data + [":" + DATA_LABEL] + native.glob(
360364
["site-packages/**/*"],
361365
exclude = _data_exclude,
362366
allow_empty = True,
363367
)
368+
data = data + select({
369+
_IS_VENV_SITE_PACKAGES_YES: [DATA_LABEL],
370+
"//conditions:default": [],
371+
})
364372

365373
pyi_srcs = native.glob(
366374
["site-packages/**/*.pyi"],
@@ -369,7 +377,7 @@ def whl_library_targets(
369377

370378
if not enable_implicit_namespace_pkgs:
371379
generated_namespace_package_files = select({
372-
Label("//python/config_settings:is_venvs_site_packages"): [],
380+
_IS_VENV_SITE_PACKAGES_YES: [],
373381
"//conditions:default": rules.create_inits(
374382
srcs = srcs + data + pyi_srcs,
375383
ignored_dirnames = [], # If you need to ignore certain folders, you can patch rules_python here to do so.

‎python/private/venv_runfiles.bzl‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -275,10 +275,16 @@ def _get_file_venv_path(ctx, f, site_packages_root):
275275
276276
Returns:
277277
A tuple `(venv_path, rf_root_path)` if the file is under
278-
`site_packages_root`, otherwise `(None, None)`.
278+
`site_packages_root` or data/, bin/, include/ otherwise `(None, None)`.
279279
"""
280280
rf_root_path = runfiles_root_path(ctx, f.short_path)
281281
_, _, repo_rel_path = rf_root_path.partition("/")
282+
283+
# Check for wheel data/bin/include folders first
284+
for prefix in ["data/", "bin/", "include/"]:
285+
if repo_rel_path.startswith(prefix):
286+
return (repo_rel_path, rf_root_path)
287+
282288
head, found_sp_root, venv_path = repo_rel_path.partition(site_packages_root)
283289
if head or not found_sp_root:
284290
# If head is set, then the path didn't start with site_packages_root
@@ -452,19 +458,36 @@ def get_venv_symlinks(
452458

453459
# Finally, for each group, we create the VenvSymlinkEntry objects
454460
for venv_path, files in optimized_groups.items():
461+
if venv_path.startswith("data/"):
462+
out_venv_path = venv_path[len("data/"):]
463+
kind = VenvSymlinkKind.DATA
464+
prefix = ""
465+
elif venv_path.startswith("include/"):
466+
out_venv_path = venv_path[len("include/"):]
467+
kind = VenvSymlinkKind.INCLUDE
468+
prefix = ""
469+
elif venv_path.startswith("bin/"):
470+
out_venv_path = venv_path[len("bin/"):]
471+
kind = VenvSymlinkKind.BIN
472+
prefix = ""
473+
else:
474+
out_venv_path = venv_path
475+
kind = VenvSymlinkKind.LIB
476+
prefix = site_packages_root
477+
455478
link_to_path = (
456479
_get_label_runfiles_repo(ctx, files[0].owner) +
457480
"/" +
458-
site_packages_root +
481+
prefix +
459482
venv_path
460483
)
461484
venv_symlinks[venv_path] = VenvSymlinkEntry(
462-
kind = VenvSymlinkKind.LIB,
485+
kind = kind,
463486
link_to_path = link_to_path,
464487
link_to_file = None,
465488
package = package,
466489
version = version_str,
467-
venv_path = venv_path,
490+
venv_path = out_venv_path,
468491
files = depset(files),
469492
)
470493

‎tests/pypi/whl_library_targets/whl_library_targets_tests.bzl‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ def _test_whl_and_library_deps_from_requires(env):
245245
env.expect.that_dict(py_library_call).contains_exactly({
246246
"name": "pkg",
247247
"srcs": ["site-packages/foo/SRCS.py"] + select({
248-
Label("//python/config_settings:is_venvs_site_packages"): [],
248+
Label("//python/config_settings:_is_venvs_site_packages_yes"): [],
249249
"//conditions:default": ["_create_inits_target"],
250250
}),
251251
"pyi_srcs": ["site-packages/foo/PYI.pyi"],
@@ -259,7 +259,7 @@ def _test_whl_and_library_deps_from_requires(env):
259259
"visibility": ["//visibility:public"],
260260
"experimental_venvs_site_packages": Label("//python/config_settings:venvs_site_packages"),
261261
"namespace_package_files": [] + select({
262-
Label("//python/config_settings:is_venvs_site_packages"): [],
262+
Label("//python/config_settings:_is_venvs_site_packages_yes"): [],
263263
"//conditions:default": ["_create_inits_target"],
264264
}),
265265
}) # buildifier: @unsorted-dict-items
@@ -361,7 +361,7 @@ def _test_whl_and_library_deps(env):
361361
env.expect.that_dict(py_library_calls[0]).contains_exactly({
362362
"name": "pkg",
363363
"srcs": ["site-packages/foo/SRCS.py"] + select({
364-
Label("//python/config_settings:is_venvs_site_packages"): [],
364+
Label("//python/config_settings:_is_venvs_site_packages_yes"): [],
365365
"//conditions:default": ["_create_inits_target"],
366366
}),
367367
"pyi_srcs": ["site-packages/foo/PYI.pyi"],
@@ -386,7 +386,7 @@ def _test_whl_and_library_deps(env):
386386
"visibility": ["//visibility:public"],
387387
"experimental_venvs_site_packages": Label("//python/config_settings:venvs_site_packages"),
388388
"namespace_package_files": [] + select({
389-
Label("//python/config_settings:is_venvs_site_packages"): [],
389+
Label("//python/config_settings:_is_venvs_site_packages_yes"): [],
390390
"//conditions:default": ["_create_inits_target"],
391391
}),
392392
}) # buildifier: @unsorted-dict-items
@@ -444,7 +444,7 @@ def _test_group(env):
444444
).contains_exactly({
445445
"name": "_pkg",
446446
"srcs": ["site-packages/foo/srcs.py"] + select({
447-
Label("//python/config_settings:is_venvs_site_packages"): [],
447+
Label("//python/config_settings:_is_venvs_site_packages_yes"): [],
448448
"//conditions:default": ["_create_inits_target"],
449449
}),
450450
"pyi_srcs": ["site-packages/foo/pyi.pyi"],
@@ -459,7 +459,7 @@ def _test_group(env):
459459
"visibility": ["@pypi__config//_groups:__pkg__"],
460460
"experimental_venvs_site_packages": Label("//python/config_settings:venvs_site_packages"),
461461
"namespace_package_files": [] + select({
462-
Label("//python/config_settings:is_venvs_site_packages"): [],
462+
Label("//python/config_settings:_is_venvs_site_packages_yes"): [],
463463
"//conditions:default": ["_create_inits_target"],
464464
}),
465465
}) # buildifier: @unsorted-dict-items

‎tests/repos/whl_with_data/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
exports_files(glob(["*"]))

‎tests/repos/whl_with_data/whl_with_data-1.0.data/data/whl_with_data/data_data_file.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
from .data/data

0 commit comments

Comments
 (0)