diff --git a/py/private/py_venv/BUILD.bazel b/py/private/py_venv/BUILD.bazel index 917a4b51e..6cefb34f2 100644 --- a/py/private/py_venv/BUILD.bazel +++ b/py/private/py_venv/BUILD.bazel @@ -1,13 +1,14 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") -package(default_visibility = ["//py:__subpackages__"]) +package(default_visibility = ["//visibility:private"]) exports_files([ "templates/_virtualenv.py", "templates/link.py", "templates/venv.tmpl.sh", "templates/venv_activate.tmpl.sh", + "templates/console_script.tmpl.sh", ]) bool_flag( @@ -20,42 +21,30 @@ config_setting( flag_values = { ":debug_venv": "True", }, - # This needs to be set public for global use since we select() on it from - # macros which analyze visibility wrt the expanding package. - # - # TODO: Eliminate this in favor of a proper bool flag or something. visibility = [ "//visibility:public", ], ) bzl_library( - name = "types", - srcs = ["types.bzl"], - visibility = [ - "//py/private:__subpackages__", - "//uv/private:__subpackages__", - ], + name = "toolchains_resolver", + srcs = ["toolchains_resolver.bzl"], deps = [ + "@bazel_lib//lib:paths", ], ) bzl_library( - name = "venv", - srcs = ["venv.bzl"], + name = "assemble_venv", + srcs = ["assemble_venv.bzl"], deps = [ ":toolchains_resolver", ":virtuals_resolvers", "//py/private:py_library", + "//py/private/toolchain:types", ], ) -bzl_library( - name = "toolchains_resolver", - srcs = ["toolchains_resolver.bzl"], - deps = ["@bazel_lib//lib:paths"], -) - bzl_library( name = "py_venv_exec", srcs = ["py_venv_exec.bzl"], @@ -75,9 +64,9 @@ bzl_library( name = "py_venv", srcs = ["py_venv.bzl"], deps = [ + ":assemble_venv", ":py_venv_exec", ":types", - ":venv", "//py/private:py_library", "//py/private:py_semantics", "//py/private:transitions", @@ -90,9 +79,7 @@ bzl_library( bzl_library( name = "defs", srcs = ["defs.bzl"], - visibility = [ - "//py:__subpackages__", - ], + visibility = ["//py:__subpackages__"], deps = [ ":py_venv", ":py_venv_exec", @@ -100,6 +87,11 @@ bzl_library( ], ) +bzl_library( + name = "types", + srcs = ["types.bzl"], +) + bzl_library( name = "virtuals_resolvers", srcs = ["virtuals_resolvers.bzl"], diff --git a/py/private/py_venv/assemble_venv.bzl b/py/private/py_venv/assemble_venv.bzl new file mode 100644 index 000000000..81711b502 --- /dev/null +++ b/py/private/py_venv/assemble_venv.bzl @@ -0,0 +1,360 @@ +"""Build-time assembly of a Python virtualenv via ctx.actions.symlink + write. + +This module is the single place in rules_py that declares the files making +up a Python venv. Both ``py_binary`` / ``py_test`` (each with its own +internal venv, unless ``expose_venv = True`` routes them to a sibling +py_venv) and the standalone ``py_venv`` rule call ``assemble_venv`` to +keep their layouts bit-identical. + +The venv shape mirrors what CPython's ``python -m venv`` + pip install +produces, so downstream tools (IDEs, ``$VIRTUAL_ENV``-aware shells, +distutils, etc.) treat it as a real venv: + + / + pyvenv.cfg + bin/ + python symlink -> py_toolchain.python + python3 symlink -> python + python3.. symlink -> python + activate bash/zsh activation script + one per wheel-declared entry point + lib/python./site-packages/ + .pth first-party + fallback .pth + _virtualenv.py distutils-compat shim + _virtualenv.pth loads the shim at site init + symlink to a wheel's subdir + / merged PEP 420 namespace + -.dist-info symlink when the wheel needs no + whole-wheel fallback + +The whole tree is declared at analysis time as individual +``ctx.actions.declare_file`` / ``ctx.actions.declare_symlink`` outputs so +Bazel's action cache treats each piece independently (no tree-artifact ++ remote-exec materialisation surprises). +""" + +load("//py/private:py_library.bzl", _py_library = "py_library_utils") +load("//py/private/toolchain:types.bzl", "EXEC_TOOLS_TOOLCHAIN") +load(":toolchains_resolver.bzl", "resolve_venv_toolchain") +load(":virtuals_resolvers.bzl", "compute_wheel_plan", "enforce_collision_policy") + +def _dict_to_exports(env): + """Render an env dict as ``export KEY="VALUE"`` shell lines.""" + return ["export %s=\"%s\"" % (k, v) for (k, v) in env.items()] + +def _make_pth_formatter(fully_covered, known_layout, escape, venv_escape): + """Build a ``map_each`` closure for ``.pth`` import formatting. + + Three branches per import path: + + * Fully covered wheel → ``None`` (omitted from the ``.pth``). + * Layout-unknown wheel whose path ends in ``site-packages`` → + ``site.addsitedir`` with a ``sys.prefix``-relative path that + survives RBE sandbox layouts. + * Everything else → plain relative escape + import path. + """ + + def _format_imp(imp): + if imp in fully_covered: + return None + if imp.endswith("site-packages") and imp not in known_layout: + return ("import os, sys, site; " + + "site.addsitedir(os.path.normpath(os.path.join(" + + "sys.prefix, \"{}\", \"{}\")))").format(venv_escape, imp) + return "{}/{}".format(escape, imp) + + return _format_imp + +def _declare_toplevel_symlinks(ctx, top_level_to_site_pkgs, tc, declared): + """Declare per-top-level site-packages symlinks into owning wheels. + + Each symlink escapes from ``site-packages/`` up to the runfiles root, + then descends into the owning wheel's ``site_packages_rfpath``. + ``/``-joined top-levels (merged namespace packages, e.g. + ``jaraco/functools``) get one extra ``../`` per segment. A per-wheel + prefix cache avoids recomputing the escape path. + """ + prefix = tc.site_packages_rel + "/" + target_prefix_by_sp = {} + for tl, wheel_sp in top_level_to_site_pkgs.items(): + out = ctx.actions.declare_symlink(prefix + tl) + target_prefix = target_prefix_by_sp.get(wheel_sp) + if target_prefix == None: + target_prefix = tc.escape + "/" + wheel_sp + "/" + target_prefix_by_sp[wheel_sp] = target_prefix + target_path = target_prefix + tl + if "/" in tl: + target_path = "../" * tl.count("/") + target_path + ctx.actions.symlink(output = out, target_path = target_path) + declared.append(out) + +def _run_site_merges( + ctx, + merge_groups, + tree_by_sp, + tc, + site_merge_script_py, + package_collisions, + exec_runtime, + declared): + """Run ``PySiteMerge`` actions for physical package merges. + + Each group's subtree is copied from every contributing wheel into a + real directory inside site-packages — the layout a flat ``pip install`` + produces. The venv's own site-packages precedes per-wheel ``.pth`` + entries on ``sys.path``, so the merged copy is what Python binds the + regular package's ``__path__`` to; per-wheel originals are shadowed. + """ + for group in merge_groups: + merged_dir = ctx.actions.declare_directory( + "{}/{}".format(tc.site_packages_rel, group.root), + ) + arguments = ctx.actions.args() + arguments.add(site_merge_script_py) + arguments.add_all([merged_dir], expand_directories = False, before_each = "--into") + arguments.add("--collision-policy", package_collisions) + trees = [] + for sp in group.site_packages_list: + tree = tree_by_sp[sp] + trees.append(tree) + arguments.add_all( + [tree], + expand_directories = False, + before_each = "--src", + format_each = "%s/lib/{}/site-packages/{}".format(tc.wheel_py_ver, group.root), + ) + ctx.actions.run( + mnemonic = "PySiteMerge", + executable = exec_runtime.interpreter, + toolchain = EXEC_TOOLS_TOOLCHAIN, + arguments = [arguments], + inputs = depset( + direct = [site_merge_script_py] + trees, + transitive = [exec_runtime.files], + ), + outputs = [merged_dir], + execution_requirements = {"supports-path-mapping": "1"}, + ) + declared.append(merged_dir) + +def _declare_pth_file(ctx, imports_depset, format_imp, tc, safe_name, declared): + """Generate the ``.pth`` file for first-party and fallback imports. + + Uses ``map_each`` with ``allow_closure`` so the imports depset is + consumed at execution time (param-file writing), not at analysis + time — avoiding an unnecessary ``.to_list()``. + """ + pth_lines = ctx.actions.args() + pth_lines.use_param_file("%s", use_always = True) + pth_lines.set_param_file_format("multiline") + pth_lines.add(tc.escape) + pth_lines.add_all(imports_depset, map_each = format_imp, allow_closure = True) + + out = ctx.actions.declare_file( + "{}/{}.pth".format(tc.site_packages_rel, safe_name), + ) + ctx.actions.write(output = out, content = pth_lines) + declared.append(out) + +def _declare_pyvenv_cfg( + ctx, + tc, + py_toolchain, + venv_name, + include_system, + include_user, + declared): + """Write ``pyvenv.cfg`` with version, home, and site-packages flags.""" + home_line = "home =\n" if tc.pyvenv_home == "" else "home = {}\n".format(tc.pyvenv_home) + out = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv_name)) + ctx.actions.write( + output = out, + content = home_line + ( + "implementation = CPython\n" + + "version_info = {major}.{minor}.{micro}\n" + + "include-system-site-packages = {include_system}\n" + + "aspect-include-user-site-packages = {include_user}\n" + + "relocatable = true\n" + ).format( + major = py_toolchain.interpreter_version_info.major, + minor = py_toolchain.interpreter_version_info.minor, + micro = py_toolchain.interpreter_version_info.micro, + include_system = str(include_system).lower(), + include_user = str(include_user).lower(), + ), + ) + declared.append(out) + +def _declare_python_bin(ctx, tc, venv_name, declared): + """Declare ``bin/python`` + versioned symlinks (``python3``, ``python3.X`` …). + + Returns the ``bin/python`` File for launcher consumption. + """ + bin_python = ctx.actions.declare_symlink("{}/bin/python".format(venv_name)) + ctx.actions.symlink(output = bin_python, target_path = tc.bin_python_target_path) + declared.append(bin_python) + + for name in tc.versioned_python_names: + sym = ctx.actions.declare_symlink("{}/bin/{}".format(venv_name, name)) + ctx.actions.symlink(output = sym, target_path = "python") + declared.append(sym) + + return bin_python + +def _declare_activate(ctx, default_env, venv_activate_tmpl, venv_name, declared): + """Expand the ``bin/activate`` template with env exports and unsets.""" + out = ctx.actions.declare_file("{}/bin/activate".format(venv_name)) + exports = "\n".join(_dict_to_exports(default_env)).strip() + unsets = "\n".join([" unset {}".format(k) for k in default_env.keys()]) + ctx.actions.expand_template( + template = venv_activate_tmpl, + output = out, + substitutions = {"{{ENVVARS}}": exports, "{{ENVVARS_UNSET}}": unsets}, + is_executable = True, + ) + declared.append(out) + +def _declare_virtualenv_shim(ctx, site_packages_rel, virtualenv_shim_py, declared): + """Materialise ``_virtualenv.py`` + ``_virtualenv.pth`` in site-packages. + + Uses ``expand_template`` with empty substitutions (verbatim copy) + so the shim is a real file, not a symlink into the source tree — + ``os.path.realpath`` resolves inside the venv for tar/OCI consumers. + """ + shim_py = ctx.actions.declare_file("{}/_virtualenv.py".format(site_packages_rel)) + ctx.actions.expand_template( + template = virtualenv_shim_py, + output = shim_py, + substitutions = {}, + ) + declared.append(shim_py) + + shim_pth = ctx.actions.declare_file("{}/_virtualenv.pth".format(site_packages_rel)) + ctx.actions.write(output = shim_pth, content = "import _virtualenv\n") + declared.append(shim_pth) + +def _declare_console_scripts(ctx, console_scripts_map, console_script_tmpl, venv_name, declared): + """Expand one shell wrapper per wheel-declared entry point.""" + for name, target in console_scripts_map.items(): + script = ctx.actions.declare_file("{}/bin/{}".format(venv_name, name)) + ctx.actions.expand_template( + template = console_script_tmpl, + output = script, + substitutions = { + "{{name}}": name, + "{{module}}": target.module, + "{{func}}": target.func, + }, + is_executable = True, + ) + declared.append(script) + +def assemble_venv( + ctx, + *, + safe_name, + py_toolchain, + imports_depset, + is_windows, + package_collisions, + include_system_site_packages, + include_user_site_packages, + default_env, + venv_activate_tmpl, + virtualenv_shim_py, + site_merge_script_py, + console_script_tmpl, + venv_name): + """Declare every file + symlink that makes up a venv for a target. + + Args: + ctx: The rule context. + safe_name: Directory-name-safe stem for the venv dir. + py_toolchain: Resolved Python toolchain struct from py_semantics. + imports_depset: Depset of import paths (from + ``py_library_utils.make_imports_depset``). + is_windows: Whether the venv targets Windows. + package_collisions: ``"error"`` / ``"warning"`` / ``"ignore"``. + include_system_site_packages: Value for pyvenv.cfg's key. + include_user_site_packages: Value for the Aspect extension key. + default_env: Env-var dict for the activate script. + venv_activate_tmpl: File — activate template. + virtualenv_shim_py: File — ``_virtualenv.py`` source. + site_merge_script_py: File — ``site_merge.py`` tool. + console_script_tmpl: File — console-script template. + venv_name: The venv dir basename (e.g. ``.``). + + Returns: + struct with ``bin_python`` (File) and ``all_files`` (list[File]). + """ + tc = resolve_venv_toolchain( + ctx, + py_toolchain = py_toolchain, + is_windows = is_windows, + venv_name = venv_name, + ) + + plan = compute_wheel_plan(ctx, _py_library.make_wheels_depset(ctx).to_list()) + enforce_collision_policy(plan.collisions, package_collisions) + + top_level_to_site_pkgs = plan.top_level_to_site_pkgs + fully_covered = plan.fully_covered + console_scripts_map = plan.console_scripts_map + merge_groups = plan.merge_groups + tree_by_sp = plan.tree_by_sp + known_layout = plan.known_layout + declared = [] + + _declare_toplevel_symlinks(ctx, top_level_to_site_pkgs, tc, declared) + + if merge_groups: + exec_toolchain = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN] + exec_runtime = exec_toolchain.exec_tools.exec_runtime if exec_toolchain else None + if exec_runtime == None: + fail(("{}: wheels {} all contribute to the regular package `{}` — merging it " + + "requires an exec-configuration Python interpreter, but no `{}` toolchain " + + "was registered.").format( + ctx.label, + merge_groups[0].site_packages_list, + merge_groups[0].root, + EXEC_TOOLS_TOOLCHAIN, + )) + _run_site_merges( + ctx, + merge_groups, + tree_by_sp, + tc, + site_merge_script_py, + package_collisions, + exec_runtime, + declared, + ) + + format_imp = _make_pth_formatter( + fully_covered, + known_layout, + tc.escape, + tc.venv_to_runfiles_escape, + ) + _declare_pth_file(ctx, imports_depset, format_imp, tc, safe_name, declared) + + _declare_pyvenv_cfg( + ctx, + tc, + py_toolchain, + venv_name, + include_system_site_packages, + include_user_site_packages, + declared, + ) + + bin_python = _declare_python_bin(ctx, tc, venv_name, declared) + + _declare_activate(ctx, default_env, venv_activate_tmpl, venv_name, declared) + _declare_virtualenv_shim(ctx, tc.site_packages_rel, virtualenv_shim_py, declared) + _declare_console_scripts(ctx, console_scripts_map, console_script_tmpl, venv_name, declared) + + return struct( + bin_python = bin_python, + all_files = declared, + ) diff --git a/py/private/py_venv/py_venv.bzl b/py/private/py_venv/py_venv.bzl index 52199eb1e..46e429bfc 100644 --- a/py/private/py_venv/py_venv.bzl +++ b/py/private/py_venv/py_venv.bzl @@ -1,28 +1,23 @@ -"""Implementation for the `py_venv` rule + `py_venv_link` and `py_binary_with_venv` macros. - -- `py_venv` — a rule that builds a Python virtualenv and produces an - executable that activates it and exec's `bin/python`. Emits - `VirtualenvInfo` consumed by `py_binary_with_venv` (the - `expose_venv = True` codepath). `bazel run :name` on a py_venv drops - into the hermetic interpreter with the venv activated — useful for - interactive Python sessions. - -- `py_binary_with_venv` — shared helper invoked by - `py_binary(expose_venv = True, ...)` / `py_test(expose_venv = True, ...)`. - Splits the call into a sibling `:.venv` `py_venv` + a - `py_binary` / `py_test` rule that consumes it via the internal - `venv` attribute. `bazel run :.venv` drops into the - interpreter. - -- `py_venv_link` — opt-in macro that emits a runnable target whose - `bazel run` links the target's complete runfiles tree into the - workspace and prints the nested `py_venv` path. Pair with - `py_binary(expose_venv = True)` to give your IDE a stable interpreter - path. +"""Implementation for the ``py_venv`` rule + ``py_venv_link`` and +``py_binary_with_venv`` macros. + +- ``py_venv`` — builds a Python virtualenv and produces an executable + that activates it and exec's ``bin/python``. Emits ``VirtualenvInfo`` + consumed by ``py_binary_with_venv`` (the ``expose_venv = True`` + codepath). ``bazel run :name`` drops into the hermetic interpreter + with the venv activated. + +- ``py_binary_with_venv`` — shared helper invoked by + ``py_binary(expose_venv = True, ...)`` / + ``py_test(expose_venv = True, ...)``. Splits the call into a sibling + ``:.venv`` ``py_venv`` + a launcher rule that consumes it. + +- ``py_venv_link`` — opt-in macro that emits a runnable target whose + ``bazel run`` links the runfiles tree into the workspace and prints + the venv path for IDE integration. Shared venv-assembly logic lives in -`//py/private/py_venv:venv.bzl::assemble_venv`. See that file's header for the -layout details. +``//py/private/py_venv:assemble_venv.bzl::assemble_venv``. """ load("@bazel_lib//lib:expand_make_vars.bzl", "expand_locations", "expand_variables") @@ -31,25 +26,85 @@ load("//py/private:py_library.bzl", _py_library = "py_library_utils") load("//py/private:py_semantics.bzl", _py_semantics = "semantics") load("//py/private:transitions.bzl", "python_transition") load("//py/private/toolchain:types.bzl", "EXEC_TOOLS_TOOLCHAIN", "PY_TOOLCHAIN") +load(":assemble_venv.bzl", "assemble_venv") load(":py_venv_exec.bzl", _py_venv_exec = "py_venv_exec") load(":types.bzl", "VirtualenvInfo", "venv_root") -load(":venv.bzl", "assemble_venv") + +_VENV_ONLY_ATTRS = [ + "resolutions", + "virtual_deps", + "package_collisions", + "include_system_site_packages", + "include_user_site_packages", + "python_version", + "dep_group", +] + +_SHARED_ATTRS = [ + "interpreter_options", + "env", + "env_inherit", +] def _interpreter_flags(ctx): - args = _py_semantics.interpreter_flags + ctx.attr.interpreter_options + """Build the interpreter flag list for the venv's own REPL. - # py_venv strips `-I` so the interpreter picks up PYTHONPATH and - # script dir — useful when users `bazel run` the venv for an - # interactive python session and want their shell's env to apply. - # The per-binary py_binary launcher keeps `-I` (see py_venv_exec.bzl). - args = [it for it in args if it not in ["-I"]] + ``-I`` is stripped from the **semantics base** so the interactive + session picks up PYTHONPATH and the script directory. User + ``interpreter_options`` are appended verbatim — an explicit ``-I`` + survives. + """ + base = [f for f in _py_semantics.interpreter_flags if f != "-I"] + return base + list(ctx.attr.interpreter_options) - return args +def _default_env(ctx): + """Build the Bazel-contextual env vars injected into ``activate``.""" + return { + "BAZEL_TARGET": str(ctx.label).lstrip("@"), + "BAZEL_WORKSPACE": ctx.workspace_name, + "BAZEL_TARGET_NAME": ctx.attr.name, + } -def _assemble_shared(ctx): - """Resolve the py toolchain, virtual deps, imports depset — then run - the shared venv-assembly helper. +def _expand_env_vars(ctx, env): + """Expand ``$(location)`` and make-vars in every value of *env*.""" + result = dict(env) + for k, v in result.items(): + result[k] = expand_variables( + ctx, + expand_locations(ctx, v, ctx.attr.data), + attribute_name = "env", + ) + return result + +def _expand_launcher(ctx, shared): + """Expand the venv launcher template (``venv.tmpl.sh``).""" + ctx.actions.expand_template( + template = ctx.file._run_tmpl, + output = ctx.outputs.executable, + substitutions = { + "{{BASH_RLOCATION_FN}}": BASH_RLOCATION_FUNCTION.strip(), + "{{INTERPRETER_FLAGS}}": " ".join(_interpreter_flags(ctx)), + "{{ARG_VENV_PYTHON}}": to_rlocation_path(ctx, shared.venv.bin_python), + "{{DEBUG}}": str(ctx.attr.debug).lower(), + }, + is_executable = True, + ) + +def _build_run_env(ctx, shared): + """Construct the ``RunEnvironmentInfo.environment`` dict. + + Fails if the user set ``VIRTUAL_ENV`` explicitly. The value is a + runfiles-relative rootpath; ``venv.tmpl.sh`` overrides it with the + absolute rlocation-resolved path at runtime. """ + if "VIRTUAL_ENV" in ctx.attr.env: + fail("py_venv {}: `VIRTUAL_ENV` is set by the rule and cannot be overridden via `env`.".format(ctx.label)) + env = _expand_env_vars(ctx, ctx.attr.env) + env["VIRTUAL_ENV"] = venv_root(shared.venv.bin_python) + return env + +def _assemble_shared(ctx): + """Resolve toolchain, virtual deps, imports — then run ``assemble_venv``.""" py_toolchain = _py_semantics.resolve_toolchain(ctx) virtual_resolution = _py_library.resolve_virtuals(ctx) imports_depset = _py_library.make_imports_depset( @@ -57,12 +112,6 @@ def _assemble_shared(ctx): extra_imports_depsets = virtual_resolution.imports, ) - default_env = { - "BAZEL_TARGET": str(ctx.label).lstrip("@"), - "BAZEL_WORKSPACE": ctx.workspace_name, - "BAZEL_TARGET_NAME": ctx.attr.name, - } - safe_name = ctx.attr.name.replace("/", "_") venv = assemble_venv( @@ -76,20 +125,18 @@ def _assemble_shared(ctx): package_collisions = ctx.attr.package_collisions, include_system_site_packages = ctx.attr.include_system_site_packages, include_user_site_packages = ctx.attr.include_user_site_packages, - default_env = default_env, + default_env = _default_env(ctx), venv_activate_tmpl = ctx.file._venv_activate_tmpl, virtualenv_shim_py = ctx.file._virtualenv_shim, site_merge_script_py = ctx.file._site_merge_script, + console_script_tmpl = ctx.file._console_script_tmpl, venv_name = ".{}".format(safe_name), ) srcs_depset = _py_library.make_srcs_depset(ctx) runfiles = _py_library.make_merged_runfiles( ctx, - extra_depsets = [ - py_toolchain.files, - srcs_depset, - ] + virtual_resolution.srcs + virtual_resolution.runfiles, + extra_depsets = [py_toolchain.files, srcs_depset] + virtual_resolution.srcs + virtual_resolution.runfiles, extra_runfiles = venv.all_files, extra_runfiles_depsets = [ ctx.attr._runfiles_lib[DefaultInfo].default_runfiles, @@ -105,20 +152,22 @@ def _assemble_shared(ctx): ) def _common_providers(ctx, shared, executable = None): - """Providers emitted by both the executable and lib variants.""" + """Providers emitted by both the executable and lib variants. + + Deliberately omits ``PyInfo``: a venv is a terminal artifact, not a + source of imports for downstream ``py_library`` consumers. + """ return [ DefaultInfo( files = depset([executable]) if executable != None else None, executable = executable, runfiles = shared.runfiles, ), - # Deliberately no PyInfo: a venv is a terminal artifact, not a source of imports. VirtualenvInfo( bin_python = shared.venv.bin_python, imports = shared.imports_depset, transitive_sources = shared.srcs_depset, ), - # `bazel coverage` finds this by walking the consumer's `venv` attr. coverage_common.instrumented_files_info( ctx, source_attributes = ["srcs"], @@ -128,49 +177,27 @@ def _common_providers(ctx, shared, executable = None): ] def _py_venv_rule_impl(ctx): - """A virtualenv target whose own executable activates the venv and - exec's the interpreter — a `bazel run :name`-able venv.""" - + """Executable venv target — ``bazel run :name`` drops into the REPL.""" shared = _assemble_shared(ctx) - - ctx.actions.expand_template( - template = ctx.file._run_tmpl, - output = ctx.outputs.executable, - substitutions = { - "{{BASH_RLOCATION_FN}}": BASH_RLOCATION_FUNCTION.strip(), - "{{INTERPRETER_FLAGS}}": " ".join(_interpreter_flags(ctx)), - "{{ARG_VENV_PYTHON}}": to_rlocation_path(ctx, shared.venv.bin_python), - "{{DEBUG}}": str(ctx.attr.debug).lower(), - }, - is_executable = True, - ) - - if "VIRTUAL_ENV" in ctx.attr.env: - fail("py_venv {}: `VIRTUAL_ENV` is set by the rule and cannot be overridden via `env`.".format(ctx.label)) - - passed_env = dict(ctx.attr.env) - for k, v in passed_env.items(): - passed_env[k] = expand_variables( - ctx, - expand_locations(ctx, v, ctx.attr.data), - attribute_name = "env", - ) - - # `VIRTUAL_ENV` as the venv root's rootpath. `venv.tmpl.sh` - # overrides with its own absolute value when invoked directly. - passed_env["VIRTUAL_ENV"] = venv_root(shared.venv.bin_python) - + _expand_launcher(ctx, shared) + env = _build_run_env(ctx, shared) return _common_providers(ctx, shared, executable = ctx.outputs.executable) + [ - # Read by the sibling `expose_venv = True` py_binary/py_test; - # the binary's own `env` wins on key conflicts (py_venv_exec.bzl). RunEnvironmentInfo( - environment = passed_env, + environment = env, inherited_environment = ctx.attr.env_inherit, ), ] -# Attrs read by both the executable and lib variants — venv assembly, -# package-collision detection, py_library inputs. +def _py_venv_lib_rule_impl(ctx): + """Non-executable venv variant — same providers, no launcher. + + Bazel rejects ``RunEnvironmentInfo`` on non-executable targets, so + ``py_venv_exec.bzl`` gates its read on ``if RunEnvironmentInfo in + venv``. + """ + shared = _assemble_shared(ctx) + return _common_providers(ctx, shared) + _lib_attrs = dict({ "dep_group": attr.string( default = "", @@ -210,34 +237,30 @@ does not reinsert a wheel. default = False, doc = """`pyvenv.cfg` feature flag for the `aspect-include-user-site-packages` extension key.""", ), - # Required for py_version attribute "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", ), "_runfiles_lib": attr.label( default = "@bazel_tools//tools/bash/runfiles", ), - # Read by py_semantics — freethreaded interpreters expect site-packages - # at `lib/python.t/site-packages/`, not the default `.../python./`. "_freethreaded_flag": attr.label( default = "//py/private/interpreter:freethreaded", ), - # Shared with py_binary via the venv-assembly helper. "_venv_activate_tmpl": attr.label( allow_single_file = True, default = "//py/private/py_venv:templates/venv_activate.tmpl.sh", ), "_virtualenv_shim": attr.label( + allow_single_file = True, + default = "templates/_virtualenv.py", + ), + "_console_script_tmpl": attr.label( allow_single_file = True, default = "//py/private/py_venv:templates/_virtualenv.py", ), "_windows_constraint": attr.label( default = "@platforms//os:windows", ), - # Tool for physically merging a regular package that spans wheels or - # collides as a top-level directory - # (e.g. azure-core + azure-core-tracing-opentelemetry both - # installing into `azure/core/tracing/`) — see assemble_venv. "_site_merge_script": attr.label( allow_single_file = True, default = "//py/tools/site_merge:site_merge.py", @@ -246,8 +269,6 @@ does not reinsert a wheel. _lib_attrs.update(**_py_library.attrs) -# Attrs only the executable variant reads — launcher template, REPL -# flags, env vars forwarded via RunEnvironmentInfo. _attrs = _lib_attrs | dict({ "interpreter_options": attr.string_list( doc = "Additional options to pass to the Python interpreter.", @@ -280,84 +301,43 @@ _py_venv = rule( attrs = _attrs, toolchains = [ PY_TOOLCHAIN, - # Optional: only consulted when a regular package needs a physical merge - # and assemble_venv needs an exec-config interpreter to run the - # site_merge action. Optional so venvs keep analyzing in setups - # that never registered rules_py's exec-tools toolchain. config_common.toolchain_type(EXEC_TOOLS_TOOLCHAIN, mandatory = False), ], executable = True, cfg = python_transition, ) -def _py_venv_lib_rule_impl(ctx): - """Non-executable variant of `_py_venv` — same providers but no - launcher and no RunEnvironmentInfo (Bazel rejects it on - non-executable targets; py_venv_exec.bzl gates its read on - `if RunEnvironmentInfo in venv`).""" - shared = _assemble_shared(ctx) - return _common_providers(ctx, shared) - -# Internal-only non-executable variant. Uses `_lib_attrs` — the -# launcher-only attrs (`debug`, `interpreter_options`, `_run_tmpl`, -# `env`, `env_inherit`) aren't part of its rule contract. _py_venv_lib = rule( implementation = _py_venv_lib_rule_impl, attrs = _lib_attrs, toolchains = [ PY_TOOLCHAIN, - # Same optional exec-tools dependency as `_py_venv`: assemble_venv - # needs it to run the site_merge action when a package needs a merge. config_common.toolchain_type(EXEC_TOOLS_TOOLCHAIN, mandatory = False), ], cfg = python_transition, ) -def _wrap_with_debug(rule): +def _wrap_with_debug(rule_fn): + """Wrap a rule so ``debug`` is driven by the ``:debug_venv_setting`` flag.""" + def helper(**kwargs): kwargs["debug"] = select({ Label(":debug_venv_setting"): True, "//conditions:default": False, }) - return rule(**kwargs) + return rule_fn(**kwargs) return helper py_venv = _wrap_with_debug(_py_venv) -# Attrs that the macro routes to the generated `py_venv`. Two flavors: -# venv-only (popped from kwargs, the launcher never sees them) and -# shared (copied into venv kwargs but kept in kwargs so they also reach -# the launcher rule). -_VENV_ONLY_ATTRS = [ - "resolutions", - "virtual_deps", - "package_collisions", - "include_system_site_packages", - "include_user_site_packages", - "python_version", - "dep_group", -] - -# Shared attrs are only forwarded when the venv is the executable -# variant (`expose_venv = True`). The lib variant doesn't read them, so -# leave them on the launcher kwargs only. -_SHARED_ATTRS = [ - # Launcher constructs `python main.py`; venv forwards them - # to its REPL so `bazel run :name.venv` matches the binary's flags. - "interpreter_options", - # Forwarded so `bazel run :name.venv` sees the same env as `bazel - # run :name`. The launcher reads venv's RunEnvironmentInfo too, so - # the duplicate set is harmless (same keys, same values). - "env", - "env_inherit", -] - def _split_kwargs_for_venv(kwargs, expose_venv): - """Build the kwargs dict to pass to the venv rule. `kwargs` is - mutated: venv-only attrs are popped; shared attrs are copied (left - in `kwargs` so they also reach the launcher) — but only for the - executable variant. + """Route attrs between the sibling venv and the launcher. + + ``_VENV_ONLY_ATTRS`` are popped from *kwargs* (the launcher never + sees them). ``_SHARED_ATTRS`` are copied into the venv kwargs only + when the venv is the executable variant (``expose_venv = True``); + the lib variant does not read them. """ venv_kwargs = {} for name in _VENV_ONLY_ATTRS: @@ -369,28 +349,40 @@ def _split_kwargs_for_venv(kwargs, expose_venv): venv_kwargs[name] = kwargs[name] return venv_kwargs -def py_binary_with_venv(py_rule, name, main, srcs = [], deps = [], data = [], imports = [], tags = None, testonly = None, visibility = None, isolated = True, expose_venv = None, expose_venv_link = False, **kwargs): - """Split `py_rule(name, ...)` into a sibling py_venv target + a - `py_rule` call routed at it via the internal `venv` rule - attribute. Called for every `py_binary` / `py_test` macro invocation. - - `expose_venv = True` emits a public `:{name}.venv` py_venv: - runnable (`bazel run :{name}.venv` drops into the hermetic - interpreter) and pairable with `py_venv_link` for IDE integration. - The venv inherits the binary's visibility. Default `None` (unset); - treated as `False` unless `expose_venv_link = True` promotes it. - - `expose_venv_link = True` additionally emits a public - `:{name}.venv_link` py_venv_link. `bazel run :{name}.venv_link` - materialises a workspace-local symlink to the target's runfiles tree and - prints the venv path below that link, suitable for pointing an IDE at. - Implies `expose_venv = True` — the link target needs a public venv to - point at. Passing `expose_venv = False, expose_venv_link = True` - explicitly is contradictory and fails. - - All venv-shaping attrs (`deps`, `imports`, `package_collisions`, - `include_*_site_packages`, `interpreter_options`) land on the - sibling venv. +def _venv_target_config(name, safe_name, expose_venv, visibility): + """Return ``(label, visibility, tags, rule_fn)`` for the sibling venv.""" + if expose_venv: + return "{}.venv".format(name), visibility, None, py_venv + return "_{}.venv".format(safe_name), ["//visibility:private"], ["manual"], _py_venv_lib + +def py_binary_with_venv( + py_rule, + name, + main, + srcs = [], + deps = [], + data = [], + imports = [], + tags = None, + testonly = None, + visibility = None, + isolated = True, + expose_venv = None, + expose_venv_link = False, + **kwargs): + """Split ``py_rule(name, ...)`` into a sibling py_venv + a launcher. + + Called for every ``py_binary`` / ``py_test`` macro invocation. + + ``expose_venv = True`` emits a public ``:{name}.venv`` py_venv: + runnable and pairable with ``py_venv_link`` for IDE integration. + ``expose_venv_link = True`` additionally emits a + ``:{name}.venv_link`` target; implies ``expose_venv = True``. + + ``data`` lands on both targets: the venv merges it into runfiles + (inherited by every consumer), while the launcher needs it directly + for ``$(location)`` expansion and the coverage walk. ``srcs`` is + similarly duplicated so label resolution works on both sides. """ if expose_venv_link: if expose_venv == False: @@ -405,19 +397,13 @@ def py_binary_with_venv(py_rule, name, main, srcs = [], deps = [], data = [], im venv_kwargs["imports"] = imports venv_kwargs["data"] = data - # Target names can contain `/` (Bazel allows it), but venv labels - # and the on-disk venv basename must be slash-free. safe_name = name.replace("/", "_") - if expose_venv: - venv_label = "{}.venv".format(name) - venv_visibility = visibility - venv_tags = None - venv_rule = py_venv - else: - venv_label = "_{}.venv".format(safe_name) - venv_visibility = ["//visibility:private"] - venv_tags = ["manual"] - venv_rule = _py_venv_lib + venv_label, venv_visibility, venv_tags, venv_rule = _venv_target_config( + name, + safe_name, + expose_venv, + visibility, + ) venv_rule( name = venv_label, @@ -440,11 +426,6 @@ def py_binary_with_venv(py_rule, name, main, srcs = [], deps = [], data = [], im name = name, main = main, srcs = srcs, - # `data` lands on both targets: the venv merges it into the - # runfiles every consumer inherits, while the launcher needs it - # directly so `args` / `env` `$(location :data_file)` expansion - # and the coverage walk can resolve the label (srcs are kept on - # the launcher for the same reason). data = data, tags = tags, testonly = testonly, @@ -455,26 +436,21 @@ def py_binary_with_venv(py_rule, name, main, srcs = [], deps = [], data = [], im ) def py_venv_link(name, venv, link_name = None, **kwargs): - """Emit a runnable target that materialises `venv` into the workspace. + """Emit a runnable target that materialises *venv* into the workspace. - `bazel run :` creates a symlink in `$BUILD_WORKING_DIRECTORY` - (typically the workspace root) that points at the target's complete - runfiles tree. The command prints `venv`'s nested path below that link; - preserving the runfiles directory layout keeps the venv's relative paths - valid for Python and IDEs. This requires directory-based runfiles; a - manifest alone cannot expose a runfiles tree. + ``bazel run :`` creates a symlink in + ``$BUILD_WORKING_DIRECTORY`` that points at the target's complete + runfiles tree. The command prints the venv's nested path below that + link, suitable for pointing an IDE at. Requires directory-based + runfiles; a manifest alone cannot expose a runfiles tree. Args: - name: Runnable target name. `bazel run :` materialises - the runfiles symlink. - venv: Label of a `py_venv` target to link. Typically the - `:.venv` target auto-emitted by - `py_binary(expose_venv = True, ...)`, or a standalone - `py_venv` shared across many binaries. - link_name: Workspace-relative basename for the created - runfiles symlink. Defaults to a safely-escaped version of the - target's package + venv name. - **kwargs: Forwarded to the underlying `py_binary`. + name: Runnable target name. + venv: Label of a ``py_venv`` target to link. + link_name: Workspace-relative basename for the runfiles symlink. + Defaults to a safely-escaped version of the target's package + + venv name. + **kwargs: Forwarded to the underlying ``py_venv_exec``. """ link_script = str(Label("//py/private/py_venv:templates/link.py")) _py_venv_exec( diff --git a/py/private/py_venv/py_venv_exec.bzl b/py/private/py_venv/py_venv_exec.bzl index be403cbe5..a649ef119 100644 --- a/py/private/py_venv/py_venv_exec.bzl +++ b/py/private/py_venv/py_venv_exec.bzl @@ -1,8 +1,8 @@ """Implementation for the py_venv_exec and py_venv_exec_test rules. -Both are thin launchers that consume a sibling `py_venv` (passed via the -internal `venv` attr) and exec its `bin/python`. The public -`py_binary` / `py_test` macros wrap them and route all venv-shaping +Both are thin launchers that consume a sibling ``py_venv`` (passed via +the internal ``venv`` attr) and exec its ``bin/python``. The public +``py_binary`` / ``py_test`` macros wrap them and route all venv-shaping attrs to the auto-generated sibling. """ @@ -13,39 +13,39 @@ load("//py/private:py_semantics.bzl", _py_semantics = "semantics") load("//py/private:transitions.bzl", "reset_python_flags_transition") load(":types.bzl", "VirtualenvInfo", "venv_root") -# Identifiers the launcher always sets to the analysing rule's contextual -# values. Excluded from `inherited_environment` so that a stray -# `env_inherit` entry can't let an outer shell shadow the contextual -# label at run time. _CONTEXTUAL_ENV_KEYS = ("BAZEL_TARGET", "BAZEL_WORKSPACE", "BAZEL_TARGET_NAME") -def _py_venv_exec_impl(ctx): - # The launcher itself doesn't need a python toolchain — it just - # exec's the sibling venv's `bin/python`, whose path was already - # resolved when the venv was analysed. Default interpreter flags - # come from a shared constant. - # - # The macro layer routes srcs / deps to the sibling py_venv (always - # set as `venv`) and passes an explicit `main =` to the rule. - # `main` is the only first-party file the rule contributes; - # everything else flows through the sibling venv. - if not ctx.attr.main: - fail("py_binary {}: main is required.".format(ctx.label)) +def _validate_main(ctx): + """Return the main module ``File``, failing if absent or not ``.py``.""" main = ctx.file.main + if not main: + fail("py_binary {}: main is required.".format(ctx.label)) if not main.basename.endswith(".py"): fail("main must end in '.py', got: " + main.basename) + return main - venv = ctx.attr.venv - vinfo = venv[VirtualenvInfo] +def _set_contextual_env(ctx, passed_env): + """Set Bazel contextual identifiers that always override user env.""" + passed_env["BAZEL_TARGET"] = str(ctx.label).lstrip("@") + passed_env["BAZEL_WORKSPACE"] = ctx.workspace_name + passed_env["BAZEL_TARGET_NAME"] = ctx.attr.name + +def _strip_contextual_from_inherited(inherited_env): + """Remove contextual keys so a stray ``env_inherit`` can't shadow them. - # Merge env vars: start from the venv's `env` (if any), then - # overlay the binary's own — binary wins on key conflicts. Same - # merge for inherited env-var names. Bazel-contextual identifiers - # (BAZEL_TARGET, etc.) overlay last and are stripped from - # `inherited_env` so a stray `env_inherit` entry can't let the - # caller's shell shadow the contextual label — per - # https://bazel.build/rules/lib/providers/RunEnvironmentInfo, an - # inherited value wins over `environment` when both are present. + Per RunEnvironmentInfo semantics, an inherited value wins over + ``environment`` when both are present — stripping here prevents + the caller's shell from overriding the contextual identifiers. + """ + return [n for n in inherited_env if n not in _CONTEXTUAL_ENV_KEYS] + +def _merge_environment(ctx, venv, vinfo): + """Merge env vars from the sibling venv, the binary, and contextual keys. + + Precedence (last wins): venv ``RunEnvironmentInfo`` -> ``VIRTUAL_ENV`` + -> binary ``env`` -> contextual keys. Returns + ``(passed_env, inherited_env)``. + """ passed_env = {} inherited_env = [] if RunEnvironmentInfo in venv: @@ -53,13 +53,9 @@ def _py_venv_exec_impl(ctx): passed_env = dict(venv_env.environment) inherited_env = list(venv_env.inherited_environment) - # Owned by the rule. The lib venv variant carries no `env` to guard, - # so guard here to match the executable variant's check. if "VIRTUAL_ENV" in ctx.attr.env: fail("py_binary/py_test {}: `VIRTUAL_ENV` is set by the rule and cannot be overridden via `env`.".format(ctx.label)) - # Set here so it's present even for the lib venv variant, which has - # no RunEnvironmentInfo to carry it. passed_env["VIRTUAL_ENV"] = venv_root(vinfo.bin_python) for k, v in ctx.attr.env.items(): passed_env[k] = expand_variables( @@ -67,27 +63,39 @@ def _py_venv_exec_impl(ctx): expand_locations(ctx, v, ctx.attr.data), attribute_name = "env", ) + + inherited_set = {n: True for n in inherited_env} for name in ctx.attr.env_inherit: - if name not in inherited_env: + if name not in inherited_set: inherited_env.append(name) - passed_env["BAZEL_TARGET"] = str(ctx.label).lstrip("@") - passed_env["BAZEL_WORKSPACE"] = ctx.workspace_name - passed_env["BAZEL_TARGET_NAME"] = ctx.attr.name - inherited_env = [n for n in inherited_env if n not in _CONTEXTUAL_ENV_KEYS] + inherited_set[name] = True + + _set_contextual_env(ctx, passed_env) + inherited_env = _strip_contextual_from_inherited(inherited_env) + + return passed_env, inherited_env - # When `isolated = False`, drop Python's `-I` flag so PYTHONPATH is - # honored and the script directory is auto-added to sys.path. - flags = list(_py_semantics.interpreter_flags) + ctx.attr.interpreter_options +def _interpreter_flags(ctx): + """Build the interpreter flag list. + + Base flags come from ``py_semantics``. When ``isolated = False``, + ``-I`` is stripped from the **base** flags so PYTHONPATH is honored. + User ``interpreter_options`` are always appended verbatim — a user + explicitly passing ``-I`` survives the isolated-stripping. + """ + base = list(_py_semantics.interpreter_flags) if not ctx.attr.isolated: - flags = [f for f in flags if f != "-I"] + base = [f for f in base if f != "-I"] + return base + list(ctx.attr.interpreter_options) + +def _build_launcher(ctx, vinfo, flags, main): + """Compile the native launcher stub via ``hermetic_launcher``. - # Native launcher via hermetic_launcher. Embedded argv: - # [0] venv's bin/python (runfiles-resolved) - # [1+] interpreter flags (literal, e.g. -I, -X importtime) - # [N] main module path (runfiles-resolved) - # The launcher runtime resolves transformed-arg positions through - # the Bazel runfiles manifest, then `execve`s the venv python. - executable_launcher = ctx.actions.declare_file(ctx.attr.name) + Embedded argv: venv's ``bin/python``, interpreter flags, then the + main module path. The launcher runtime resolves runfiles positions + and ``execve``'s the venv python. + """ + executable = ctx.actions.declare_file(ctx.attr.name) embedded_args, transformed_args = launcher.args_from_entrypoint(vinfo.bin_python) for flag in flags: embedded_args, transformed_args = launcher.append_embedded_arg( @@ -104,43 +112,48 @@ def _py_venv_exec_impl(ctx): ctx = ctx, embedded_args = embedded_args, transformed_args = transformed_args, - output_file = executable_launcher, + output_file = executable, ) + return executable - # Merge runfiles, supporting `py_venv_exec(main)` not being in the `py_venv` runfiles. - runfiles = ctx.runfiles( +def _build_runfiles(ctx, venv, main): + """Merge data files, the main module, and the venv's runfiles.""" + return ctx.runfiles( files = ctx.files.data + [main], ).merge_all( [target[DefaultInfo].default_runfiles for target in ctx.attr.data] + [venv[DefaultInfo].default_runfiles], ) - instrumented_files_info = coverage_common.instrumented_files_info( - ctx, - source_attributes = ["main"], - dependency_attributes = ["data", "venv"], - extensions = ["py"], - ) +def _py_venv_exec_impl(ctx): + main = _validate_main(ctx) + venv = ctx.attr.venv + vinfo = venv[VirtualenvInfo] + + passed_env, inherited_env = _merge_environment(ctx, venv, vinfo) + flags = _interpreter_flags(ctx) + executable = _build_launcher(ctx, vinfo, flags, main) + runfiles = _build_runfiles(ctx, venv, main) return [ DefaultInfo( - files = depset([executable_launcher, main]), - executable = executable_launcher, + files = depset([executable, main]), + executable = executable, runfiles = runfiles, ), PyInfo( - # Surface the venv's imports + transitive_sources through - # PyInfo so downstream consumers (e.g. py_pex_binary's - # `--sys-path=`) see the same sys.path / source closure the - # launcher will run with. `srcs` / `deps` live on the - # sibling venv, not on this rule. imports = vinfo.imports, transitive_sources = vinfo.transitive_sources, has_py2_only_sources = False, has_py3_only_sources = True, uses_shared_libraries = False, ), - instrumented_files_info, + coverage_common.instrumented_files_info( + ctx, + source_attributes = ["main"], + dependency_attributes = ["data", "venv"], + extensions = ["py"], + ), RunEnvironmentInfo( environment = passed_env, inherited_environment = inherited_env, @@ -158,20 +171,8 @@ _attrs = dict({ ), "main": attr.label( allow_single_file = True, - doc = """ -Script to execute with the Python interpreter. - -Must be a label pointing to a `.py` source file. -If such a label is provided, it will be honored. - -If no label is provided AND there is only one `srcs` file, that `srcs` file will be used. - -If there are more than one `srcs`, a file matching `{name}.py` is searched for. -This is for historical compatibility with the Bazel native `py_binary` and `rules_python`. -Relying on this behavior is STRONGLY discouraged, may produce warnings and may -be deprecated in the future. - -""", + mandatory = True, + doc = "Python source file to execute. The macro layer resolves this from `srcs` when not set explicitly.", ), "venv": attr.label( providers = [[VirtualenvInfo]], @@ -201,11 +202,6 @@ sibling venv has `include_user_site_packages = True` set. The deprecated `py_venv_binary` / `py_venv_test` aliases default this to False to match their historical permissive behaviour.""", ), - # `data` is the only py_library attr the launcher reads (env-var - # location expansion, runfiles merge, coverage walk). `srcs`, - # `deps`, `imports`, `resolutions`, and `virtual_deps` are routed - # to the sibling py_venv by the macro layer and have no role on - # the launcher rule. "data": attr.label_list( doc = """Runtime dependencies of the program. @@ -219,13 +215,12 @@ that must match the terminal's Python environment in `deps`. allow_files = True, cfg = reset_python_flags_transition, ), - # Forwarded to the sibling py_venv (which is where srcs actually - # feed sys.path). Carried on the launcher only so Bazel's `args` - # location-expansion (`args = ["$(location :foo.py)"]`) can resolve - # the label against the same files the user wrote on - # `py_binary` / `py_test`. "srcs": attr.label_list( - doc = "Python source files. Forwarded to the sibling py_venv.", + doc = """Python source files. Forwarded to the sibling py_venv where +they feed sys.path. Carried on the launcher so that Bazel's `args` +location-expansion (`args = ["$(location :foo.py)"]`) can resolve the +label. The files reach runfiles transitively through the venv's +default_runfiles, not from this attribute directly.""", allow_files = [".py"], ), "_allowlist_function_transition": attr.label( @@ -234,15 +229,15 @@ that must match the terminal's Python environment in `deps`. }) _test_attrs = dict({ - # Magic attribute to make coverage --combined_report flag work. - # There's no docs about this. - # See https://github.com/bazelbuild/bazel/blob/fde4b67009d377a3543a3dc8481147307bd37d36/tools/test/collect_coverage.sh#L186-L194 - # NB: rules_python ALSO includes this attribute on the py_binary rule, but we think that's a mistake. - # see https://github.com/aspect-build/rules_py/pull/520#pullrequestreview-2579076197 "_lcov_merger": attr.label( default = configuration_field(fragment = "coverage", name = "output_generator"), executable = True, cfg = "exec", + doc = """Coverage output generator for `--combined_report`. + +Required by Bazel's `collect_coverage.sh` (L186-194 of the script at +https://github.com/bazelbuild/bazel/blob/fde4b67/tools/test/collect_coverage.sh). +Only needed on the test variant, not the binary.""", ), }) diff --git a/py/private/py_venv/templates/console_script.tmpl.sh b/py/private/py_venv/templates/console_script.tmpl.sh new file mode 100644 index 000000000..be19b80c3 --- /dev/null +++ b/py/private/py_venv/templates/console_script.tmpl.sh @@ -0,0 +1,6 @@ +#!/bin/sh +case $0 in + */*) script_dir=${0%/*} ;; + *) script_dir=. ;; +esac +exec "${script_dir}/python" -c 'import sys; from importlib import import_module; from operator import attrgetter; sys.argv[0] = "{{name}}"; sys.exit(attrgetter("{{func}}")(import_module("{{module}}"))())' "$@" diff --git a/py/private/py_venv/tests/BUILD.bazel b/py/private/py_venv/tests/BUILD.bazel index b59d0dac3..d9c4b25a3 100644 --- a/py/private/py_venv/tests/BUILD.bazel +++ b/py/private/py_venv/tests/BUILD.bazel @@ -1,5 +1,7 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") load("@bazel_lib//lib:write_source_files.bzl", "write_source_files") +load("@bazel_skylib//lib:unittest.bzl", "unittest") +load("//py:defs.bzl", "py_test") load("//py:defs.bzl", "py_test") load("//py/private/py_venv:defs.bzl", "py_venv", "py_venv_exec") load(":venv_tree.bzl", "venv_tree") diff --git a/py/private/py_venv/venv.bzl b/py/private/py_venv/venv.bzl deleted file mode 100644 index 9a46c0fad..000000000 --- a/py/private/py_venv/venv.bzl +++ /dev/null @@ -1,376 +0,0 @@ -"""Build-time assembly of a Python virtualenv via ctx.actions.symlink + write. - -This module is the single place in rules_py that declares the files making -up a Python venv. Both `py_binary` / `py_test` (each with its own internal -venv, unless `expose_venv = True` routes them to a sibling py_venv) and -the standalone `py_venv` rule call `assemble_venv` to keep their layouts -bit-identical. - -The venv shape mirrors what CPython's `python -m venv` + pip install -produces, so downstream tools (IDEs, `$VIRTUAL_ENV`-aware shells, -distutils, etc.) treat it as a real venv: - - / - pyvenv.cfg - bin/ - python symlink -> py_toolchain.python - python3 symlink -> python - python3.. symlink -> python - activate bash/zsh activation script - one per wheel-declared entry point - lib/python./site-packages/ - .pth first-party + fallback .pth - _virtualenv.py distutils-compat shim - _virtualenv.pth loads the shim at site init - symlink to a wheel's subdir - / merged PEP 420 namespace: real - / dir, per-entry symlinks - into each contributing wheel - -.dist-info symlink when the wheel needs no - whole-wheel fallback - -The whole tree is declared at analysis time as individual -`ctx.actions.declare_file` / `ctx.actions.declare_symlink` outputs so -Bazel's action cache treats each piece independently (no tree-artifact -+ remote-exec materialisation surprises). -""" - -load("//py/private:py_library.bzl", _py_library = "py_library_utils") -load("//py/private/toolchain:types.bzl", "EXEC_TOOLS_TOOLCHAIN") -load(":toolchains_resolver.bzl", "resolve_venv_toolchain") -load(":virtuals_resolvers.bzl", "enforce_collision_policy", "resolve_wheel_collisions") - -# Template for console-script wrappers under /bin/. A tiny shell -# script that execs the venv's own bin/python with an inline `-c` to import -# the entry point and call it. Keeping this as pure sh (no polyglot) sidesteps -# tokenisation quirks with `$` in strings, and the inline Python is short -# enough that not having a real `__file__` doesn't matter in practice. -# `"$@"` preserves the original argv, while sys.argv[0] is patched so the -# called function sees the script's own name. Entry-point object references -# may contain dotted attributes: -# https://packaging.python.org/en/latest/specifications/entry-points/#data-model -_CONSOLE_SCRIPT_TEMPLATE = """\ -#!/bin/sh -case $0 in - */*) script_dir=${{0%/*}} ;; - *) script_dir=. ;; -esac -exec "${{script_dir}}/python" -c 'import sys; from importlib import import_module; from operator import attrgetter; sys.argv[0] = "{name}"; sys.exit(attrgetter("{func}")(import_module("{module}"))())' "$@" -""" - -def _dict_to_exports(env): - return ["export %s=\"%s\"" % (k, v) for (k, v) in env.items()] - -def assemble_venv( - ctx, - *, - safe_name, - py_toolchain, - imports_depset, - is_windows, - package_collisions, - include_system_site_packages, - include_user_site_packages, - default_env, - venv_activate_tmpl, - virtualenv_shim_py, - site_merge_script_py, - venv_name): - """Declare every file + symlink that makes up a venv for a target. - - Args: - ctx: The rule context. - safe_name: Directory-name-safe stem for the venv dir. Slashes in the - target name should be replaced by the caller (e.g. "_"). - py_toolchain: Resolved Python toolchain struct from py_semantics. - imports_depset: Depset of first-party + transitive wheel import - paths (as returned by py_library_utils.make_imports_depset). - is_windows: Bool — whether the venv targets Windows. - package_collisions: "error" / "warning" / "ignore" — policy applied - when two wheels claim the same top-level (non-namespace case), - distribution metadata entry, or console-script name. - include_system_site_packages: Value for pyvenv.cfg's - `include-system-site-packages` key. - include_user_site_packages: Value for the Aspect extension - `aspect-include-user-site-packages` key. - default_env: Dict of env-var name → value. Exported at the top of - the generated activate script and unset in `deactivate`. - venv_activate_tmpl: File — the activate-script template (usually - `ctx.file._venv_activate_tmpl`). - virtualenv_shim_py: File — the `_virtualenv.py` distutils shim - source (usually `ctx.file._virtualenv_shim`). - site_merge_script_py: File — the site_merge.py tool source - (usually `ctx.file._site_merge_script`). Only needed when the - wheel graph contains a regular package needing a physical merge; the - merge action also requires the rule to declare the (optional) - EXEC_TOOLS_TOOLCHAIN for an exec-configuration interpreter. - venv_name: str — the venv dir basename (e.g. "." + safe_name). - - Returns: - struct with: - bin_python: File — the venv's bin/python symlink, for launchers - to rlocation-resolve and exec. - all_files: list[File] — every declared output, ready for runfiles - / DefaultInfo aggregation. - """ - - wheels_depset = _py_library.make_wheels_depset(ctx) - wheels = wheels_depset.to_list() - top_level_to_site_pkgs, fully_covered_site_pkgs, console_scripts_map, merge_groups, collisions = resolve_wheel_collisions(ctx, wheels) - enforce_collision_policy(collisions, package_collisions) - - # All toolchain-derived path/flag math (runfiles escape arithmetic, - # wheel/venv lib-dir names, pyvenv.cfg home, versioned python names, - # bin/python target_path) is concentrated in resolve_venv_toolchain. - tc = resolve_venv_toolchain( - ctx, - py_toolchain = py_toolchain, - is_windows = is_windows, - venv_name = venv_name, - ) - escape = tc.escape - venv_to_runfiles_escape = tc.venv_to_runfiles_escape - wheel_py_ver = tc.wheel_py_ver - site_packages_rel = tc.site_packages_rel - - # site_packages_rfpath → install_tree, used only by the regular-package - # merge action below. The per-top-level symlinks and .pth lines locate - # each wheel by its runfiles path directly, not through this map. - tree_by_sp = {w.site_packages_rfpath: w.install_tree for w in wheels} - - # site_packages_rfpath → True for wheels whose top-level layout is known - # (they declare `top_levels`), so the per-top-level symlink loop projects - # their root entries — including any root `.pth` files — into the venv - # site-packages. Wheels that carry only `console_scripts` (e.g. source-built - # scripts) leave `top_levels` empty: nothing is projected for them, so their - # `.pth` line must use `site.addsitedir` (see `_format_imp`). - known_layout_site_pkgs = {w.site_packages_rfpath: True for w in wheels if w.top_levels} - - declared = [] # accumulator for all outputs - - # Per-top-level site-packages symlink: a relative symlink escaping from - # site-packages up to the runfiles root, then down into the owning - # wheel's `site_packages_rfpath`/. Works for both install_tree and - # rules_python pip wheels (both stage content at their rfpath). - # `/`-joined top-levels (merged namespace packages, e.g. - # `jaraco/functools`) need one extra `..` per segment. - # This loop runs per (binary x top-level); build the target paths from - # a cached per-wheel prefix rather than formatting every component. - site_packages_prefix = site_packages_rel + "/" - target_prefix_by_sp = {} - for tl, wheel_site_pkgs in top_level_to_site_pkgs.items(): - out = ctx.actions.declare_symlink(site_packages_prefix + tl) - target_prefix = target_prefix_by_sp.get(wheel_site_pkgs) - if target_prefix == None: - target_prefix = escape + "/" + wheel_site_pkgs + "/" - target_prefix_by_sp[wheel_site_pkgs] = target_prefix - target_path = target_prefix + tl - if "/" in tl: - target_path = "../" * tl.count("/") + target_path - ctx.actions.symlink( - output = out, - target_path = target_path, - ) - declared.append(out) - - # Physical merges for regular packages that span wheels or collide at the - # top level (see - # resolve_wheel_collisions). Each group's subtree is copied from - # every contributing wheel into a real directory inside our - # site-packages — the layout a flat `pip install` produces. The - # venv's own site-packages precedes the per-wheel `.pth` entries on - # sys.path, so the merged copy is the one Python binds the regular - # package's `__path__` to; the per-wheel originals are shadowed. - # - # The merge runs as a build action under the exec-configuration - # interpreter (same shape as WhlInstall's unpack action). Every - # PyWheelsInfo record carries an install_tree (see providers.bzl), - # so each contributing wheel resolves in tree_by_sp. - for group in merge_groups: - exec_toolchain = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN] - exec_runtime = exec_toolchain.exec_tools.exec_runtime if exec_toolchain else None - if exec_runtime == None: - fail(("{}: wheels {} all contribute to the regular package `{}` — merging it " + - "requires an exec-configuration Python interpreter, but no `{}` toolchain " + - "was registered.").format( - ctx.label, - group.site_packages_list, - group.root, - EXEC_TOOLS_TOOLCHAIN, - )) - - merged_dir = ctx.actions.declare_directory( - "{}/{}".format(site_packages_rel, group.root), - ) - arguments = ctx.actions.args() - arguments.add(site_merge_script_py) - arguments.add_all([merged_dir], expand_directories = False, before_each = "--into") - arguments.add("--collision-policy", package_collisions) - trees = [] - for sp in group.site_packages_list: - tree = tree_by_sp[sp] - trees.append(tree) - arguments.add_all( - [tree], - expand_directories = False, - before_each = "--src", - format_each = "%s/lib/{}/site-packages/{}".format(wheel_py_ver, group.root), - ) - ctx.actions.run( - mnemonic = "PySiteMerge", - executable = exec_runtime.interpreter, - toolchain = EXEC_TOOLS_TOOLCHAIN, - arguments = [arguments], - inputs = depset( - direct = [site_merge_script_py] + trees, - transitive = [exec_runtime.files], - ), - outputs = [merged_dir], - execution_requirements = { - "supports-path-mapping": "1", - }, - ) - declared.append(merged_dir) - - # A wheel-root `.pth` shim only fires when its file sits in the venv's own - # site-packages. Wheels that declare `top_levels` (known layout) had their - # root entries — including any root `.pth` files — projected there by the - # per-top-level symlink loop above, so they emit a plain path line; - # `site.addsitedir` would re-scan the wheel root and run the shim a second - # time. Wheels with no projected layout (console-script-only, e.g. - # source-built scripts, or metadata-free `py_unpacked_wheel`s) fall back to - # `site.addsitedir` so their site-packages joins sys.path and root `.pth` - # shims run at all; the path is sys.prefix-relative to survive RBE sandbox - # layouts. - def _format_imp(imp): - if imp in fully_covered_site_pkgs: - return None - if imp.endswith("site-packages") and imp not in known_layout_site_pkgs: - return ("import os, sys, site; " + - "site.addsitedir(os.path.normpath(os.path.join(" + - "sys.prefix, \"{venv_escape}\", \"{imp}\")))").format( - venv_escape = venv_to_runfiles_escape, - imp = imp, - ) - return "{}/{}".format(escape, imp) - - pth_lines = ctx.actions.args() - pth_lines.use_param_file("%s", use_always = True) - pth_lines.set_param_file_format("multiline") - pth_lines.add(escape) - - # allow_closure lets _format_imp capture fully_covered_site_pkgs / - # known_layout_site_pkgs so we don't have to materialise imports_depset - # via .to_list(). - pth_lines.add_all(imports_depset, map_each = _format_imp, allow_closure = True) - - site_packages_pth_file = ctx.actions.declare_file( - "{}/{}.pth".format(site_packages_rel, safe_name), - ) - ctx.actions.write( - output = site_packages_pth_file, - content = pth_lines, - ) - declared.append(site_packages_pth_file) - - pyvenv_cfg = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv_name)) - home_line = "home =\n" if tc.pyvenv_home == "" else "home = {}\n".format(tc.pyvenv_home) - ctx.actions.write( - output = pyvenv_cfg, - content = home_line + ( - "implementation = CPython\n" + - "version_info = {major}.{minor}.{micro}\n" + - "include-system-site-packages = {include_system}\n" + - "aspect-include-user-site-packages = {include_user}\n" + - "relocatable = true\n" - ).format( - major = py_toolchain.interpreter_version_info.major, - minor = py_toolchain.interpreter_version_info.minor, - micro = py_toolchain.interpreter_version_info.micro, - include_system = str(include_system_site_packages).lower(), - include_user = str(include_user_site_packages).lower(), - ), - ) - declared.append(pyvenv_cfg) - - # bin/python — unresolved symlink (declare_symlink + target_path) - # rather than declare_file + target_file: target_file lets Bazel pick - # relative vs absolute per version (Bazel 8 rel, Bazel 9 abs), and an - # absolute target bakes in the build-host execroot path that does not - # exist inside an OCI container, leaving a dangling symlink. The - # target path itself is computed by resolve_venv_toolchain. - bin_python = ctx.actions.declare_symlink("{}/bin/python".format(venv_name)) - ctx.actions.symlink( - output = bin_python, - target_path = tc.bin_python_target_path, - ) - declared.append(bin_python) - - # Versioned python symlinks (python3, python3.., and - # python3..t for freethreaded) all point at the sibling - # `python`; names resolved by resolve_venv_toolchain. - for versioned_name in tc.versioned_python_names: - sym = ctx.actions.declare_symlink("{}/bin/{}".format(venv_name, versioned_name)) - ctx.actions.symlink( - output = sym, - target_path = "python", - ) - declared.append(sym) - - # bin/activate - bin_activate = ctx.actions.declare_file("{}/bin/activate".format(venv_name)) - envvar_exports = "\n".join(_dict_to_exports(default_env)).strip() - envvar_unsets = "\n".join( - [" unset {}".format(k) for k in default_env.keys()], - ) - ctx.actions.expand_template( - template = venv_activate_tmpl, - output = bin_activate, - substitutions = { - "{{ENVVARS}}": envvar_exports, - "{{ENVVARS_UNSET}}": envvar_unsets, - }, - is_executable = True, - ) - declared.append(bin_activate) - - # _virtualenv.py + _virtualenv.pth — distutils shim for pip interop. - # Materialised as a real file, not a symlink into the rules_py source - # tree, so tar/OCI/rsync etc. consumers of the venv can resolve the file. - virtualenv_shim_py_out = ctx.actions.declare_file( - "{}/_virtualenv.py".format(site_packages_rel), - ) - ctx.actions.expand_template( - template = virtualenv_shim_py, - output = virtualenv_shim_py_out, - substitutions = {}, - ) - declared.append(virtualenv_shim_py_out) - - virtualenv_shim_pth = ctx.actions.declare_file( - "{}/_virtualenv.pth".format(site_packages_rel), - ) - ctx.actions.write( - output = virtualenv_shim_pth, - content = "import _virtualenv\n", - ) - declared.append(virtualenv_shim_pth) - - # Console-script wrappers under /bin/. - for name, target in console_scripts_map.items(): - script = ctx.actions.declare_file("{}/bin/{}".format(venv_name, name)) - ctx.actions.write( - output = script, - content = _CONSOLE_SCRIPT_TEMPLATE.format( - name = name, - module = target.module, - func = target.func, - ), - is_executable = True, - ) - declared.append(script) - - return struct( - bin_python = bin_python, - all_files = declared, - )