diff --git a/py/private/py_venv/BUILD.bazel b/py/private/py_venv/BUILD.bazel index 86249c161..ae0927a5e 100644 --- a/py/private/py_venv/BUILD.bazel +++ b/py/private/py_venv/BUILD.bazel @@ -42,12 +42,13 @@ bzl_library( ) bzl_library( - name = "venv", - srcs = ["venv.bzl"], + name = "assemble_venv", + srcs = ["assemble_venv.bzl"], deps = [ ":toolchains_resolver", ":virtuals_resolvers", "//py/private:py_library", + "//py/private/toolchain:types", ], ) @@ -76,9 +77,9 @@ bzl_library( name = "py_venv", srcs = ["py_venv.bzl"], deps = [ + ":assemble_venv", ":py_venv_exec", ":types", - ":venv", "//py/private:py_library", "//py/private:py_semantics", "//py/private:transitions", diff --git a/py/private/py_venv/assemble_venv.bzl b/py/private/py_venv/assemble_venv.bzl new file mode 100644 index 000000000..81711b502 --- /dev/null +++ b/py/private/py_venv/assemble_venv.bzl @@ -0,0 +1,360 @@ +"""Build-time assembly of a Python virtualenv via ctx.actions.symlink + write. + +This module is the single place in rules_py that declares the files making +up a Python venv. Both ``py_binary`` / ``py_test`` (each with its own +internal venv, unless ``expose_venv = True`` routes them to a sibling +py_venv) and the standalone ``py_venv`` rule call ``assemble_venv`` to +keep their layouts bit-identical. + +The venv shape mirrors what CPython's ``python -m venv`` + pip install +produces, so downstream tools (IDEs, ``$VIRTUAL_ENV``-aware shells, +distutils, etc.) treat it as a real venv: + + / + pyvenv.cfg + bin/ + python symlink -> py_toolchain.python + python3 symlink -> python + python3.. symlink -> python + activate bash/zsh activation script + one per wheel-declared entry point + lib/python./site-packages/ + .pth first-party + fallback .pth + _virtualenv.py distutils-compat shim + _virtualenv.pth loads the shim at site init + symlink to a wheel's subdir + / merged PEP 420 namespace + -.dist-info symlink when the wheel needs no + whole-wheel fallback + +The whole tree is declared at analysis time as individual +``ctx.actions.declare_file`` / ``ctx.actions.declare_symlink`` outputs so +Bazel's action cache treats each piece independently (no tree-artifact ++ remote-exec materialisation surprises). +""" + +load("//py/private:py_library.bzl", _py_library = "py_library_utils") +load("//py/private/toolchain:types.bzl", "EXEC_TOOLS_TOOLCHAIN") +load(":toolchains_resolver.bzl", "resolve_venv_toolchain") +load(":virtuals_resolvers.bzl", "compute_wheel_plan", "enforce_collision_policy") + +def _dict_to_exports(env): + """Render an env dict as ``export KEY="VALUE"`` shell lines.""" + return ["export %s=\"%s\"" % (k, v) for (k, v) in env.items()] + +def _make_pth_formatter(fully_covered, known_layout, escape, venv_escape): + """Build a ``map_each`` closure for ``.pth`` import formatting. + + Three branches per import path: + + * Fully covered wheel → ``None`` (omitted from the ``.pth``). + * Layout-unknown wheel whose path ends in ``site-packages`` → + ``site.addsitedir`` with a ``sys.prefix``-relative path that + survives RBE sandbox layouts. + * Everything else → plain relative escape + import path. + """ + + def _format_imp(imp): + if imp in fully_covered: + return None + if imp.endswith("site-packages") and imp not in known_layout: + return ("import os, sys, site; " + + "site.addsitedir(os.path.normpath(os.path.join(" + + "sys.prefix, \"{}\", \"{}\")))").format(venv_escape, imp) + return "{}/{}".format(escape, imp) + + return _format_imp + +def _declare_toplevel_symlinks(ctx, top_level_to_site_pkgs, tc, declared): + """Declare per-top-level site-packages symlinks into owning wheels. + + Each symlink escapes from ``site-packages/`` up to the runfiles root, + then descends into the owning wheel's ``site_packages_rfpath``. + ``/``-joined top-levels (merged namespace packages, e.g. + ``jaraco/functools``) get one extra ``../`` per segment. A per-wheel + prefix cache avoids recomputing the escape path. + """ + prefix = tc.site_packages_rel + "/" + target_prefix_by_sp = {} + for tl, wheel_sp in top_level_to_site_pkgs.items(): + out = ctx.actions.declare_symlink(prefix + tl) + target_prefix = target_prefix_by_sp.get(wheel_sp) + if target_prefix == None: + target_prefix = tc.escape + "/" + wheel_sp + "/" + target_prefix_by_sp[wheel_sp] = target_prefix + target_path = target_prefix + tl + if "/" in tl: + target_path = "../" * tl.count("/") + target_path + ctx.actions.symlink(output = out, target_path = target_path) + declared.append(out) + +def _run_site_merges( + ctx, + merge_groups, + tree_by_sp, + tc, + site_merge_script_py, + package_collisions, + exec_runtime, + declared): + """Run ``PySiteMerge`` actions for physical package merges. + + Each group's subtree is copied from every contributing wheel into a + real directory inside site-packages — the layout a flat ``pip install`` + produces. The venv's own site-packages precedes per-wheel ``.pth`` + entries on ``sys.path``, so the merged copy is what Python binds the + regular package's ``__path__`` to; per-wheel originals are shadowed. + """ + for group in merge_groups: + merged_dir = ctx.actions.declare_directory( + "{}/{}".format(tc.site_packages_rel, group.root), + ) + arguments = ctx.actions.args() + arguments.add(site_merge_script_py) + arguments.add_all([merged_dir], expand_directories = False, before_each = "--into") + arguments.add("--collision-policy", package_collisions) + trees = [] + for sp in group.site_packages_list: + tree = tree_by_sp[sp] + trees.append(tree) + arguments.add_all( + [tree], + expand_directories = False, + before_each = "--src", + format_each = "%s/lib/{}/site-packages/{}".format(tc.wheel_py_ver, group.root), + ) + ctx.actions.run( + mnemonic = "PySiteMerge", + executable = exec_runtime.interpreter, + toolchain = EXEC_TOOLS_TOOLCHAIN, + arguments = [arguments], + inputs = depset( + direct = [site_merge_script_py] + trees, + transitive = [exec_runtime.files], + ), + outputs = [merged_dir], + execution_requirements = {"supports-path-mapping": "1"}, + ) + declared.append(merged_dir) + +def _declare_pth_file(ctx, imports_depset, format_imp, tc, safe_name, declared): + """Generate the ``.pth`` file for first-party and fallback imports. + + Uses ``map_each`` with ``allow_closure`` so the imports depset is + consumed at execution time (param-file writing), not at analysis + time — avoiding an unnecessary ``.to_list()``. + """ + pth_lines = ctx.actions.args() + pth_lines.use_param_file("%s", use_always = True) + pth_lines.set_param_file_format("multiline") + pth_lines.add(tc.escape) + pth_lines.add_all(imports_depset, map_each = format_imp, allow_closure = True) + + out = ctx.actions.declare_file( + "{}/{}.pth".format(tc.site_packages_rel, safe_name), + ) + ctx.actions.write(output = out, content = pth_lines) + declared.append(out) + +def _declare_pyvenv_cfg( + ctx, + tc, + py_toolchain, + venv_name, + include_system, + include_user, + declared): + """Write ``pyvenv.cfg`` with version, home, and site-packages flags.""" + home_line = "home =\n" if tc.pyvenv_home == "" else "home = {}\n".format(tc.pyvenv_home) + out = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv_name)) + ctx.actions.write( + output = out, + content = home_line + ( + "implementation = CPython\n" + + "version_info = {major}.{minor}.{micro}\n" + + "include-system-site-packages = {include_system}\n" + + "aspect-include-user-site-packages = {include_user}\n" + + "relocatable = true\n" + ).format( + major = py_toolchain.interpreter_version_info.major, + minor = py_toolchain.interpreter_version_info.minor, + micro = py_toolchain.interpreter_version_info.micro, + include_system = str(include_system).lower(), + include_user = str(include_user).lower(), + ), + ) + declared.append(out) + +def _declare_python_bin(ctx, tc, venv_name, declared): + """Declare ``bin/python`` + versioned symlinks (``python3``, ``python3.X`` …). + + Returns the ``bin/python`` File for launcher consumption. + """ + bin_python = ctx.actions.declare_symlink("{}/bin/python".format(venv_name)) + ctx.actions.symlink(output = bin_python, target_path = tc.bin_python_target_path) + declared.append(bin_python) + + for name in tc.versioned_python_names: + sym = ctx.actions.declare_symlink("{}/bin/{}".format(venv_name, name)) + ctx.actions.symlink(output = sym, target_path = "python") + declared.append(sym) + + return bin_python + +def _declare_activate(ctx, default_env, venv_activate_tmpl, venv_name, declared): + """Expand the ``bin/activate`` template with env exports and unsets.""" + out = ctx.actions.declare_file("{}/bin/activate".format(venv_name)) + exports = "\n".join(_dict_to_exports(default_env)).strip() + unsets = "\n".join([" unset {}".format(k) for k in default_env.keys()]) + ctx.actions.expand_template( + template = venv_activate_tmpl, + output = out, + substitutions = {"{{ENVVARS}}": exports, "{{ENVVARS_UNSET}}": unsets}, + is_executable = True, + ) + declared.append(out) + +def _declare_virtualenv_shim(ctx, site_packages_rel, virtualenv_shim_py, declared): + """Materialise ``_virtualenv.py`` + ``_virtualenv.pth`` in site-packages. + + Uses ``expand_template`` with empty substitutions (verbatim copy) + so the shim is a real file, not a symlink into the source tree — + ``os.path.realpath`` resolves inside the venv for tar/OCI consumers. + """ + shim_py = ctx.actions.declare_file("{}/_virtualenv.py".format(site_packages_rel)) + ctx.actions.expand_template( + template = virtualenv_shim_py, + output = shim_py, + substitutions = {}, + ) + declared.append(shim_py) + + shim_pth = ctx.actions.declare_file("{}/_virtualenv.pth".format(site_packages_rel)) + ctx.actions.write(output = shim_pth, content = "import _virtualenv\n") + declared.append(shim_pth) + +def _declare_console_scripts(ctx, console_scripts_map, console_script_tmpl, venv_name, declared): + """Expand one shell wrapper per wheel-declared entry point.""" + for name, target in console_scripts_map.items(): + script = ctx.actions.declare_file("{}/bin/{}".format(venv_name, name)) + ctx.actions.expand_template( + template = console_script_tmpl, + output = script, + substitutions = { + "{{name}}": name, + "{{module}}": target.module, + "{{func}}": target.func, + }, + is_executable = True, + ) + declared.append(script) + +def assemble_venv( + ctx, + *, + safe_name, + py_toolchain, + imports_depset, + is_windows, + package_collisions, + include_system_site_packages, + include_user_site_packages, + default_env, + venv_activate_tmpl, + virtualenv_shim_py, + site_merge_script_py, + console_script_tmpl, + venv_name): + """Declare every file + symlink that makes up a venv for a target. + + Args: + ctx: The rule context. + safe_name: Directory-name-safe stem for the venv dir. + py_toolchain: Resolved Python toolchain struct from py_semantics. + imports_depset: Depset of import paths (from + ``py_library_utils.make_imports_depset``). + is_windows: Whether the venv targets Windows. + package_collisions: ``"error"`` / ``"warning"`` / ``"ignore"``. + include_system_site_packages: Value for pyvenv.cfg's key. + include_user_site_packages: Value for the Aspect extension key. + default_env: Env-var dict for the activate script. + venv_activate_tmpl: File — activate template. + virtualenv_shim_py: File — ``_virtualenv.py`` source. + site_merge_script_py: File — ``site_merge.py`` tool. + console_script_tmpl: File — console-script template. + venv_name: The venv dir basename (e.g. ``.``). + + Returns: + struct with ``bin_python`` (File) and ``all_files`` (list[File]). + """ + tc = resolve_venv_toolchain( + ctx, + py_toolchain = py_toolchain, + is_windows = is_windows, + venv_name = venv_name, + ) + + plan = compute_wheel_plan(ctx, _py_library.make_wheels_depset(ctx).to_list()) + enforce_collision_policy(plan.collisions, package_collisions) + + top_level_to_site_pkgs = plan.top_level_to_site_pkgs + fully_covered = plan.fully_covered + console_scripts_map = plan.console_scripts_map + merge_groups = plan.merge_groups + tree_by_sp = plan.tree_by_sp + known_layout = plan.known_layout + declared = [] + + _declare_toplevel_symlinks(ctx, top_level_to_site_pkgs, tc, declared) + + if merge_groups: + exec_toolchain = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN] + exec_runtime = exec_toolchain.exec_tools.exec_runtime if exec_toolchain else None + if exec_runtime == None: + fail(("{}: wheels {} all contribute to the regular package `{}` — merging it " + + "requires an exec-configuration Python interpreter, but no `{}` toolchain " + + "was registered.").format( + ctx.label, + merge_groups[0].site_packages_list, + merge_groups[0].root, + EXEC_TOOLS_TOOLCHAIN, + )) + _run_site_merges( + ctx, + merge_groups, + tree_by_sp, + tc, + site_merge_script_py, + package_collisions, + exec_runtime, + declared, + ) + + format_imp = _make_pth_formatter( + fully_covered, + known_layout, + tc.escape, + tc.venv_to_runfiles_escape, + ) + _declare_pth_file(ctx, imports_depset, format_imp, tc, safe_name, declared) + + _declare_pyvenv_cfg( + ctx, + tc, + py_toolchain, + venv_name, + include_system_site_packages, + include_user_site_packages, + declared, + ) + + bin_python = _declare_python_bin(ctx, tc, venv_name, declared) + + _declare_activate(ctx, default_env, venv_activate_tmpl, venv_name, declared) + _declare_virtualenv_shim(ctx, tc.site_packages_rel, virtualenv_shim_py, declared) + _declare_console_scripts(ctx, console_scripts_map, console_script_tmpl, venv_name, declared) + + return struct( + bin_python = bin_python, + all_files = declared, + ) diff --git a/py/private/py_venv/py_venv.bzl b/py/private/py_venv/py_venv.bzl index d65112d22..7b57659ae 100644 --- a/py/private/py_venv/py_venv.bzl +++ b/py/private/py_venv/py_venv.bzl @@ -31,9 +31,9 @@ load("//py/private:py_library.bzl", _py_library = "py_library_utils") load("//py/private:py_semantics.bzl", _py_semantics = "semantics") load("//py/private:transitions.bzl", "python_transition") load("//py/private/toolchain:types.bzl", "EXEC_TOOLS_TOOLCHAIN", "PY_TOOLCHAIN") +load(":assemble_venv.bzl", "assemble_venv") load(":py_venv_exec.bzl", _py_venv_exec = "py_venv_exec") load(":types.bzl", "VirtualenvInfo", "venv_root") -load(":venv.bzl", "assemble_venv") def _interpreter_flags(ctx): args = _py_semantics.interpreter_flags + ctx.attr.interpreter_options diff --git a/py/private/py_venv/py_venv_exec.bzl b/py/private/py_venv/py_venv_exec.bzl index be403cbe5..a649ef119 100644 --- a/py/private/py_venv/py_venv_exec.bzl +++ b/py/private/py_venv/py_venv_exec.bzl @@ -1,8 +1,8 @@ """Implementation for the py_venv_exec and py_venv_exec_test rules. -Both are thin launchers that consume a sibling `py_venv` (passed via the -internal `venv` attr) and exec its `bin/python`. The public -`py_binary` / `py_test` macros wrap them and route all venv-shaping +Both are thin launchers that consume a sibling ``py_venv`` (passed via +the internal ``venv`` attr) and exec its ``bin/python``. The public +``py_binary`` / ``py_test`` macros wrap them and route all venv-shaping attrs to the auto-generated sibling. """ @@ -13,39 +13,39 @@ load("//py/private:py_semantics.bzl", _py_semantics = "semantics") load("//py/private:transitions.bzl", "reset_python_flags_transition") load(":types.bzl", "VirtualenvInfo", "venv_root") -# Identifiers the launcher always sets to the analysing rule's contextual -# values. Excluded from `inherited_environment` so that a stray -# `env_inherit` entry can't let an outer shell shadow the contextual -# label at run time. _CONTEXTUAL_ENV_KEYS = ("BAZEL_TARGET", "BAZEL_WORKSPACE", "BAZEL_TARGET_NAME") -def _py_venv_exec_impl(ctx): - # The launcher itself doesn't need a python toolchain — it just - # exec's the sibling venv's `bin/python`, whose path was already - # resolved when the venv was analysed. Default interpreter flags - # come from a shared constant. - # - # The macro layer routes srcs / deps to the sibling py_venv (always - # set as `venv`) and passes an explicit `main =` to the rule. - # `main` is the only first-party file the rule contributes; - # everything else flows through the sibling venv. - if not ctx.attr.main: - fail("py_binary {}: main is required.".format(ctx.label)) +def _validate_main(ctx): + """Return the main module ``File``, failing if absent or not ``.py``.""" main = ctx.file.main + if not main: + fail("py_binary {}: main is required.".format(ctx.label)) if not main.basename.endswith(".py"): fail("main must end in '.py', got: " + main.basename) + return main - venv = ctx.attr.venv - vinfo = venv[VirtualenvInfo] +def _set_contextual_env(ctx, passed_env): + """Set Bazel contextual identifiers that always override user env.""" + passed_env["BAZEL_TARGET"] = str(ctx.label).lstrip("@") + passed_env["BAZEL_WORKSPACE"] = ctx.workspace_name + passed_env["BAZEL_TARGET_NAME"] = ctx.attr.name + +def _strip_contextual_from_inherited(inherited_env): + """Remove contextual keys so a stray ``env_inherit`` can't shadow them. - # Merge env vars: start from the venv's `env` (if any), then - # overlay the binary's own — binary wins on key conflicts. Same - # merge for inherited env-var names. Bazel-contextual identifiers - # (BAZEL_TARGET, etc.) overlay last and are stripped from - # `inherited_env` so a stray `env_inherit` entry can't let the - # caller's shell shadow the contextual label — per - # https://bazel.build/rules/lib/providers/RunEnvironmentInfo, an - # inherited value wins over `environment` when both are present. + Per RunEnvironmentInfo semantics, an inherited value wins over + ``environment`` when both are present — stripping here prevents + the caller's shell from overriding the contextual identifiers. + """ + return [n for n in inherited_env if n not in _CONTEXTUAL_ENV_KEYS] + +def _merge_environment(ctx, venv, vinfo): + """Merge env vars from the sibling venv, the binary, and contextual keys. + + Precedence (last wins): venv ``RunEnvironmentInfo`` -> ``VIRTUAL_ENV`` + -> binary ``env`` -> contextual keys. Returns + ``(passed_env, inherited_env)``. + """ passed_env = {} inherited_env = [] if RunEnvironmentInfo in venv: @@ -53,13 +53,9 @@ def _py_venv_exec_impl(ctx): passed_env = dict(venv_env.environment) inherited_env = list(venv_env.inherited_environment) - # Owned by the rule. The lib venv variant carries no `env` to guard, - # so guard here to match the executable variant's check. if "VIRTUAL_ENV" in ctx.attr.env: fail("py_binary/py_test {}: `VIRTUAL_ENV` is set by the rule and cannot be overridden via `env`.".format(ctx.label)) - # Set here so it's present even for the lib venv variant, which has - # no RunEnvironmentInfo to carry it. passed_env["VIRTUAL_ENV"] = venv_root(vinfo.bin_python) for k, v in ctx.attr.env.items(): passed_env[k] = expand_variables( @@ -67,27 +63,39 @@ def _py_venv_exec_impl(ctx): expand_locations(ctx, v, ctx.attr.data), attribute_name = "env", ) + + inherited_set = {n: True for n in inherited_env} for name in ctx.attr.env_inherit: - if name not in inherited_env: + if name not in inherited_set: inherited_env.append(name) - passed_env["BAZEL_TARGET"] = str(ctx.label).lstrip("@") - passed_env["BAZEL_WORKSPACE"] = ctx.workspace_name - passed_env["BAZEL_TARGET_NAME"] = ctx.attr.name - inherited_env = [n for n in inherited_env if n not in _CONTEXTUAL_ENV_KEYS] + inherited_set[name] = True + + _set_contextual_env(ctx, passed_env) + inherited_env = _strip_contextual_from_inherited(inherited_env) + + return passed_env, inherited_env - # When `isolated = False`, drop Python's `-I` flag so PYTHONPATH is - # honored and the script directory is auto-added to sys.path. - flags = list(_py_semantics.interpreter_flags) + ctx.attr.interpreter_options +def _interpreter_flags(ctx): + """Build the interpreter flag list. + + Base flags come from ``py_semantics``. When ``isolated = False``, + ``-I`` is stripped from the **base** flags so PYTHONPATH is honored. + User ``interpreter_options`` are always appended verbatim — a user + explicitly passing ``-I`` survives the isolated-stripping. + """ + base = list(_py_semantics.interpreter_flags) if not ctx.attr.isolated: - flags = [f for f in flags if f != "-I"] + base = [f for f in base if f != "-I"] + return base + list(ctx.attr.interpreter_options) + +def _build_launcher(ctx, vinfo, flags, main): + """Compile the native launcher stub via ``hermetic_launcher``. - # Native launcher via hermetic_launcher. Embedded argv: - # [0] venv's bin/python (runfiles-resolved) - # [1+] interpreter flags (literal, e.g. -I, -X importtime) - # [N] main module path (runfiles-resolved) - # The launcher runtime resolves transformed-arg positions through - # the Bazel runfiles manifest, then `execve`s the venv python. - executable_launcher = ctx.actions.declare_file(ctx.attr.name) + Embedded argv: venv's ``bin/python``, interpreter flags, then the + main module path. The launcher runtime resolves runfiles positions + and ``execve``'s the venv python. + """ + executable = ctx.actions.declare_file(ctx.attr.name) embedded_args, transformed_args = launcher.args_from_entrypoint(vinfo.bin_python) for flag in flags: embedded_args, transformed_args = launcher.append_embedded_arg( @@ -104,43 +112,48 @@ def _py_venv_exec_impl(ctx): ctx = ctx, embedded_args = embedded_args, transformed_args = transformed_args, - output_file = executable_launcher, + output_file = executable, ) + return executable - # Merge runfiles, supporting `py_venv_exec(main)` not being in the `py_venv` runfiles. - runfiles = ctx.runfiles( +def _build_runfiles(ctx, venv, main): + """Merge data files, the main module, and the venv's runfiles.""" + return ctx.runfiles( files = ctx.files.data + [main], ).merge_all( [target[DefaultInfo].default_runfiles for target in ctx.attr.data] + [venv[DefaultInfo].default_runfiles], ) - instrumented_files_info = coverage_common.instrumented_files_info( - ctx, - source_attributes = ["main"], - dependency_attributes = ["data", "venv"], - extensions = ["py"], - ) +def _py_venv_exec_impl(ctx): + main = _validate_main(ctx) + venv = ctx.attr.venv + vinfo = venv[VirtualenvInfo] + + passed_env, inherited_env = _merge_environment(ctx, venv, vinfo) + flags = _interpreter_flags(ctx) + executable = _build_launcher(ctx, vinfo, flags, main) + runfiles = _build_runfiles(ctx, venv, main) return [ DefaultInfo( - files = depset([executable_launcher, main]), - executable = executable_launcher, + files = depset([executable, main]), + executable = executable, runfiles = runfiles, ), PyInfo( - # Surface the venv's imports + transitive_sources through - # PyInfo so downstream consumers (e.g. py_pex_binary's - # `--sys-path=`) see the same sys.path / source closure the - # launcher will run with. `srcs` / `deps` live on the - # sibling venv, not on this rule. imports = vinfo.imports, transitive_sources = vinfo.transitive_sources, has_py2_only_sources = False, has_py3_only_sources = True, uses_shared_libraries = False, ), - instrumented_files_info, + coverage_common.instrumented_files_info( + ctx, + source_attributes = ["main"], + dependency_attributes = ["data", "venv"], + extensions = ["py"], + ), RunEnvironmentInfo( environment = passed_env, inherited_environment = inherited_env, @@ -158,20 +171,8 @@ _attrs = dict({ ), "main": attr.label( allow_single_file = True, - doc = """ -Script to execute with the Python interpreter. - -Must be a label pointing to a `.py` source file. -If such a label is provided, it will be honored. - -If no label is provided AND there is only one `srcs` file, that `srcs` file will be used. - -If there are more than one `srcs`, a file matching `{name}.py` is searched for. -This is for historical compatibility with the Bazel native `py_binary` and `rules_python`. -Relying on this behavior is STRONGLY discouraged, may produce warnings and may -be deprecated in the future. - -""", + mandatory = True, + doc = "Python source file to execute. The macro layer resolves this from `srcs` when not set explicitly.", ), "venv": attr.label( providers = [[VirtualenvInfo]], @@ -201,11 +202,6 @@ sibling venv has `include_user_site_packages = True` set. The deprecated `py_venv_binary` / `py_venv_test` aliases default this to False to match their historical permissive behaviour.""", ), - # `data` is the only py_library attr the launcher reads (env-var - # location expansion, runfiles merge, coverage walk). `srcs`, - # `deps`, `imports`, `resolutions`, and `virtual_deps` are routed - # to the sibling py_venv by the macro layer and have no role on - # the launcher rule. "data": attr.label_list( doc = """Runtime dependencies of the program. @@ -219,13 +215,12 @@ that must match the terminal's Python environment in `deps`. allow_files = True, cfg = reset_python_flags_transition, ), - # Forwarded to the sibling py_venv (which is where srcs actually - # feed sys.path). Carried on the launcher only so Bazel's `args` - # location-expansion (`args = ["$(location :foo.py)"]`) can resolve - # the label against the same files the user wrote on - # `py_binary` / `py_test`. "srcs": attr.label_list( - doc = "Python source files. Forwarded to the sibling py_venv.", + doc = """Python source files. Forwarded to the sibling py_venv where +they feed sys.path. Carried on the launcher so that Bazel's `args` +location-expansion (`args = ["$(location :foo.py)"]`) can resolve the +label. The files reach runfiles transitively through the venv's +default_runfiles, not from this attribute directly.""", allow_files = [".py"], ), "_allowlist_function_transition": attr.label( @@ -234,15 +229,15 @@ that must match the terminal's Python environment in `deps`. }) _test_attrs = dict({ - # Magic attribute to make coverage --combined_report flag work. - # There's no docs about this. - # See https://github.com/bazelbuild/bazel/blob/fde4b67009d377a3543a3dc8481147307bd37d36/tools/test/collect_coverage.sh#L186-L194 - # NB: rules_python ALSO includes this attribute on the py_binary rule, but we think that's a mistake. - # see https://github.com/aspect-build/rules_py/pull/520#pullrequestreview-2579076197 "_lcov_merger": attr.label( default = configuration_field(fragment = "coverage", name = "output_generator"), executable = True, cfg = "exec", + doc = """Coverage output generator for `--combined_report`. + +Required by Bazel's `collect_coverage.sh` (L186-194 of the script at +https://github.com/bazelbuild/bazel/blob/fde4b67/tools/test/collect_coverage.sh). +Only needed on the test variant, not the binary.""", ), }) diff --git a/py/private/py_venv/venv.bzl b/py/private/py_venv/venv.bzl deleted file mode 100644 index e92c66332..000000000 --- a/py/private/py_venv/venv.bzl +++ /dev/null @@ -1,362 +0,0 @@ -"""Build-time assembly of a Python virtualenv via ctx.actions.symlink + write. - -This module is the single place in rules_py that declares the files making -up a Python venv. Both `py_binary` / `py_test` (each with its own internal -venv, unless `expose_venv = True` routes them to a sibling py_venv) and -the standalone `py_venv` rule call `assemble_venv` to keep their layouts -bit-identical. - -The venv shape mirrors what CPython's `python -m venv` + pip install -produces, so downstream tools (IDEs, `$VIRTUAL_ENV`-aware shells, -distutils, etc.) treat it as a real venv: - - / - pyvenv.cfg - bin/ - python symlink -> py_toolchain.python - python3 symlink -> python - python3.. symlink -> python - activate bash/zsh activation script - one per wheel-declared entry point - lib/python./site-packages/ - .pth first-party + fallback .pth - _virtualenv.py distutils-compat shim - _virtualenv.pth loads the shim at site init - symlink to a wheel's subdir - / merged PEP 420 namespace: real - / dir, per-entry symlinks - into each contributing wheel - -.dist-info symlink when the wheel needs no - whole-wheel fallback - -The whole tree is declared at analysis time as individual -`ctx.actions.declare_file` / `ctx.actions.declare_symlink` outputs so -Bazel's action cache treats each piece independently (no tree-artifact -+ remote-exec materialisation surprises). -""" - -load("//py/private:py_library.bzl", _py_library = "py_library_utils") -load("//py/private/toolchain:types.bzl", "EXEC_TOOLS_TOOLCHAIN") -load(":toolchains_resolver.bzl", "resolve_venv_toolchain") -load(":virtuals_resolvers.bzl", "enforce_collision_policy", "resolve_wheel_collisions") - -def _dict_to_exports(env): - return ["export %s=\"%s\"" % (k, v) for (k, v) in env.items()] - -def assemble_venv( - ctx, - *, - safe_name, - py_toolchain, - imports_depset, - is_windows, - package_collisions, - include_system_site_packages, - include_user_site_packages, - default_env, - venv_activate_tmpl, - virtualenv_shim_py, - site_merge_script_py, - console_script_tmpl, - venv_name): - """Declare every file + symlink that makes up a venv for a target. - - Args: - ctx: The rule context. - safe_name: Directory-name-safe stem for the venv dir. Slashes in the - target name should be replaced by the caller (e.g. "_"). - py_toolchain: Resolved Python toolchain struct from py_semantics. - imports_depset: Depset of first-party + transitive wheel import - paths (as returned by py_library_utils.make_imports_depset). - is_windows: Bool — whether the venv targets Windows. - package_collisions: "error" / "warning" / "ignore" — policy applied - when two wheels claim the same top-level (non-namespace case), - distribution metadata entry, or console-script name. - include_system_site_packages: Value for pyvenv.cfg's - `include-system-site-packages` key. - include_user_site_packages: Value for the Aspect extension - `aspect-include-user-site-packages` key. - default_env: Dict of env-var name → value. Exported at the top of - the generated activate script and unset in `deactivate`. - venv_activate_tmpl: File — the activate-script template (usually - `ctx.file._venv_activate_tmpl`). - virtualenv_shim_py: File — the `_virtualenv.py` distutils shim - source (usually `ctx.file._virtualenv_shim`). - site_merge_script_py: File — the site_merge.py tool source - (usually `ctx.file._site_merge_script`). Only needed when the - wheel graph contains a regular package needing a physical merge; the - merge action also requires the rule to declare the (optional) - EXEC_TOOLS_TOOLCHAIN for an exec-configuration interpreter. - console_script_tmpl: File — the console-script wrapper template - (usually `ctx.file._console_script_tmpl`). - venv_name: str — the venv dir basename (e.g. "." + safe_name). - - Returns: - struct with: - bin_python: File — the venv's bin/python symlink, for launchers - to rlocation-resolve and exec. - all_files: list[File] — every declared output, ready for runfiles - / DefaultInfo aggregation. - """ - - wheels_depset = _py_library.make_wheels_depset(ctx) - wheels = wheels_depset.to_list() - top_level_to_site_pkgs, fully_covered_site_pkgs, console_scripts_map, merge_groups, collisions = resolve_wheel_collisions(ctx, wheels) - enforce_collision_policy(collisions, package_collisions) - - # All toolchain-derived path/flag math (runfiles escape arithmetic, - # wheel/venv lib-dir names, pyvenv.cfg home, versioned python names, - # bin/python target_path) is concentrated in resolve_venv_toolchain. - tc = resolve_venv_toolchain( - ctx, - py_toolchain = py_toolchain, - is_windows = is_windows, - venv_name = venv_name, - ) - escape = tc.escape - venv_to_runfiles_escape = tc.venv_to_runfiles_escape - wheel_py_ver = tc.wheel_py_ver - site_packages_rel = tc.site_packages_rel - - # site_packages_rfpath → install_tree, used only by the regular-package - # merge action below. The per-top-level symlinks and .pth lines locate - # each wheel by its runfiles path directly, not through this map. - tree_by_sp = {w.site_packages_rfpath: w.install_tree for w in wheels} - - # site_packages_rfpath → True for wheels whose top-level layout is known - # (they declare `top_levels`), so the per-top-level symlink loop projects - # their root entries — including any root `.pth` files — into the venv - # site-packages. Wheels that carry only `console_scripts` (e.g. source-built - # scripts) leave `top_levels` empty: nothing is projected for them, so their - # `.pth` line must use `site.addsitedir` (see `_format_imp`). - known_layout_site_pkgs = {w.site_packages_rfpath: True for w in wheels if w.top_levels} - - declared = [] # accumulator for all outputs - - # Per-top-level site-packages symlink: a relative symlink escaping from - # site-packages up to the runfiles root, then down into the owning - # wheel's `site_packages_rfpath`/. Works for both install_tree and - # rules_python pip wheels (both stage content at their rfpath). - # `/`-joined top-levels (merged namespace packages, e.g. - # `jaraco/functools`) need one extra `..` per segment. - # This loop runs per (binary x top-level); build the target paths from - # a cached per-wheel prefix rather than formatting every component. - site_packages_prefix = site_packages_rel + "/" - target_prefix_by_sp = {} - for tl, wheel_site_pkgs in top_level_to_site_pkgs.items(): - out = ctx.actions.declare_symlink(site_packages_prefix + tl) - target_prefix = target_prefix_by_sp.get(wheel_site_pkgs) - if target_prefix == None: - target_prefix = escape + "/" + wheel_site_pkgs + "/" - target_prefix_by_sp[wheel_site_pkgs] = target_prefix - target_path = target_prefix + tl - if "/" in tl: - target_path = "../" * tl.count("/") + target_path - ctx.actions.symlink( - output = out, - target_path = target_path, - ) - declared.append(out) - - # Physical merges for regular packages that span wheels or collide at the - # top level (see - # resolve_wheel_collisions). Each group's subtree is copied from - # every contributing wheel into a real directory inside our - # site-packages — the layout a flat `pip install` produces. The - # venv's own site-packages precedes the per-wheel `.pth` entries on - # sys.path, so the merged copy is the one Python binds the regular - # package's `__path__` to; the per-wheel originals are shadowed. - # - # The merge runs as a build action under the exec-configuration - # interpreter (same shape as WhlInstall's unpack action). Every - # PyWheelsInfo record carries an install_tree (see providers.bzl), - # so each contributing wheel resolves in tree_by_sp. - for group in merge_groups: - exec_toolchain = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN] - exec_runtime = exec_toolchain.exec_tools.exec_runtime if exec_toolchain else None - if exec_runtime == None: - fail(("{}: wheels {} all contribute to the regular package `{}` — merging it " + - "requires an exec-configuration Python interpreter, but no `{}` toolchain " + - "was registered.").format( - ctx.label, - group.site_packages_list, - group.root, - EXEC_TOOLS_TOOLCHAIN, - )) - - merged_dir = ctx.actions.declare_directory( - "{}/{}".format(site_packages_rel, group.root), - ) - arguments = ctx.actions.args() - arguments.add(site_merge_script_py) - arguments.add_all([merged_dir], expand_directories = False, before_each = "--into") - arguments.add("--collision-policy", package_collisions) - trees = [] - for sp in group.site_packages_list: - tree = tree_by_sp[sp] - trees.append(tree) - arguments.add_all( - [tree], - expand_directories = False, - before_each = "--src", - format_each = "%s/lib/{}/site-packages/{}".format(wheel_py_ver, group.root), - ) - ctx.actions.run( - mnemonic = "PySiteMerge", - executable = exec_runtime.interpreter, - toolchain = EXEC_TOOLS_TOOLCHAIN, - arguments = [arguments], - inputs = depset( - direct = [site_merge_script_py] + trees, - transitive = [exec_runtime.files], - ), - outputs = [merged_dir], - execution_requirements = { - "supports-path-mapping": "1", - }, - ) - declared.append(merged_dir) - - # A wheel-root `.pth` shim only fires when its file sits in the venv's own - # site-packages. Wheels that declare `top_levels` (known layout) had their - # root entries — including any root `.pth` files — projected there by the - # per-top-level symlink loop above, so they emit a plain path line; - # `site.addsitedir` would re-scan the wheel root and run the shim a second - # time. Wheels with no projected layout (console-script-only, e.g. - # source-built scripts, or metadata-free `py_unpacked_wheel`s) fall back to - # `site.addsitedir` so their site-packages joins sys.path and root `.pth` - # shims run at all; the path is sys.prefix-relative to survive RBE sandbox - # layouts. - def _format_imp(imp): - if imp in fully_covered_site_pkgs: - return None - if imp.endswith("site-packages") and imp not in known_layout_site_pkgs: - return ("import os, sys, site; " + - "site.addsitedir(os.path.normpath(os.path.join(" + - "sys.prefix, \"{venv_escape}\", \"{imp}\")))").format( - venv_escape = venv_to_runfiles_escape, - imp = imp, - ) - return "{}/{}".format(escape, imp) - - pth_lines = ctx.actions.args() - pth_lines.use_param_file("%s", use_always = True) - pth_lines.set_param_file_format("multiline") - pth_lines.add(escape) - - # allow_closure lets _format_imp capture fully_covered_site_pkgs / - # known_layout_site_pkgs so we don't have to materialise imports_depset - # via .to_list(). - pth_lines.add_all(imports_depset, map_each = _format_imp, allow_closure = True) - - site_packages_pth_file = ctx.actions.declare_file( - "{}/{}.pth".format(site_packages_rel, safe_name), - ) - ctx.actions.write( - output = site_packages_pth_file, - content = pth_lines, - ) - declared.append(site_packages_pth_file) - - pyvenv_cfg = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv_name)) - home_line = "home =\n" if tc.pyvenv_home == "" else "home = {}\n".format(tc.pyvenv_home) - ctx.actions.write( - output = pyvenv_cfg, - content = home_line + ( - "implementation = CPython\n" + - "version_info = {major}.{minor}.{micro}\n" + - "include-system-site-packages = {include_system}\n" + - "aspect-include-user-site-packages = {include_user}\n" + - "relocatable = true\n" - ).format( - major = py_toolchain.interpreter_version_info.major, - minor = py_toolchain.interpreter_version_info.minor, - micro = py_toolchain.interpreter_version_info.micro, - include_system = str(include_system_site_packages).lower(), - include_user = str(include_user_site_packages).lower(), - ), - ) - declared.append(pyvenv_cfg) - - # bin/python — unresolved symlink (declare_symlink + target_path) - # rather than declare_file + target_file: target_file lets Bazel pick - # relative vs absolute per version (Bazel 8 rel, Bazel 9 abs), and an - # absolute target bakes in the build-host execroot path that does not - # exist inside an OCI container, leaving a dangling symlink. The - # target path itself is computed by resolve_venv_toolchain. - bin_python = ctx.actions.declare_symlink("{}/bin/python".format(venv_name)) - ctx.actions.symlink( - output = bin_python, - target_path = tc.bin_python_target_path, - ) - declared.append(bin_python) - - # Versioned python symlinks (python3, python3.., and - # python3..t for freethreaded) all point at the sibling - # `python`; names resolved by resolve_venv_toolchain. - for versioned_name in tc.versioned_python_names: - sym = ctx.actions.declare_symlink("{}/bin/{}".format(venv_name, versioned_name)) - ctx.actions.symlink( - output = sym, - target_path = "python", - ) - declared.append(sym) - - # bin/activate - bin_activate = ctx.actions.declare_file("{}/bin/activate".format(venv_name)) - envvar_exports = "\n".join(_dict_to_exports(default_env)).strip() - envvar_unsets = "\n".join( - [" unset {}".format(k) for k in default_env.keys()], - ) - ctx.actions.expand_template( - template = venv_activate_tmpl, - output = bin_activate, - substitutions = { - "{{ENVVARS}}": envvar_exports, - "{{ENVVARS_UNSET}}": envvar_unsets, - }, - is_executable = True, - ) - declared.append(bin_activate) - - # _virtualenv.py + _virtualenv.pth — distutils shim for pip interop. - # Materialised as a real file, not a symlink into the rules_py source - # tree, so tar/OCI/rsync etc. consumers of the venv can resolve the file. - virtualenv_shim_py_out = ctx.actions.declare_file( - "{}/_virtualenv.py".format(site_packages_rel), - ) - ctx.actions.expand_template( - template = virtualenv_shim_py, - output = virtualenv_shim_py_out, - substitutions = {}, - ) - declared.append(virtualenv_shim_py_out) - - virtualenv_shim_pth = ctx.actions.declare_file( - "{}/_virtualenv.pth".format(site_packages_rel), - ) - ctx.actions.write( - output = virtualenv_shim_pth, - content = "import _virtualenv\n", - ) - declared.append(virtualenv_shim_pth) - - # Console-script wrappers under /bin/. - for name, target in console_scripts_map.items(): - script = ctx.actions.declare_file("{}/bin/{}".format(venv_name, name)) - ctx.actions.expand_template( - template = console_script_tmpl, - output = script, - substitutions = { - "{{name}}": name, - "{{module}}": target.module, - "{{func}}": target.func, - }, - is_executable = True, - ) - declared.append(script) - - return struct( - bin_python = bin_python, - all_files = declared, - )