fix(py_venv): link complete runfiles tree (#1147)

`py_venv` contains relative `.pth` entries and symlinks that leave the
venv for source files and wheels in the target's runfiles tree. A
workspace symlink to only the venv changes their lexical base, so both
CPython and IDEs can resolve those paths outside the declared runfiles
layout.

Make `py_venv_link` link the complete directory-based runfiles tree and
print the nested venv path for IDE and shell configuration. This
preserves the generated venv layout, keeps `.pth` entries declarative,
and avoids either rewriting startup paths or adding a recursive
runfiles backedge.

Manifest-only launches cannot expose a directory and fail explicitly.
Windows therefore requires Bazel runfiles trees. The launcher must also
prefer a valid runfiles directory when Bazel supplies both sources; that
behavior is implemented by

tamird/hermetic-launcher@45d9b79.

Author: tamird

README.md

@@ -335,13 +335,13 @@ py_binary(
   with the venv activated. Useful for ad-hoc Python sessions matching your
   binary's deps.
 - `bazel run //:my_app.venv_link` — materialises a workspace-local symlink
-  pointing at the venv tree under `bazel-bin`. **This is what your IDE should
-  point at.**
+  pointing at the target's complete runfiles tree and prints the venv's nested
+  path below that link. **Point your IDE at the printed venv path.**
 
-Then point your IDE to the symlinked virtualenv:
+Then point your IDE to the virtualenv path printed by the command:
 
-- **VSCode**: Set `python.defaultInterpreterPath` to the symlinked path
-- **PyCharm**: Add the symlinked venv as a Python interpreter
+- **VSCode**: Set `python.defaultInterpreterPath` to the printed path
+- **PyCharm**: Add the printed venv path as a Python interpreter
 - **Neovim/LSP**: Configure `python-lsp-server` or `pyright` to use the virtualenv
 
 ### Underlying mechanism

e2e/cases/uv-include-group/BUILD.bazel

@@ -1,4 +1,5 @@
 load("@aspect_rules_py//py:defs.bzl", "py_test", "py_unpacked_wheel")
+load("@rules_shell//shell:sh_test.bzl", "sh_test")
 load("//tools:pth_snapshot.bzl", "extract_venv_pth")
 
 # setuptools vendors `packaging` (among others), which collides at
@@ -54,12 +55,18 @@ py_test(
     name = "wheel_root_pth_test",
     srcs = ["wheel_root_pth_test.py"],
     dep_group = "build",
-    expose_venv = True,
+    expose_venv_link = True,
     isolated = False,
     main = "wheel_root_pth_test.py",
     deps = [":setuptools_no_top_levels"],
 )
 
+sh_test(
+    name = "wheel_root_pth_link_test",
+    srcs = ["wheel_root_pth_link_test.sh"],
+    data = [":wheel_root_pth_test.venv_link"],
+)
+
 # Snapshot fixture: pins the `site.addsitedir(...)` line shape that
 # the non-keyed branch must emit. If anyone collapses it back to a
 # plain path entry, this diff surfaces alongside the runtime test

e2e/cases/uv-include-group/wheel_root_pth_link_test.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env sh
+
+set -ex
+
+ROOT=$(CDPATH= cd "${0%/*}" && pwd -P)
+LINK_DIR=${TEST_TMPDIR}/linked
+mkdir "${LINK_DIR}"
+BAZEL_TARGET=//cases/uv-include-group:wheel_root_pth_test.venv_link \
+    BAZEL_WORKSPACE=_main \
+    BUILD_WORKING_DIRECTORY=${LINK_DIR} \
+    VIRTUAL_ENV=cases/uv-include-group/.wheel_root_pth_test.venv \
+    "${ROOT}/wheel_root_pth_test.venv_link" --name=wheel-root-pth.venv
+LINKED_VENV=${LINK_DIR}/wheel-root-pth.venv/_main/cases/uv-include-group/.wheel_root_pth_test.venv
+test -L "${LINK_DIR}/wheel-root-pth.venv"
+test -d "${LINKED_VENV}"
+"${LINKED_VENV}/bin/python" \
+    "${ROOT}/wheel_root_pth_test.py"

py/defs.bzl

@@ -113,12 +113,13 @@ def py_binary(name, srcs = [], main = None, **kwargs):
               the hermetic interpreter).
             * `expose_venv_link` (bool, default `False`) — when `True`,
               additionally emit a `:{name}.venv_link` py_venv_link.
-              `bazel run :{name}.venv_link` materialises a
-              workspace-local symlink to the venv tree, suitable for an
-              IDE's interpreter setting. Implies `expose_venv = True`;
-              passing `expose_venv = False, expose_venv_link = True`
-              explicitly is rejected with a clear error. Equivalent to
-              declaring an explicit
+              `bazel run :{name}.venv_link` links the target's runfiles
+              tree into the workspace and prints the nested venv path
+              suitable for an IDE's interpreter setting. Implies
+              `expose_venv = True`; passing
+              `expose_venv = False, expose_venv_link = True` explicitly
+              is rejected with a clear error. Equivalent to declaring an
+              explicit
               `py_venv_link(name = "{name}.venv_link", venv = ":{name}.venv")`
               alongside the binary.
     """

py/private/py_venv/link.py

@@ -2,7 +2,7 @@
 
 """%(prog)s [options]
 
-Helper to create a symlink to a virtualenv in the source tree.
+Helper to link a target's runfiles tree into the source tree.
 """
 
 import argparse
@@ -21,9 +21,28 @@ def munge_venv_name(target_package, virtualenv_name):
 
 
 if __name__ == "__main__":
-    virtualenv_home = os.path.normpath(os.environ["VIRTUAL_ENV"])
-    virtualenv_name = os.path.basename(virtualenv_home)
-    runfiles_dir = os.path.normpath(os.environ["RUNFILES_DIR"])
+    try:
+        runfiles_dir = Path(os.path.abspath(os.environ["RUNFILES_DIR"]))
+    except KeyError:
+        raise SystemExit(
+            "py_venv_link requires directory-based runfiles (RUNFILES_DIR)"
+        ) from None
+    if not runfiles_dir.is_dir():
+        raise SystemExit(
+            "py_venv_link RUNFILES_DIR is not a directory: {}".format(runfiles_dir)
+        )
+
+    virtualenv_relative = Path(
+        os.environ["BAZEL_WORKSPACE"],
+        os.environ["VIRTUAL_ENV"],
+    )
+    virtualenv_home = runfiles_dir / virtualenv_relative
+    if not virtualenv_home.is_dir():
+        raise SystemExit(
+            "py_venv_link virtualenv is not a directory: {}".format(virtualenv_home)
+        )
+
+    virtualenv_name = virtualenv_home.name
     builddir = os.path.normpath(os.environ["BUILD_WORKING_DIRECTORY"])
     target_package, target_name = os.environ["BAZEL_TARGET"].split("//", 1)[1].split(":")
 
@@ -37,38 +56,45 @@ def munge_venv_name(target_package, virtualenv_name):
         "--dest",
         dest="dest",
         default=builddir,
-        help="Dir to link the virtualenv into. Default is $BUILD_WORKING_DIRECTORY.",
+        help="Directory in which to create the runfiles link. Default is $BUILD_WORKING_DIRECTORY.",
     )
 
     PARSER.add_argument(
         "--name",
         dest="name",
         default=munge_venv_name(target_package, virtualenv_name),
-        help="Name to link the virtualenv as.",
+        help="Name of the workspace-local runfiles link.",
     )
 
     opts = PARSER.parse_args()
-    dest = Path(os.path.join(opts.dest, opts.name))
+    runfiles_link = Path(opts.dest, opts.name)
+    virtualenv_path = runfiles_link / virtualenv_relative
     print("""
 
-Linking: {venv_home} -> {venv_path}
+Linking runfiles: {runfiles_dir} -> {runfiles_link}
+Virtualenv: {venv_path}
 """.format(
-    venv_home = virtualenv_home,
-    venv_path = dest,
+    runfiles_dir = runfiles_dir,
+    runfiles_link = runfiles_link,
+    venv_path = virtualenv_path,
 ))
 
-    if dest.exists() and dest.is_symlink() and dest.readlink() == Path(virtualenv_home):
+    if (
+        runfiles_link.exists()
+        and runfiles_link.is_symlink()
+        and runfiles_link.readlink() == runfiles_dir
+    ):
         print("Link is up to date!")
 
     else:
         try:
-            dest.lstat()
-            dest.unlink()
+            runfiles_link.lstat()
+            runfiles_link.unlink()
         except FileNotFoundError:
             pass
 
         # From -> to
-        dest.symlink_to(virtualenv_home, target_is_directory=True)
+        runfiles_link.symlink_to(runfiles_dir, target_is_directory=True)
         print("Link created!")
 
     print("""
@@ -85,7 +111,6 @@ def munge_venv_name(target_package, virtualenv_name):
 virtualenvwrapper users may further want to
     $ ln -s {venv_path} $WORKON_HOME/{venv_name}
 """.format(
-    venv_home = virtualenv_home,
     venv_name = opts.name,
-    venv_path = dest,
+    venv_path = virtualenv_path,
 ))

py/private/py_venv/py_venv.bzl

@@ -15,9 +15,10 @@
   interpreter.
 
 - `py_venv_link` — opt-in macro that emits a runnable target whose
-  `bazel run` materialises a workspace-local symlink to an existing
-  `py_venv`'s tree. Pair with `py_binary(expose_venv = True)` to hand
-  your IDE a stable `.venv` symlink to point at.
+  `bazel run` links the target's complete runfiles tree into the
+  workspace and prints the nested `py_venv` path. Pair with
+  `py_binary(expose_venv = True)` to give your IDE a stable interpreter
+  path.
 
 Shared venv-assembly logic lives in
 `//py/private/py_venv:venv.bzl::assemble_venv`. See that file's header for the
@@ -341,10 +342,10 @@ def py_binary_with_venv(py_rule, name, main, srcs = [], deps = [], data = [], im
 
     `expose_venv_link = True` additionally emits a public
     `:{name}.venv_link` py_venv_link. `bazel run :{name}.venv_link`
-    materialises a workspace-local symlink at the venv's tree under
-    `bazel-bin`, suitable for pointing an IDE at. Implies
-    `expose_venv = True` — the link target needs a public venv to point
-    at. Passing `expose_venv = False, expose_venv_link = True`
+    materialises a workspace-local symlink to the target's runfiles tree and
+    prints the venv path below that link, suitable for pointing an IDE at.
+    Implies `expose_venv = True` — the link target needs a public venv to
+    point at. Passing `expose_venv = False, expose_venv_link = True`
     explicitly is contradictory and fails.
 
     All venv-shaping attrs (`deps`, `imports`, `package_collisions`,
@@ -415,21 +416,21 @@ def py_venv_link(name, venv, link_name = None, **kwargs):
     """Emit a runnable target that materialises `venv` into the workspace.
 
     `bazel run :<name>` creates a symlink in `$BUILD_WORKING_DIRECTORY`
-    (typically the workspace root) that points at `venv`'s materialised
-    venv tree in bazel-bin, so IDEs / LSPs can resolve interpreter and
-    site-packages via a stable workspace-local path.
+    (typically the workspace root) that points at the target's complete
+    runfiles tree. The command prints `venv`'s nested path below that link;
+    preserving the runfiles directory layout keeps the venv's relative paths
+    valid for Python and IDEs. This requires directory-based runfiles; a
+    manifest alone cannot expose a runfiles tree.
 
     Args:
         name: Runnable target name. `bazel run :<name>` materialises
-            the symlink; pick something like `<something>.venv` by
-            convention so `python.defaultInterpreterPath` readers
-            recognise it, but any label works.
+            the runfiles symlink.
         venv: Label of a `py_venv` target to link. Typically the
             `:<binary_name>.venv` target auto-emitted by
             `py_binary(expose_venv = True, ...)`, or a standalone
             `py_venv` shared across many binaries.
         link_name: Workspace-relative basename for the created
-            symlink. Defaults to a safely-escaped version of the
+            runfiles symlink. Defaults to a safely-escaped version of the
             target's package + venv name.
         **kwargs: Forwarded to the underlying `py_binary`.
     """

py/tests/py-venv-standalone-interpreter/BUILD.bazel

@@ -1,7 +1,7 @@
 load("@rules_shell//shell:sh_test.bzl", "sh_test")
 load("//py:defs.bzl", "py_binary")
 
-# Regression test: `py_binary(expose_venv = True, isolated = False)` —
+# Regression test: `py_binary(expose_venv_link = True, isolated = False)` —
 # the opt-in split shape — produces a runnable venv whose interpreter
 # honors `-I`-less semantics (PYTHONPATH, script-dir-on-sys.path,
 # optionally user-site). This is the shape users land on when they
@@ -11,7 +11,7 @@ py_binary(
     srcs = [
         "ex.py",
     ],
-    expose_venv = True,
+    expose_venv_link = True,
     imports = [
         ".",
     ],
@@ -29,5 +29,6 @@ sh_test(
     ],
     data = [
         ":ex",
+        ":ex.venv_link",
     ],
 )

py/tests/py-venv-standalone-interpreter/test.sh

@@ -2,10 +2,24 @@
 
 set -ex
 
-ROOT="$(dirname $0)"
+ROOT=$(CDPATH= cd "${0%/*}" && pwd -P)
 
 "$ROOT"/.ex.venv/bin/python --help >/dev/null 2>&1
 
-if [ "Hello, world!" != "$($ROOT/.ex.venv/bin/python -c 'from ex import hello; print(hello())')" ]; then
+if [ "Hello, world!" != "$("${ROOT}/.ex.venv/bin/python" -c 'from ex import hello; print(hello())')" ]; then
     exit 1
 fi
+
+LINK_DIR=${TEST_TMPDIR}/linked
+mkdir "${LINK_DIR}"
+BAZEL_TARGET=//py/tests/py-venv-standalone-interpreter:ex.venv_link \
+    BAZEL_WORKSPACE=_main \
+    BUILD_WORKING_DIRECTORY=${LINK_DIR} \
+    VIRTUAL_ENV=py/tests/py-venv-standalone-interpreter/.ex.venv \
+    "${ROOT}/ex.venv_link" --name=ex.venv
+cd "${LINK_DIR}"
+LINKED_VENV=${LINK_DIR}/ex.venv/_main/py/tests/py-venv-standalone-interpreter/.ex.venv
+test -L "${LINK_DIR}/ex.venv"
+test -d "${LINKED_VENV}"
+"${LINKED_VENV}/bin/python" -c \
+    'import cowsay, sys; from ex import hello; assert hello() == "Hello, world!"; assert sys.prefix != sys.base_prefix'