diff --git a/docs/uv.md b/docs/uv.md index 876214218..100980383 100644 --- a/docs/uv.md +++ b/docs/uv.md @@ -20,10 +20,14 @@ configurations, such as going from a Darwin macbook to a Linux container image using only the normal Bazel `platforms` machinery. **Correct source builds** - Because uv performs package source builds as a -normal part of your build, it's able to use hermetic or even source built Python -toolchains in addition to Bazel-defined dependencies and C compilers. Future -support for sysroots is planned. Due to its phasing, `pip.parse` is stuck doing -all this non-hermetically. +normal part of your build, it can use hermetic or even source-built Python +toolchains in addition to Bazel-defined dependencies. Native builds select the +C and C++ tools from Bazel's configured compile actions, but do not reuse their +action flags or environment because PEP 517 backends own the compile and link +commands. Tools that require target, sysroot, resource, wrapper, or environment +configuration must provide a backend-compatible command through the package +`env`; generic C++ toolchain projection is not supported. Due to its phasing, +`pip.parse` is stuck doing all this non-hermetically. **Editable requirements** - Uv provides an `uv.override_requirement()` tag which allows locked requirements to be replaced with 1stparty Bazel `py_library` diff --git a/e2e/cases/uv-sdist-native-build/BUILD.bazel b/e2e/cases/uv-sdist-native-build/BUILD.bazel index 09c7930ee..e7c2602f7 100644 --- a/e2e/cases/uv-sdist-native-build/BUILD.bazel +++ b/e2e/cases/uv-sdist-native-build/BUILD.bazel @@ -1,26 +1,29 @@ # Regression test: python-geohash (sdist-only, C++ extension) must be buildable -# when native_build_toolchain_type is registered via @rules_py_tools//:all. +# with native_build_toolchain_type registered via @rules_py_tools//:all. # # The pep517_native_whl rule requires the native_build_toolchain_type to resolve, -# asserting that the exec and target platforms match. This test verifies that the -# per-platform toolchain entries generated into @rules_py_tools are reachable. +# asserting that the exec and target platforms match. The source patch also +# introduces a C++ standard-library dependency, so linking fails if the helper +# invokes the configured compiler in C rather than C++ driver mode. load("@aspect_rules_py//py:defs.bzl", "py_test") +exports_files(["require_cxx_driver.patch"]) + py_test( name = "test", srcs = ["test_geohash.py"], + dep_group = "uv-sdist-native-build", main = "test_geohash.py", - target_compatible_with = ["@platforms//os:linux"], deps = ["@pypi_uv_sdist_native_build//python_geohash"], ) py_test( name = "venv_test", srcs = ["test_geohash.py"], + dep_group = "uv-sdist-native-build", expose_venv = True, isolated = False, main = "test_geohash.py", - target_compatible_with = ["@platforms//os:linux"], deps = ["@pypi_uv_sdist_native_build//python_geohash"], ) diff --git a/e2e/cases/uv-sdist-native-build/pyproject.toml b/e2e/cases/uv-sdist-native-build/pyproject.toml index ea59763e6..57dbf8e7f 100644 --- a/e2e/cases/uv-sdist-native-build/pyproject.toml +++ b/e2e/cases/uv-sdist-native-build/pyproject.toml @@ -3,5 +3,7 @@ name = "uv-sdist-native-build" version = "0.0.0" requires-python = ">=3.11" dependencies = [ + "build==1.4.0", "python-geohash", + "setuptools==75.8.2", ] diff --git a/e2e/cases/uv-sdist-native-build/require_cxx_driver.patch b/e2e/cases/uv-sdist-native-build/require_cxx_driver.patch new file mode 100644 index 000000000..c3e889dbd --- /dev/null +++ b/e2e/cases/uv-sdist-native-build/require_cxx_driver.patch @@ -0,0 +1,30 @@ +diff --git a/setup.py b/setup.py +--- a/setup.py ++++ b/setup.py +@@ -9,0 +10 @@ c1=Extension('_geohash', ++c1.sources.append('src/native_probe.c') +diff --git a/src/geohash.cpp b/src/geohash.cpp +--- a/src/geohash.cpp ++++ b/src/geohash.cpp +@@ -19,0 +20,4 @@ ++#include ++ ++extern "C" int native_c_probe(void); ++ +@@ -725,0 +731,6 @@ ++ ++static PyObject *py_native_probe(PyObject *, PyObject *) { ++ std::string suffix = "++"; ++ return PyLong_FromLong(native_c_probe() + suffix.size()); ++} ++ +@@ -732,0 +745 @@ ++ {"native_probe", py_native_probe, METH_NOARGS, "exercise C and C++ compilation and linkage"}, +diff --git a/src/native_probe.c b/src/native_probe.c +new file mode 100644 +--- /dev/null ++++ b/src/native_probe.c +@@ -0,0 +1,3 @@ ++int native_c_probe(void) { ++ return 40; ++} diff --git a/e2e/cases/uv-sdist-native-build/setup.MODULE.bazel b/e2e/cases/uv-sdist-native-build/setup.MODULE.bazel index 64f499350..7625a15fd 100644 --- a/e2e/cases/uv-sdist-native-build/setup.MODULE.bazel +++ b/e2e/cases/uv-sdist-native-build/setup.MODULE.bazel @@ -22,6 +22,8 @@ uv.project( uv.override_package( name = "python-geohash", lock = "//cases/uv-sdist-native-build:uv.lock", + pre_build_patch_strip = 1, + pre_build_patches = ["//cases/uv-sdist-native-build:require_cxx_driver.patch"], resource_set = "mem_4g", ) diff --git a/e2e/cases/uv-sdist-native-build/test_geohash.py b/e2e/cases/uv-sdist-native-build/test_geohash.py index 8366edb8d..78782f08e 100644 --- a/e2e/cases/uv-sdist-native-build/test_geohash.py +++ b/e2e/cases/uv-sdist-native-build/test_geohash.py @@ -1,9 +1,11 @@ """Verify that python-geohash (built from sdist via pep517_native_whl) is importable.""" import geohash +import _geohash def test_encode_decode(): + assert _geohash.native_probe() == 42 lat, lon = 37.7749, -122.4194 encoded = geohash.encode(lat, lon) assert encoded, "geohash.encode should return a non-empty string" diff --git a/e2e/cases/uv-sdist-native-build/uv.lock b/e2e/cases/uv-sdist-native-build/uv.lock index 72655613c..880fd1b11 100644 --- a/e2e/cases/uv-sdist-native-build/uv.lock +++ b/e2e/cases/uv-sdist-native-build/uv.lock @@ -5,7 +5,7 @@ requires-python = ">=3.11" [[package]] name = "build" version = "1.4.0" -source = { registry = "https://pypi.org/simple" } +source = { registry = "https://pypi.org/simple/" } dependencies = [ { name = "colorama", marker = "os_name == 'nt'" }, { name = "packaging" }, @@ -19,7 +19,7 @@ wheels = [ [[package]] name = "colorama" version = "0.4.6" -source = { registry = "https://pypi.org/simple" } +source = { registry = "https://pypi.org/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, @@ -28,7 +28,7 @@ wheels = [ [[package]] name = "packaging" version = "26.0" -source = { registry = "https://pypi.org/simple" } +source = { registry = "https://pypi.org/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/65/ee/299d360cdc32edc7d2cf530f3accf79c4fca01e96ffc950d8a52213bd8e4/packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4", size = 143416, upload-time = "2026-01-21T20:50:39.064Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, @@ -37,7 +37,7 @@ wheels = [ [[package]] name = "pyproject-hooks" version = "1.2.0" -source = { registry = "https://pypi.org/simple" } +source = { registry = "https://pypi.org/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/e7/82/28175b2414effca1cdac8dc99f76d660e7a4fb0ceefa4b4ab8f5f6742925/pyproject_hooks-1.2.0.tar.gz", hash = "sha256:1e859bd5c40fae9448642dd871adf459e5e2084186e8d2c2a79a824c970da1f8", size = 19228, upload-time = "2024-09-29T09:24:13.293Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, @@ -46,13 +46,13 @@ wheels = [ [[package]] name = "python-geohash" version = "0.8.5" -source = { registry = "https://pypi.org/simple" } +source = { registry = "https://pypi.org/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/9c/e2/1a3507af7c8f91f8a4975d651d4aeb6a846dfdf74713954186ade4205850/python-geohash-0.8.5.tar.gz", hash = "sha256:05a21fcf4eda1a5eddbd291890ade23fc5ddaa6bb98f2ee23d2d384ed14f086d", size = 17636, upload-time = "2014-08-13T07:09:57.318Z" } [[package]] name = "setuptools" version = "75.8.2" -source = { registry = "https://pypi.org/simple" } +source = { registry = "https://pypi.org/simple/" } sdist = { url = "https://files.pythonhosted.org/packages/d1/53/43d99d7687e8cdef5ab5f9ec5eaf2c0423c2b35133a2b7e7bc276fc32b21/setuptools-75.8.2.tar.gz", hash = "sha256:4880473a969e5f23f2a2be3646b2dfd84af9028716d398e46192f84bc36900d2", size = 1344083, upload-time = "2025-02-26T20:45:19.103Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a9/38/7d7362e031bd6dc121e5081d8cb6aa6f6fedf2b67bf889962134c6da4705/setuptools-75.8.2-py3-none-any.whl", hash = "sha256:558e47c15f1811c1fa7adbd0096669bf76c1d3f433f58324df69f3f5ecac4e8f", size = 1229385, upload-time = "2025-02-26T20:45:17.259Z" }, @@ -63,8 +63,14 @@ name = "uv-sdist-native-build" version = "0.0.0" source = { virtual = "." } dependencies = [ + { name = "build" }, { name = "python-geohash" }, + { name = "setuptools" }, ] [package.metadata] -requires-dist = [{ name = "python-geohash" }] +requires-dist = [ + { name = "build", specifier = "==1.4.0" }, + { name = "python-geohash" }, + { name = "setuptools", specifier = "==75.8.2" }, +] diff --git a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel index daa114fdc..bf569951e 100644 --- a/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel +++ b/e2e/snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel @@ -20,14 +20,9 @@ pep517_native_whl( "@@bazel_tools//tools/jdk:current_java_runtime", ], env = { - "AR": "$(AR)", - "CC": "$(CC)", - "CXX": "$(CC)", "JAR": "$(JAVABASE)/bin/jar", "JAVA": "$(JAVA)", "JAVA_HOME": "$(JAVABASE)", - "LD": "$(LD)", - "STRIP": "$(STRIP)", }, visibility = ["//visibility:public"], ) diff --git a/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel b/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel index 6cbd37583..5654c7c46 100644 --- a/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel +++ b/e2e/snapshots/sdist_build.uv_sdist_native_build.python_geohash.BUILD.bazel @@ -16,15 +16,13 @@ pep517_native_whl( version = "0.8.5", args = [], resource_set = "mem_4g", + pre_build_patches = ["@@//cases/uv-sdist-native-build:require_cxx_driver.patch"], + pre_build_patch_strip = 1, toolchains = [ "@bazel_tools//tools/cpp:current_cc_toolchain", ], env = { - "AR": "$(AR)", - "CC": "$(CC)", - "CXX": "$(CC)", - "LD": "$(LD)", - "STRIP": "$(STRIP)", + }, visibility = ["//visibility:public"], ) diff --git a/uv/private/extension/defs.bzl b/uv/private/extension/defs.bzl index 38584211e..1494e157e 100644 --- a/uv/private/extension/defs.bzl +++ b/uv/private/extension/defs.bzl @@ -407,9 +407,9 @@ def _parse_projects(module_ctx, hub_specs): pre_build_patches = [str(p) for p in pkg_override.pre_build_patches] pre_build_patch_strip = pkg_override.pre_build_patch_strip - # `toolchains` / `env` on `uv.override_package` augment - # the defaults baked into sdist_build's BUILD template — - # they don't replace them. Empty == no augmentation. + # `toolchains` on `uv.override_package` extends the + # defaults baked into sdist_build's BUILD template. `env` + # is merged over those defaults, replacing named values. extra_toolchains = [] extra_env = {} resource_set = "default" @@ -782,7 +782,7 @@ _override_package_tag = tag_class( ), "env": attr.string_dict( default = {}, - doc = "Extra environment variables merged into the build action's `env` dict. Values may reference $(VAR) make-variables sourced from the default CC toolchain or any extra `toolchains` listed above.", + doc = "Environment variables merged over the native-build defaults. Values may reference $(VAR) make-variables sourced from the default CC toolchain or any extra `toolchains` listed above.", ), # Pre-build patches: applied to extracted sdist source before wheel build. diff --git a/uv/private/pep517_whl/BUILD.bazel b/uv/private/pep517_whl/BUILD.bazel index 5b52d993a..fea51a4ad 100644 --- a/uv/private/pep517_whl/BUILD.bazel +++ b/uv/private/pep517_whl/BUILD.bazel @@ -1,8 +1,18 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_shell//shell:sh_binary.bzl", "sh_binary") +load("//py:defs.bzl", "py_test") load(":rule.bzl", "pep517_native_whl") -load(":test.bzl", "pep517_native_whl_toolchain_env_test") +load( + ":test.bzl", + "compiler_selection_toolchain", + "pep517_native_whl_compiler_selection_test", + "pep517_native_whl_disabled_cxx_test", + "pep517_native_whl_missing_cxx_test", + "pep517_native_whl_same_driver_clang_test", + "pep517_native_whl_same_driver_gcc_test", + "pep517_native_whl_toolchain_env_test", +) package(default_visibility = [ "//uv/private:__subpackages__", @@ -19,6 +29,17 @@ bzl_library( deps = [ "//py/private/toolchain:types", "@bazel_lib//lib:resource_sets", + "@rules_cc//cc:action_names_bzl", + ], +) + +py_test( + name = "build_helper_native_tools_test", + srcs = ["build_helper_native_tools_test.py"], + data = [":build_helper.py"], + deps = [ + "@pypi//build", + "@pypi//setuptools", ], ) @@ -29,6 +50,7 @@ bzl_library( deps = [ ":rule", "@bazel_skylib//lib:unittest", + "@rules_cc//cc:bzl_srcs", ], ) @@ -69,11 +91,6 @@ pep517_native_whl( name = "__toolchain_env_fixture", src = ":__stub_sdist", env = { - "CC": "$(CC)", - "CXX": "$(CC)", - "AR": "$(AR)", - "LD": "$(LD)", - "STRIP": "$(STRIP)", "JAVA_HOME": "$(JAVABASE)", "JAVA": "$(JAVA)", "JAR": "$(JAVABASE)/bin/jar", @@ -89,5 +106,186 @@ pep517_native_whl( pep517_native_whl_toolchain_env_test( name = "toolchain_env_test", + check_toolchain_env = True, + target_under_test = ":__toolchain_env_fixture", +) + +compiler_selection_toolchain( + name = "__compiler_selection", + ar_driver = "ar_driver_test.sh", + c_driver = "c_driver_test.sh", + compiler = "gcc", + compiler_actions = [ + "CC", + "CXX", + ], + cxx_driver = "cxx_driver_test.sh", + ld_driver = "ld_driver_test.sh", + no_legacy_features = False, + optional_tools = [ + "AR", + "STRIP", + ], + strip_driver = "strip_driver_test.sh", +) + +compiler_selection_toolchain( + name = "__same_driver_clang", + ar_driver = "ar_driver_test.sh", + c_driver = "c_driver_test.sh", + compiler = "clang", + compiler_actions = [ + "CC", + "CXX", + ], + cxx_driver = "c_driver_test.sh", + ld_driver = "ld_driver_test.sh", + no_legacy_features = False, + optional_tools = [ + "AR", + "STRIP", + ], + strip_driver = "strip_driver_test.sh", +) + +compiler_selection_toolchain( + name = "__same_driver_gcc", + ar_driver = "ar_driver_test.sh", + c_driver = "c_driver_test.sh", + compiler = "gcc-12", + compiler_actions = [ + "CC", + "CXX", + ], + cxx_driver = "c_driver_test.sh", + ld_driver = "ld_driver_test.sh", + no_legacy_features = False, + optional_tools = [], + strip_driver = "strip_driver_test.sh", +) + +compiler_selection_toolchain( + name = "__missing_cxx", + ar_driver = "ar_driver_test.sh", + c_driver = "c_driver_test.sh", + compiler = "gcc", + compiler_actions = ["CC"], + cxx_driver = "cxx_driver_test.sh", + ld_driver = "ld_driver_test.sh", + no_legacy_features = False, + optional_tools = [], + strip_driver = "strip_driver_test.sh", +) + +compiler_selection_toolchain( + name = "__disabled_cxx", + ar_driver = "ar_driver_test.sh", + c_driver = "c_driver_test.sh", + compiler = "gcc", + compiler_actions = ["CC"], + cxx_driver = "cxx_driver_test.sh", + ld_driver = "ld_driver_test.sh", + no_legacy_features = True, + optional_tools = [], + strip_driver = "strip_driver_test.sh", +) + +pep517_native_whl_compiler_selection_test( + name = "compiler_selection_test", + expected_tools = { + "AR": "uv/private/pep517_whl/ar_driver_test.sh", + "CC": "uv/private/pep517_whl/c_driver_test.sh", + "CXX": "uv/private/pep517_whl/cxx_driver_test.sh", + "STRIP": "uv/private/pep517_whl/strip_driver_test.sh", + }, + target_under_test = ":__toolchain_env_fixture", +) + +pep517_native_whl_same_driver_clang_test( + name = "same_driver_clang_test", + expected_cxx_args = ["--driver-mode=g++"], + expected_tools = { + "AR": "uv/private/pep517_whl/ar_driver_test.sh", + "CC": "uv/private/pep517_whl/c_driver_test.sh", + "CXX": "uv/private/pep517_whl/c_driver_test.sh", + "STRIP": "uv/private/pep517_whl/strip_driver_test.sh", + }, + target_under_test = ":__toolchain_env_fixture", +) + +pep517_native_whl_same_driver_gcc_test( + name = "same_driver_gcc_test", + expected_tool_errors = { + "CXX": "C++ toolchain 'gcc-12' uses 'uv/private/pep517_whl/c_driver_test.sh' for both CC and CXX; set an explicit CXX override", + }, + expected_tools = { + "CC": "uv/private/pep517_whl/c_driver_test.sh", + }, target_under_test = ":__toolchain_env_fixture", ) + +pep517_native_whl_missing_cxx_test( + name = "missing_cxx_test", + expected_tool_errors = { + "CXX": "C++ toolchain CXX tool is absent from all_files: uv/private/pep517_whl/DUMMY_GCC_TOOL", + }, + expected_tools = { + "CC": "uv/private/pep517_whl/c_driver_test.sh", + }, + target_under_test = ":__toolchain_env_fixture", +) + +pep517_native_whl_disabled_cxx_test( + name = "disabled_cxx_test", + expected_tool_errors = { + "CXX": "C++ toolchain does not enable the c++-compile action required for CXX", + }, + expected_tools = { + "CC": "uv/private/pep517_whl/c_driver_test.sh", + }, + target_under_test = ":__toolchain_env_fixture", +) + +pep517_native_whl( + name = "__missing_cxx_override_fixture", + src = ":__stub_sdist", + env = {"CXX": "explicit-c++"}, + tags = ["manual"], + tool = ":__stub_tool", + version = "0.0.1", +) + +pep517_native_whl_missing_cxx_test( + name = "missing_cxx_override_test", + expected_env = {"CXX": "explicit-c++"}, + expected_tools = { + "CC": "uv/private/pep517_whl/c_driver_test.sh", + }, + target_under_test = ":__missing_cxx_override_fixture", +) + +pep517_native_whl( + name = "__same_driver_gcc_override_fixture", + src = ":__stub_sdist", + env = { + "AR": "explicit-ar --mode", + "CXX": "explicit-c++", + "LD": "explicit-ld --mode", + }, + tags = ["manual"], + tool = ":__stub_tool", + version = "0.0.1", +) + +pep517_native_whl_same_driver_gcc_test( + name = "same_driver_gcc_override_test", + expected_env = { + "AR": "explicit-ar --mode", + "CXX": "explicit-c++", + "LD": "explicit-ld --mode", + }, + expected_tools = { + "CC": "uv/private/pep517_whl/c_driver_test.sh", + }, + target_under_test = ":__same_driver_gcc_override_fixture", +) diff --git a/uv/private/pep517_whl/ar_driver_test.sh b/uv/private/pep517_whl/ar_driver_test.sh new file mode 100755 index 000000000..039e4d006 --- /dev/null +++ b/uv/private/pep517_whl/ar_driver_test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/uv/private/pep517_whl/build_helper.py b/uv/private/pep517_whl/build_helper.py index 10774c6c3..3cd443434 100644 --- a/uv/private/pep517_whl/build_helper.py +++ b/uv/private/pep517_whl/build_helper.py @@ -7,6 +7,7 @@ """ from argparse import ArgumentParser +import json import os import platform as _platform import shlex @@ -28,27 +29,42 @@ ) -# `$(CC)` etc. from the pep517_native_whl rule expands to a Bazel -# workspace-relative path (e.g. external/llvm+/toolchain/gcc) that -# resolves from the action execroot but not from the build -# subprocess's cwd inside the unpacked worktree. To keep CC reachable -# after that cwd change, we drop a tiny wrapper into tmp_root (which -# is absolute, so its path survives the cwd change) and point CC / -# CXX / CPP / LDSHARED / LDCXXSHARED at the wrapper. The wrapper -# strips `-fdebug-default-version=4` (older toolchains reject it) -# and then `execv`s the compiler at its resolved absolute path. +# Configured tools may be relative to Bazel's execroot, while build backends +# run from the unpacked source tree. Resolve every executable before that cwd +# change; default compiler wrappers embed the resulting absolute paths. _DEBUG_FLAG = "-fdebug-default-version=4" _COMPILER_WRAPPER = """#!/usr/bin/env python3 import os import sys -filtered_args = [arg for arg in sys.argv[1:] if arg != "{debug_flag}"] +compiler = {compiler!r} +driver_args = {driver_args!r} +filtered_args = [ + arg for arg in driver_args + sys.argv[1:] if arg != {debug_flag!r} +] sysroot = {sysroot!r} -if sysroot and "-isysroot" not in filtered_args: +has_sysroot = any( + arg in ("-isysroot", "--sysroot") + or arg.startswith("-isysroot") + or arg.startswith("--sysroot=") + for arg in filtered_args +) +if sysroot and not has_sysroot: filtered_args = ["-isysroot", sysroot] + filtered_args -os.execv("{compiler_path}", [os.path.basename("{compiler_path}")] + filtered_args) +os.execv(compiler, [compiler] + filtered_args) +""" +_ERROR_WRAPPER = """#!/usr/bin/env python3 +import sys + +sys.stderr.write({error!r} + "\\n") +sys.exit(1) """ +_DEFAULT_COMPILERS = { + "CC": "cc", + "CXX": "c++", +} + def _darwin_sysroot(): """Return the macOS SDK path, or None if unavailable.""" @@ -60,45 +76,62 @@ def _darwin_sysroot(): return None -def _resolve_compiler_path(env, key, default): - """Extract the real compiler from the environment and resolve it to an absolute path.""" +def _resolve_command(env, key, default_command, action_root): + """Select and absolutize an explicit or toolchain-provided command.""" current = env.get(key) - if not current: - return default - parts = shlex.split(current) - if not parts: - return default - compiler = parts[0] - if os.path.isabs(compiler): - return compiler - return os.path.abspath(compiler) - - -def _make_compiler_wrapper(tmpdir, name, compiler_path, sysroot=None): - wrapper = path.join(tmpdir, ".aspect_rules_py_compilers", name) + command = shlex.split(current) if current else list(default_command) + if not command: + raise ValueError("{} command is empty".format(key)) + + executable = command[0] + if not path.isabs(executable): + if path.dirname(executable): + command[0] = path.join(action_root, executable) + else: + resolved = shutil.which(executable, path=env["PATH"]) + if resolved is None: + raise FileNotFoundError( + "{} executable not found on PATH: {}".format(key, executable), + ) + if not path.isabs(resolved): + resolved = path.join(action_root, resolved) + command[0] = resolved + return command + + +def _visible_compiler_command(command, sysroot): + command = [argument for argument in command if argument != _DEBUG_FLAG] + has_sysroot = any( + argument in ("-isysroot", "--sysroot") + or argument.startswith("-isysroot") + or argument.startswith("--sysroot=") + for argument in command[1:] + ) + if sysroot and not has_sysroot: + command[1:1] = ["-isysroot", sysroot] + return shlex.join(command) + + +def _compiler_wrapper_source(command, sysroot): + return _COMPILER_WRAPPER.format( + compiler=command[0], + debug_flag=_DEBUG_FLAG, + driver_args=command[1:], + sysroot=sysroot, + ) + + +def _write_tool_wrapper(tmpdir, name, source): + wrapper_dir = path.join(tmpdir, ".aspect_rules_py_compilers") + wrapper = path.join(wrapper_dir, name) makedirs(path.dirname(wrapper), exist_ok=True) with open(wrapper, "w") as f: - f.write(_COMPILER_WRAPPER.format( - debug_flag=_DEBUG_FLAG, - compiler_path=compiler_path, - name=name, - sysroot=sysroot, - )) + f.write(source) chmod(wrapper, 0o755) return wrapper -def _override_tool(env, key, wrapper): - current = env.get(key) - if not current: - return - parts = shlex.split(current) - if parts: - parts[0] = wrapper - env[key] = shlex.join(parts) - - -def _compiler_env(tmpdir): +def _native_tool_env(tmpdir, native_tool_config, action_root): env = dict(os.environ) env["PATH"] = pathsep.join([ path.dirname(sys.executable), @@ -108,33 +141,65 @@ def _compiler_env(tmpdir): env["TEMP"] = tmpdir env["TEMPDIR"] = tmpdir - cc_path = _resolve_compiler_path(env, "CC", "cc") - cxx_path = _resolve_compiler_path(env, "CXX", "c++") - sysroot = _darwin_sysroot() - - cc = _make_compiler_wrapper(tmpdir, "cc", cc_path, sysroot) - cxx = _make_compiler_wrapper(tmpdir, "c++", cxx_path, sysroot) - - env.setdefault("CC", cc) - env.setdefault("CXX", cxx) - - # MPI builds (e.g. mpi4py) consult $MPICC before searching PATH, so a - # plain C compiler here would shadow the real mpicc. Only set it when - # a system mpicc exists, wrapped to keep the debug-flag stripping. - mpicc_path = shutil.which("mpicc", path=env["PATH"]) - if mpicc_path: - env.setdefault("MPICC", _make_compiler_wrapper(tmpdir, "mpicc", mpicc_path, sysroot)) - env.setdefault("AR", "ar") - - for key, wrapper in [ - ("CC", cc), - ("CXX", cxx), - ("CPP", cc), - ("LDSHARED", cc), - ("LDCXXSHARED", cxx), + for key, name in [ + ("CC", "cc"), + ("CXX", "c++"), ]: - _override_tool(env, key, wrapper) + explicit = bool(env.get(key)) + configured = native_tool_config.get(key, [_DEFAULT_COMPILERS[key]]) + if isinstance(configured, dict) and not explicit: + env[key] = _write_tool_wrapper( + tmpdir, + name, + _ERROR_WRAPPER.format(error=configured["error"]), + ) + continue + default_command = ( + [_DEFAULT_COMPILERS[key]] if isinstance(configured, dict) else configured + ) + command = _resolve_command( + env, + key, + default_command, + action_root, + ) + if explicit: + env[key] = _visible_compiler_command(command, sysroot) + else: + env[key] = _write_tool_wrapper( + tmpdir, + name, + _compiler_wrapper_source(command, sysroot), + ) + + for key in ("AR", "LD", "STRIP", "CPP", "LDSHARED", "LDCXXSHARED"): + default_command = native_tool_config.get(key, []) + if not env.get(key) and not default_command: + continue + command = _resolve_command( + env, + key, + default_command, + action_root, + ) + env[key] = ( + _visible_compiler_command(command, sysroot) + if key in ("CPP", "LDSHARED", "LDCXXSHARED") + else shlex.join(command) + ) + + if env.get("MPICC"): + command = _resolve_command(env, "MPICC", [], action_root) + env["MPICC"] = _visible_compiler_command(command, sysroot) + else: + mpicc = shutil.which("mpicc", path=env["PATH"]) + if mpicc: + env["MPICC"] = _write_tool_wrapper( + tmpdir, + "mpicc", + _compiler_wrapper_source([mpicc], sysroot), + ) return env @@ -187,11 +252,14 @@ def _legacy_metadata_conflicts_with_pyproject(worktree): PARSER = ArgumentParser() PARSER.add_argument("srcarchive") PARSER.add_argument("outdir") +PARSER.add_argument("--native-tool-config") PARSER.add_argument("--validate-anyarch", action="store_true") PARSER.add_argument("--patch-strip", type=int, default=0, help="Strip count for patch (-p)") PARSER.add_argument("--patch", action="append", default=[], dest="patches", help="Patch file to apply (repeatable)") opts, args = PARSER.parse_known_args() +action_root = path.abspath(os.curdir) +native_tool_config = json.loads(opts.native_tool_config) if opts.native_tool_config else {} tmp_root = path.abspath(opts.outdir) + ".tmp" # Sandboxed/remote actions get a fresh root each run, so we don't expect a stale tmp_root to exist. makedirs(tmp_root, exist_ok=False) @@ -230,11 +298,9 @@ def _legacy_metadata_conflicts_with_pyproject(worktree): # Get a path to the outdir which will be valid after we cd outdir = path.abspath(opts.outdir) -# Preserve PATH so native sdist builds can find compilers (clang, gcc), -# and re-point CC/CXX/etc. through wrapper scripts in tmp_root so the -# Bazel-supplied workspace-relative compiler paths survive the cwd -# change into the worktree. -build_env = _compiler_env(tmp_root) +# Preserve configured compiler and linker commands through the cwd change into +# the worktree. +build_env = _native_tool_env(tmp_root, native_tool_config, action_root) if _legacy_metadata_conflicts_with_pyproject(t): print( diff --git a/uv/private/pep517_whl/build_helper_native_tools_test.py b/uv/private/pep517_whl/build_helper_native_tools_test.py new file mode 100644 index 000000000..e49b05e21 --- /dev/null +++ b/uv/private/pep517_whl/build_helper_native_tools_test.py @@ -0,0 +1,218 @@ +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys +import tarfile +import tempfile +import textwrap +import unittest + + +_SETUP_PY = textwrap.dedent( + """\ + import os + from pathlib import Path + import shlex + import subprocess + + from setuptools import Extension, setup + + for key in os.environ.get( + "NATIVE_TOOLS_TO_PROBE", + "AR,CC,CXX,LD,STRIP", + ).split(","): + command = shlex.split(os.environ[key]) + if not Path(command[0]).is_absolute(): + raise SystemExit(f"{key} executable is not absolute: {command[0]}") + subprocess.run([*command, "--probe"], check=True) + + ext_modules = ( + [Extension("native_module", ["native.c"])] + if os.environ.get("BUILD_C_EXTENSION") == "1" + else [] + ) + setup( + name="native_tools_test", + version="1.0", + py_modules=[], + ext_modules=ext_modules, + ) + """ +) + +_TOOL = """#!/bin/sh +exec {python} - "$@" <<'PY' +import os +from pathlib import Path +import shlex +import shutil +import sys + +arguments = sys.argv[1:] +with open(os.environ["NATIVE_TOOL_LOG"], "a") as log: + print({name} + ":" + shlex.join(arguments), file=log) +if "--probe" in arguments: + raise SystemExit(0) +if os.environ.get("NATIVE_TOOL_FAKE_OUTPUT") == "1": + try: + output = Path(arguments[arguments.index("-o") + 1]) + except (ValueError, IndexError): + raise SystemExit("synthetic compiler invocation has no -o output") + output.parent.mkdir(parents=True, exist_ok=True) + # macOS wheel tagging inspects extension Mach-O headers. Copy the declared + # test interpreter so compile and link outputs remain synthetic but valid. + shutil.copyfile(sys.executable, output) + raise SystemExit(0) +raise SystemExit("unexpected non-probe invocation") +PY +""" + + +def _write_tool(path, name): + path.write_text(_TOOL.format(python=shlex.quote(sys.executable), name=repr(name))) + path.chmod(0o755) + return path + + +class BuildHelperNativeToolsTest(unittest.TestCase): + def test_tools_remain_absolute_after_build_cwd_change(self): + with tempfile.TemporaryDirectory(dir=os.environ.get("TEST_TMPDIR")) as tmp: + action_root = Path(tmp) / "action-root" + tools = action_root / "tools" + relative_bin = action_root / "relative-bin" + tools.mkdir(parents=True) + relative_bin.mkdir() + + tool_paths = {} + for name in ("AR", "CC", "CXX", "STRIP"): + tool_paths[name] = _write_tool(tools / name.lower(), name) + tool_paths["LD"] = _write_tool(relative_bin / "ld-driver", "LD") + + package = action_root / "native_tools_test-1.0" + package.mkdir() + (package / "setup.py").write_text(_SETUP_PY) + (package / "native.c").write_text("int native_module(void) { return 0; }\n") + sdist = action_root / "native_tools_test-1.0.tar.gz" + with tarfile.open(sdist, "w:gz") as archive: + archive.add(package, arcname=package.name) + + outdir = action_root / "dist" + log = action_root / "native-tools.log" + helper = Path(__file__).with_name("build_helper.py") + rules_py_root = helper.parents[3] + + env = os.environ.copy() + for key in ( + "AR", + "CC", + "CPP", + "CXX", + "LD", + "LDCXXSHARED", + "LDSHARED", + "MPICC", + "STRIP", + ): + env.pop(key, None) + env.update( + HOME=tmp, + LD="ld-driver --ld-default", + NATIVE_TOOL_LOG=str(log), + PATH=os.pathsep.join(["relative-bin", env.get("PATH", os.defpath)]), + PYTHONPATH=os.pathsep.join( + [str(rules_py_root), env.get("PYTHONPATH", "")] + ), + ) + + config = { + "AR": [str(tool_paths["AR"].relative_to(action_root)), "--ar-default"], + "CC": [str(tool_paths["CC"].relative_to(action_root)), "--cc-default"], + "CXX": [ + str(tool_paths["CXX"].relative_to(action_root)), + "--cxx-default", + ], + "STRIP": [str(tool_paths["STRIP"]), "--strip-default"], + } + result = subprocess.run( + [ + sys.executable, + str(helper), + "--native-tool-config", + json.dumps(config), + str(sdist), + str(outdir), + ], + cwd=action_root, + env=env, + capture_output=True, + text=True, + ) + self.assertEqual( + 0, + result.returncode, + "build_helper failed:\nstdout:\n{}\nstderr:\n{}".format( + result.stdout, + result.stderr, + ), + ) + records = { + key: args + for key, args in ( + line.split(":", 1) for line in log.read_text().splitlines() + ) + } + self.assertEqual({"AR", "CC", "CXX", "LD", "STRIP"}, set(records)) + for key in records: + self.assertIn("--{}-default".format(key.lower()), records[key]) + self.assertIn("--probe", records[key]) + + c_only_outdir = action_root / "c-only-dist" + c_only_log = action_root / "c-only-native-tools.log" + cxx_error = "configured GCC CXX driver is unavailable" + c_only_config = { + "CC": [str(tool_paths["CC"].relative_to(action_root))], + "CXX": {"error": cxx_error}, + } + c_only_result = subprocess.run( + [ + sys.executable, + str(helper), + "--native-tool-config", + json.dumps(c_only_config), + str(sdist), + str(c_only_outdir), + ], + cwd=action_root, + env={ + **env, + "BUILD_C_EXTENSION": "1", + "NATIVE_TOOL_LOG": str(c_only_log), + "NATIVE_TOOL_FAKE_OUTPUT": "1", + "NATIVE_TOOLS_TO_PROBE": "CC", + }, + capture_output=True, + text=True, + ) + self.assertEqual(0, c_only_result.returncode, c_only_result.stderr) + self.assertEqual(1, len(list(c_only_outdir.glob("*.whl")))) + c_only_records = c_only_log.read_text().splitlines() + self.assertTrue(c_only_records) + self.assertTrue(all(record.startswith("CC:") for record in c_only_records)) + self.assertTrue(any("--probe" not in record for record in c_only_records)) + + cxx_wrapper = Path(str(c_only_outdir) + ".tmp") / ( + ".aspect_rules_py_compilers/c++" + ) + cxx_result = subprocess.run( + [cxx_wrapper, "--probe"], + capture_output=True, + text=True, + ) + self.assertNotEqual(0, cxx_result.returncode) + self.assertIn(cxx_error, cxx_result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv/private/pep517_whl/c_driver_test.sh b/uv/private/pep517_whl/c_driver_test.sh new file mode 100755 index 000000000..039e4d006 --- /dev/null +++ b/uv/private/pep517_whl/c_driver_test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/uv/private/pep517_whl/cxx_driver_test.sh b/uv/private/pep517_whl/cxx_driver_test.sh new file mode 100755 index 000000000..039e4d006 --- /dev/null +++ b/uv/private/pep517_whl/cxx_driver_test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/uv/private/pep517_whl/ld_driver_test.sh b/uv/private/pep517_whl/ld_driver_test.sh new file mode 100755 index 000000000..039e4d006 --- /dev/null +++ b/uv/private/pep517_whl/ld_driver_test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/uv/private/pep517_whl/override_test/BUILD.bazel b/uv/private/pep517_whl/override_test/BUILD.bazel new file mode 100644 index 000000000..2a0de6780 --- /dev/null +++ b/uv/private/pep517_whl/override_test/BUILD.bazel @@ -0,0 +1,30 @@ +load("//uv/private/pep517_whl:rule.bzl", "pep517_native_whl") +load("//uv/private/pep517_whl:test.bzl", "pep517_native_whl_toolchain_env_test") + +pep517_native_whl( + name = "__toolchain_env_override_fixture", + src = "//uv/private/pep517_whl:__stub_sdist", + env = { + "CC": "ccache clang --target=aarch64-linux-gnu", + "LDCXXSHARED": "clang++ -shared", + "JAVA_HOME": "$(JAVABASE)", + "JAVA": "$(JAVA)", + "JAR": "$(JAVABASE)/bin/jar", + }, + tags = ["manual"], + tool = "//uv/private/pep517_whl:__stub_tool", + toolchains = [ + "@bazel_tools//tools/cpp:current_cc_toolchain", + "@bazel_tools//tools/jdk:current_java_runtime", + ], + version = "0.0.1", +) + +pep517_native_whl_toolchain_env_test( + name = "toolchain_env_override_test", + expected_env = { + "CC": "ccache clang --target=aarch64-linux-gnu", + "LDCXXSHARED": "clang++ -shared", + }, + target_under_test = ":__toolchain_env_override_fixture", +) diff --git a/uv/private/pep517_whl/rule.bzl b/uv/private/pep517_whl/rule.bzl index 1fb57149e..35face12e 100644 --- a/uv/private/pep517_whl/rule.bzl +++ b/uv/private/pep517_whl/rule.bzl @@ -6,9 +6,17 @@ build backend the sdist declares in its `[build-system]` table. """ load("@bazel_lib//lib:resource_sets.bzl", "resource_set", "resource_set_attr") -load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") +load( + "@rules_cc//cc:action_names.bzl", + "CPP_COMPILE_ACTION_NAME", + "CPP_LINK_STATIC_LIBRARY_ACTION_NAME", + "C_COMPILE_ACTION_NAME", + "STRIP_ACTION_NAME", +) load("//py/private/toolchain:types.bzl", "NATIVE_BUILD_TOOLCHAIN", "PY_TOOLCHAIN") +_CC_TOOLCHAIN_TYPE = "@bazel_tools//tools/cpp:toolchain_type" + def _common_env(ctx): return { "SETUPTOOLS_SCM_PRETEND_VERSION": ctx.attr.version, @@ -61,43 +69,108 @@ def _collect_toolchain_inputs_and_vars(ctx): known_variables.update(target[platform_common.TemplateVariableInfo].variables) return extra_inputs, known_variables -_BAZEL_CC_WRAPPER_BASENAMES = ["gcc", "g++", "clang", "clang++"] +def _tool_is_input(tool_path, inputs): + return any([ + tool_path == f.path or tool_path.startswith(f.path + "/") + for f in inputs + ]) -def _cc_toolchain_inputs_and_compiler(ctx): - """Return (depset of CcToolchainInfo files, compiler_file_path or None). +def _cc_toolchain_inputs_and_tools(ctx, overrides): + """Return the target exec group's C++ inputs and native build tools.""" + toolchain = ctx.exec_groups["target"].toolchains[_CC_TOOLCHAIN_TYPE] + cc_toolchain = toolchain.cc if hasattr(toolchain, "cc_provider_in_toolchain") else toolchain + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features, + ) + cc_files = cc_toolchain.all_files + cc_inputs = cc_files.to_list() - Uses find_cpp_toolchain so the lookup works for both direct cc_toolchain - targets and alias wrappers such as current_cc_toolchain, which Bazel 9 no - longer exposes CcToolchainInfo on directly through the alias target. - """ - cc_toolchain = find_cpp_toolchain(ctx) - if not cc_toolchain or not hasattr(cc_toolchain, "all_files"): - return None, None - files = cc_toolchain.all_files - compiler_file = None - if hasattr(cc_toolchain, "compiler_executable"): - compiler_basename = cc_toolchain.compiler_executable.split("/")[-1] - for f in files.to_list(): - if f.basename == compiler_basename: - compiler_file = f - break - if not compiler_file: - for f in files.to_list(): - if f.basename in _BAZEL_CC_WRAPPER_BASENAMES: - compiler_file = f - break - if not compiler_file: - for f in files.to_list(): - if (f.basename.startswith("clang-") or f.basename.startswith("gcc-") or - f.basename.startswith("g++-")): - compiler_file = f - break - compiler_path = compiler_file.path if compiler_file else None - return files, compiler_path + # Query action configs rather than CcToolchainInfo's legacy executable + # fields. rules_cc fabricates paths when tool_paths are omitted: + # https://github.com/bazelbuild/rules_cc/blob/0.2.16/cc/private/rules_impl/cc_toolchain_provider_helper.bzl#L77-L101 + # + # These tools run outside Bazel's configured C++ actions, so they must be + # present in all_files to be available in the wheel-build action. + # Do not reuse the configured action argv or environment here. Those are + # parameterized for one Bazel compile action, while PEP 517 backends reuse + # CC and CXX for their own compile and link commands. Package env remains + # the explicit interface for additional driver flags. + compile_tools = {} + tool_config = {} + for key, action_name in { + "CC": C_COMPILE_ACTION_NAME, + "CXX": CPP_COMPILE_ACTION_NAME, + }.items(): + if not cc_common.action_is_enabled( + action_name = action_name, + feature_configuration = feature_configuration, + ): + if not overrides.get(key): + message = "C++ toolchain does not enable the {} action required for {}".format(action_name, key) + if key == "CXX": + tool_config[key] = {"error": message} + else: + fail(message) + continue + tool = cc_common.get_tool_for_action( + action_name = action_name, + feature_configuration = feature_configuration, + ) + compile_tools[key] = tool + if not _tool_is_input(tool, cc_inputs): + if not overrides.get(key): + message = "C++ toolchain {} tool is absent from all_files: {}".format(key, tool) + if key == "CXX": + tool_config[key] = {"error": message} + else: + fail(message) + continue + if not overrides.get(key): + tool_config[key] = [tool] + + for key, action_name in { + "AR": CPP_LINK_STATIC_LIBRARY_ACTION_NAME, + "STRIP": STRIP_ACTION_NAME, + }.items(): + if overrides.get(key): + continue + if not cc_common.action_is_enabled( + action_name = action_name, + feature_configuration = feature_configuration, + ): + continue + tool = cc_common.get_tool_for_action( + action_name = action_name, + feature_configuration = feature_configuration, + ) + if _tool_is_input(tool, cc_inputs): + tool_config[key] = [tool] + + # LD has no single C++ action equivalent: executable and shared-library + # links may select different drivers. Preserve only an explicit override. + cc = compile_tools.get("CC") + cxx = compile_tools.get("CXX") + if not overrides.get("CXX") and cc != None and cc == cxx: + if cc_toolchain.compiler == "clang": + # Clang documents this as equivalent to invoking clang++: + # https://clang.llvm.org/docs/UsersManual.html#cmdoption-driver-mode + tool_config["CXX"] = [cxx, "--driver-mode=g++"] + else: + # Clang is the only shared driver with a documented generic C++ + # mode. GCC requires g++ for C++ mode and libstdc++ linkage, and + # custom drivers likewise need an explicit CXX command: + # https://gcc.gnu.org/onlinedocs/gcc/Invoking-G_002b_002b.html + tool_config["CXX"] = { + "error": "C++ toolchain '{}' uses '{}' for both CC and CXX; set an explicit CXX override".format(cc_toolchain.compiler, cxx), + } + return cc_files, tool_config def _pep517_whl(ctx): archive = ctx.file.src - wheel_dir = ctx.actions.declare_directory("whl") + wheel_dir = ctx.actions.declare_directory(ctx.label.name) patch_args, patch_inputs = _patch_args_and_inputs(ctx) # The build tool is a py_binary wrapping build_helper.py. Using it as @@ -126,22 +199,21 @@ def _pep517_whl(ctx): def _pep517_native_whl(ctx): archive = ctx.file.src - wheel_dir = ctx.actions.declare_directory("whl") + wheel_dir = ctx.actions.declare_directory(ctx.label.name) patch_args, patch_inputs = _patch_args_and_inputs(ctx) env = _common_env(ctx) extra_inputs, known_variables = _collect_toolchain_inputs_and_vars(ctx) - cc_files, cc_compiler = _cc_toolchain_inputs_and_compiler(ctx) - if cc_files: - extra_inputs.append(cc_files) - + # Package overrides belong in this rule's env attribute. Do not let an + # ambient shell compiler replace the configured target toolchain. + for key in ["AR", "CC", "CPP", "CXX", "LD", "LDCXXSHARED", "LDSHARED", "MPICC", "STRIP"]: + env.pop(key, None) for k, v in ctx.attr.env.items(): env[k] = ctx.expand_make_variables("env", v, known_variables) - if cc_compiler: - env["CC"] = cc_compiler - env["CXX"] = cc_compiler + cc_files, native_tool_config = _cc_toolchain_inputs_and_tools(ctx, env) + extra_inputs.append(cc_files) ctx.actions.run( mnemonic = "PySdistNativeBuild", @@ -149,6 +221,8 @@ def _pep517_native_whl(ctx): executable = ctx.executable.tool, toolchain = None, arguments = ctx.attr.args + patch_args + [ + "--native-tool-config", + json.encode(native_tool_config), archive.path, wheel_dir.path, ], @@ -179,7 +253,10 @@ _PATCH_ATTRS = { _pep517_whl_attrs = { "src": attr.label(allow_single_file = True), - "tool": attr.label(executable = True, cfg = "exec"), + # The build actions use the target execution group, so their frontend must + # be built for the same execution platform: + # https://bazel.build/extending/exec-groups#defining-execution-groups + "tool": attr.label(executable = True, cfg = config.exec("target")), "version": attr.string(), "args": attr.string_list(default = ["--validate-anyarch"]), } | _PATCH_ATTRS | resource_set_attr @@ -230,6 +307,7 @@ constraints of the target platform. "`TemplateVariableInfo`).", ), }, + fragments = ["cpp"], toolchains = [ "@bazel_tools//tools/cpp:toolchain_type", ], diff --git a/uv/private/pep517_whl/strip_driver_test.sh b/uv/private/pep517_whl/strip_driver_test.sh new file mode 100755 index 000000000..039e4d006 --- /dev/null +++ b/uv/private/pep517_whl/strip_driver_test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +exit 0 diff --git a/uv/private/pep517_whl/test.bzl b/uv/private/pep517_whl/test.bzl index 9f9b758cb..01206b09a 100644 --- a/uv/private/pep517_whl/test.bzl +++ b/uv/private/pep517_whl/test.bzl @@ -1,21 +1,177 @@ -"""Analysis tests: pep517_native_whl forwards toolchain make-variables -into the PySdistNativeBuild action env. +"""Analysis tests for the pep517_native_whl toolchain boundary. Inspecting the action at analysis time avoids actually running a PEP 517 -build to verify the env wiring. +build to verify the command and environment wiring. """ load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") - -# Env vars sourced from the CC toolchain's TemplateVariableInfo. SYSROOT is -# omitted because some toolchains legitimately have an empty sysroot and -# it isn't exposed as a TemplateVariableInfo make variable today. -_CC_ENV_KEYS = ["CC", "CXX", "AR", "LD", "STRIP"] +load( + "@rules_cc//cc:action_names.bzl", + "CPP_COMPILE_ACTION_NAME", + "CPP_LINK_EXECUTABLE_ACTION_NAME", + "CPP_LINK_STATIC_LIBRARY_ACTION_NAME", + "C_COMPILE_ACTION_NAME", + "STRIP_ACTION_NAME", +) +load( + "@rules_cc//cc:cc_toolchain_config_lib.bzl", + "action_config", + "env_entry", + "env_set", + "feature", + "flag_group", + "flag_set", + "tool", +) +load("@rules_cc//cc:defs.bzl", "cc_toolchain") # Env vars sourced from the Java runtime toolchain's TemplateVariableInfo. _JDK_ENV_KEYS = ["JAVA_HOME", "JAVA", "JAR"] -_REQUIRED_ENV_KEYS = _CC_ENV_KEYS + _JDK_ENV_KEYS +_NATIVE_TOOL_ENV_KEYS = ["AR", "CC", "CXX", "LD", "STRIP"] +_REQUIRED_NATIVE_TOOL_KEYS = ["CC", "CXX"] +_UNSET_DEFAULT_ENV_KEYS = _NATIVE_TOOL_ENV_KEYS + [ + "CPP", + "LDCXXSHARED", + "LDSHARED", + "MPICC", +] + +def _compiler_selection_toolchain_config_impl(ctx): + action_tools = { + CPP_LINK_EXECUTABLE_ACTION_NAME: ctx.executable.ld_driver.basename, + } + if "CC" in ctx.attr.compiler_actions: + action_tools[C_COMPILE_ACTION_NAME] = ctx.executable.c_driver.basename + if "CXX" in ctx.attr.compiler_actions: + action_tools[CPP_COMPILE_ACTION_NAME] = ctx.executable.cxx_driver.basename + if "AR" in ctx.attr.optional_tools: + action_tools[CPP_LINK_STATIC_LIBRARY_ACTION_NAME] = ctx.executable.ar_driver.basename + if "STRIP" in ctx.attr.optional_tools: + action_tools[STRIP_ACTION_NAME] = ctx.executable.strip_driver.basename + features = [feature( + name = "sentinel_action_environment", + enabled = True, + env_sets = [env_set( + actions = [C_COMPILE_ACTION_NAME, CPP_COMPILE_ACTION_NAME], + env_entries = [env_entry(key = "SENTINEL_ACTION_ENV", value = "configured")], + )], + )] + if ctx.attr.no_legacy_features: + # rules_cc synthesizes action configs for omitted actions unless this + # compatibility feature is present: + # https://github.com/bazelbuild/rules_cc/blob/0.2.16/cc/private/toolchain_config/cc_toolchain_config_info.bzl#L76-L121 + features.append(feature(name = "no_legacy_features", enabled = True)) + return cc_common.create_cc_toolchain_config_info( + ctx = ctx, + abi_libc_version = "unknown", + abi_version = "unknown", + action_configs = [ + action_config( + action_name = action_name, + enabled = True, + flag_sets = [flag_set( + flag_groups = [flag_group(flags = ["--sentinel-action-flag=" + action_name])], + )], + tools = [tool(path = action_tools[action_name])], + ) + for action_name in action_tools + ], + compiler = ctx.attr.compiler, + features = features, + host_system_name = "local", + target_cpu = "test", + target_libc = "unknown", + target_system_name = "local", + # Deliberately action-only: rules_cc fabricates legacy executable + # fields for omitted tool_paths, so tests must exercise action configs. + tool_paths = [], + toolchain_identifier = ctx.label.name, + ) + +compiler_selection_toolchain_config = rule( + implementation = _compiler_selection_toolchain_config_impl, + attrs = { + "ar_driver": attr.label( + cfg = "exec", + allow_files = True, + executable = True, + mandatory = True, + ), + "c_driver": attr.label( + cfg = "exec", + allow_files = True, + executable = True, + mandatory = True, + ), + "cxx_driver": attr.label( + cfg = "exec", + allow_files = True, + executable = True, + mandatory = True, + ), + "compiler": attr.string(mandatory = True), + "compiler_actions": attr.string_list(mandatory = True), + "ld_driver": attr.label( + cfg = "exec", + allow_files = True, + executable = True, + mandatory = True, + ), + "no_legacy_features": attr.bool(mandatory = True), + "optional_tools": attr.string_list(mandatory = True), + "strip_driver": attr.label( + cfg = "exec", + allow_files = True, + executable = True, + mandatory = True, + ), + }, + provides = [CcToolchainConfigInfo], +) + +def compiler_selection_toolchain(name, ar_driver, c_driver, cxx_driver, compiler, compiler_actions, ld_driver, no_legacy_features, optional_tools, strip_driver): + """Declare a synthetic C++ toolchain for compiler-selection tests.""" + files = name + "_files" + config = name + "_config" + native.filegroup( + name = files, + srcs = depset([ar_driver, c_driver, cxx_driver, ld_driver, strip_driver]).to_list(), + tags = ["manual"], + ) + compiler_selection_toolchain_config( + name = config, + ar_driver = ar_driver, + c_driver = c_driver, + compiler = compiler, + compiler_actions = compiler_actions, + cxx_driver = cxx_driver, + ld_driver = ld_driver, + no_legacy_features = no_legacy_features, + optional_tools = optional_tools, + strip_driver = strip_driver, + tags = ["manual"], + ) + cc_toolchain( + name = name, + all_files = files, + ar_files = files, + as_files = files, + compiler_files = files, + dwp_files = files, + linker_files = files, + objcopy_files = files, + strip_files = files, + supports_param_files = 0, + tags = ["manual"], + toolchain_config = config, + ) + native.toolchain( + name = name + "_toolchain", + tags = ["manual"], + toolchain = name, + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", + ) def _toolchain_env_test_impl(ctx): env = analysistest.begin(ctx) @@ -29,38 +185,162 @@ def _toolchain_env_test_impl(ctx): "expected exactly one PySdistNativeBuild action", ) - action_env = build_actions[0].env - missing = [k for k in _REQUIRED_ENV_KEYS if k not in action_env] - asserts.equals( + action = build_actions[0] + action_env = action.env + asserts.false( env, - [], - missing, - "missing env keys on action; got: {}".format(sorted(action_env.keys())), + "SENTINEL_ACTION_ENV" in action_env, + "configured compile environment must not enter the wheel-build action", ) - for key in _REQUIRED_ENV_KEYS: + if ctx.attr.check_toolchain_env: + missing = [k for k in _JDK_ENV_KEYS if k not in action_env] + asserts.equals( + env, + [], + missing, + "missing env keys on action; got: {}".format(sorted(action_env.keys())), + ) + for key in _JDK_ENV_KEYS: + asserts.true( + env, + action_env.get(key), + "${} should resolve to a non-empty value".format(key), + ) + for key in _UNSET_DEFAULT_ENV_KEYS: + asserts.false( + env, + key in action_env, + "{} should not be baked into the action environment".format(key), + ) + + config_arg_indexes = [ + i + for i in range(len(action.argv)) + if action.argv[i] == "--native-tool-config" + ] + asserts.equals(env, 1, len(config_arg_indexes)) + native_tool_config_json = action.argv[config_arg_indexes[0] + 1] + native_tool_config = json.decode(native_tool_config_json) + missing_tools = [ + key + for key in _REQUIRED_NATIVE_TOOL_KEYS + if key not in ctx.attr.expected_env and key not in native_tool_config + ] + asserts.equals(env, [], missing_tools) + inputs = action.inputs.to_list() + expected_config = { + key: [compiler] + (ctx.attr.expected_cxx_args if key == "CXX" else []) + for key, compiler in ctx.attr.expected_tools.items() + } + expected_config.update({ + key: {"error": error} + for key, error in ctx.attr.expected_tool_errors.items() + }) + if expected_config: + asserts.equals(env, sorted(expected_config), sorted(native_tool_config)) + asserts.equals(env, expected_config, native_tool_config) + + # Bazel's action flags and environment describe one compile invocation, + # but PEP 517 backends reuse CC and CXX for compile and link commands. + asserts.false(env, "sentinel-action-flag" in native_tool_config_json) + asserts.false(env, "SENTINEL_ACTION_ENV" in native_tool_config_json) + for command in native_tool_config.values(): + if type(command) != "list": + continue + + # Per https://bazel.build/rules/lib/builtins/File#is_directory: + # + # File.is_directory reflects the type the file was declared as, not + # its type on the filesystem. Toolchain inputs may contain a source + # directory instead of each tool, so test path ancestry. asserts.true( env, - action_env.get(key), - "${} should resolve to a non-empty toolchain path".format(key), + any([ + command[0] == f.path or + command[0].startswith(f.path + "/") + for f in inputs + ]), + "configured tool must be covered by an action input: {}".format(command[0]), ) - # CC and CXX both derive from cc_toolchain's $(CC) make variable today. - # If we ever switch CXX to a c++ compile driver (e.g. via a custom - # TemplateVariableInfo shim), this assertion can be relaxed. - asserts.equals( - env, - action_env.get("CC"), - action_env.get("CXX"), - "CC and CXX should point at the same compiler driver", - ) + for key, expected in ctx.attr.expected_env.items(): + asserts.equals( + env, + expected, + action_env.get(key), + "explicit {} should replace the toolchain default".format(key), + ) - # JAR is constructed from $(JAVABASE)/bin/jar — sanity-check the suffix. - asserts.true( - env, - action_env.get("JAR", "").endswith("/bin/jar"), - "JAR should resolve under JAVA_HOME/bin/jar; got {}".format(action_env.get("JAR")), - ) + if ctx.attr.check_toolchain_env: + # JAR is constructed from $(JAVABASE)/bin/jar — sanity-check the suffix. + asserts.true( + env, + action_env.get("JAR", "").endswith("/bin/jar"), + "JAR should resolve under JAVA_HOME/bin/jar; got {}".format(action_env.get("JAR")), + ) return analysistest.end(env) -pep517_native_whl_toolchain_env_test = analysistest.make(_toolchain_env_test_impl) +def _toolchain_env_test_attrs(): + return { + "check_toolchain_env": attr.bool(), + "expected_cxx_args": attr.string_list(), + "expected_env": attr.string_dict(), + "expected_tool_errors": attr.string_dict(), + "expected_tools": attr.string_dict(), + } + +pep517_native_whl_toolchain_env_test = analysistest.make( + _toolchain_env_test_impl, + attrs = _toolchain_env_test_attrs(), +) + +pep517_native_whl_compiler_selection_test = analysistest.make( + _toolchain_env_test_impl, + attrs = _toolchain_env_test_attrs(), + config_settings = { + "//command_line_option:extra_toolchains": [ + "//uv/private/pep517_whl:__compiler_selection_toolchain", + ], + }, +) + +pep517_native_whl_same_driver_clang_test = analysistest.make( + _toolchain_env_test_impl, + attrs = _toolchain_env_test_attrs(), + config_settings = { + "//command_line_option:extra_toolchains": [ + "//uv/private/pep517_whl:__same_driver_clang_toolchain", + ], + }, +) + +pep517_native_whl_same_driver_gcc_test = analysistest.make( + _toolchain_env_test_impl, + attrs = _toolchain_env_test_attrs(), + config_settings = { + "//command_line_option:extra_toolchains": [ + "//uv/private/pep517_whl:__same_driver_gcc_toolchain", + ], + }, +) + +pep517_native_whl_missing_cxx_test = analysistest.make( + _toolchain_env_test_impl, + attrs = _toolchain_env_test_attrs(), + config_settings = { + "//command_line_option:extra_toolchains": [ + "//uv/private/pep517_whl:__missing_cxx_toolchain", + ], + }, +) + +pep517_native_whl_disabled_cxx_test = analysistest.make( + _toolchain_env_test_impl, + attrs = _toolchain_env_test_attrs(), + config_settings = { + "//command_line_option:extra_toolchains": [ + "//uv/private/pep517_whl:__disabled_cxx_toolchain", + ], + }, +) diff --git a/uv/private/sdist_build/repository.bzl b/uv/private/sdist_build/repository.bzl index 8ceb77b3c..7ad94be36 100644 --- a/uv/private/sdist_build/repository.bzl +++ b/uv/private/sdist_build/repository.bzl @@ -219,31 +219,19 @@ def _sdist_build_impl(repository_ctx): strip = repository_ctx.attr.pre_build_patch_strip, ) - # For native builds, emit a baked-in CC toolchain + CC/CXX/AR/LD/STRIP - # env block. Targets in `toolchains` expose `TemplateVariableInfo`; - # the env values below are make-variable references resolved at - # action analysis time. + # For native builds, emit a C++ toolchain target for its action inputs and + # make variables. pep517_native_whl derives CC/CXX and available AR/STRIP + # defaults from the selected CcToolchainInfo; extra_env contains package + # overrides and variables supplied by extra_toolchains. # - # CXX defaults to $(CC) because most clang/gcc-based toolchains use - # a single driver binary for both languages, and meson-python / - # cmake-based backends look for CXX independently of CC. - # - # `extra_toolchains` and `extra_env` augment (do not replace) the - # defaults — set via `uv.override_package(toolchains = [...], - # env = {...})` to layer JDK / Rust / etc. plumbing on top. + # `extra_toolchains` extend the defaults. `extra_env` is merged over + # them so a package can replace a named compiler or tool variable. toolchain_attrs = "" if is_native: toolchains = [ "@bazel_tools//tools/cpp:current_cc_toolchain", ] + list(repository_ctx.attr.extra_toolchains) - env = { - "AR": "$(AR)", - "CC": "$(CC)", - "CXX": "$(CC)", - "LD": "$(LD)", - "STRIP": "$(STRIP)", - } - env.update(repository_ctx.attr.extra_env) + env = dict(repository_ctx.attr.extra_env) toolchain_attrs = """ toolchains = [ {toolchains} @@ -327,7 +315,7 @@ sdist_build = repository_rule( ), "extra_env": attr.string_dict( default = {}, - doc = "Environment variables merged into the default CC env dict in the generated pep517_native_whl(...) `env` dict. Values may reference $(VAR) make-variables from any toolchain. Set via `uv.override_package(env = {...})`.", + doc = "Native-build environment overrides emitted in the generated pep517_native_whl(...) `env` dict. Values may reference $(VAR) make-variables from any toolchain. Set via `uv.override_package(env = {...})`.", ), }, )