diff --git a/annotations.toml b/annotations.toml index 8755e1bae..eeb41c76f 100644 --- a/annotations.toml +++ b/annotations.toml @@ -1,12 +1,13 @@ ## Annotations.toml # # This is an aspect_rules_py//uv specific configuration file which allows -# packages to be annotated with their build time dependencies. This fills a gap in -# both the pylock and uv lockfile formats, neither of which allow for build -# requirements to be specified. +# packages to be annotated extra information such as entrypoint. # -# For instance if a package requires cython be available, this is how you can -# configure it to be delivered. +# This file allows specifying additional build time dependencies, however +# +# should be preferred, which is loaded by `uv.project`. For instance if a +# package requires setuptools be available, this is how you can configure# +# it to be delivered. # # Takes the place of giant MODULE.bazel annotation tables. # @@ -17,15 +18,15 @@ # We version lockfiles and support semver semantics here version = "0.0.0" -# Bravado doesn't have bdists, need to build it. Mark explicitly that we need -# wheel setuptools and build in order to do so; build being the standard build -# tool driver. -[[package]] -name = "bravado-core" -build-dependencies = [ - { name = "build" }, - { name = "setuptools" }, -] +# This format is supported, but +# +# is now recommended. +#[[package]] +#name = "bravado-core" +#build-dependencies = [ +# { name = "build" }, +# { name = "setuptools" }, +#] # We can also annotate packages as providing console scripts we care about. # Declared console scripts will have Bazel targets generated for them. diff --git a/e2e/cases/BUILD.bazel b/e2e/cases/BUILD.bazel index cb7c5ce98..16e2d5193 100644 --- a/e2e/cases/BUILD.bazel +++ b/e2e/cases/BUILD.bazel @@ -116,16 +116,36 @@ write_source_files( # rule expands env keys correctly given an explicit fixture. "snapshots/sdist_build.uv_sdist_jdk_build.jpype1.BUILD.bazel": "@sdist_build__uv_sdist_jdk_build__jpype1__1_7_1//:BUILD.bazel", - # @sdist_build for cowsay with native annotation and monitor_memory - # ------------------------------------------------------------------ + # @sdist_build for cowsay with composed annotations and monitor_memory + # --------------------------------------------------------------------- # cowsay is pure Python, so only `native = true` can select # pep517_native_whl. Its build and native annotations are split so the # snapshot also pins that a native-only annotation preserves explicit - # and configure-discovered build deps. This retains the monitor-only - # source-build coverage. The runtime //uv-sdist-fallback:test + # and configure-discovered build deps, while legacy build deps compose + # with distinctly marker-qualified uv-native deps. This retains the + # monitor-only source-build coverage. The runtime //uv-sdist-fallback:test # proves that the forced-native build produces a working wheel. "snapshots/sdist_build.uv_sdist_fallback.cowsay.BUILD.bazel": "@sdist_build__uv_sdist_fallback__cowsay__6_0//:BUILD.bazel", + # @sdist_build for tqdm with tool.uv.extra-build-dependencies + # ----------------------------------------------------------- + # Pins the uv-native annotation and match-runtime paths independently + # of cowsay's composed legacy and uv-native annotations. + "snapshots/sdist_build.uv_sdist_fallback.tqdm.BUILD.bazel": "@sdist_build__uv_sdist_fallback__tqdm__4_52_0//:BUILD.bazel", + + # @sdist_build for both conflicting versions of packaging + # --------------------------------------------------------- + # Pins that a package-name keyed uv-native annotation applies to every + # locked version, not only packages with a single default version. + "snapshots/sdist_build.uv_conflict_817.packaging_21_3.BUILD.bazel": ( + "@sdist_build__ambig__packaging__21_3" + + "//:BUILD.bazel" + ), + "snapshots/sdist_build.uv_conflict_817.packaging_24_0.BUILD.bazel": ( + "@sdist_build__ambig__packaging__24_0" + + "//:BUILD.bazel" + ), + # @sdist_build for python-geohash with uv.override_package(resource_set) # ---------------------------------------------------------------------- # Covers the override_package -> repo-rule -> BUILD-template plumbing diff --git a/e2e/cases/snapshots/sdist_build.uv_conflict_817.packaging_21_3.BUILD.bazel b/e2e/cases/snapshots/sdist_build.uv_conflict_817.packaging_21_3.BUILD.bazel new file mode 100644 index 000000000..0ab8d7c50 --- /dev/null +++ b/e2e/cases/snapshots/sdist_build.uv_conflict_817.packaging_21_3.BUILD.bazel @@ -0,0 +1,23 @@ + +load("@aspect_rules_py//uv/private/pep517_whl:rule.bzl", "pep517_whl") +load("@aspect_rules_py//py:defs.bzl", "py_binary") + +py_binary( + name = "build_tool", + main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", + srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], + deps = ["@@aspect_rules_py++uv+project__ambig//:pyparsing"], +) + +pep517_whl( + name = "whl", + src = "@@aspect_rules_py++uv+sdist__packaging__dd47c42927d89ab9//file:file", + tool = ":build_tool", + version = "21.3", + visibility = ["//visibility:public"], +) + +exports_files( + ["BUILD.bazel"], + visibility = ["//visibility:public"], +) diff --git a/e2e/cases/snapshots/sdist_build.uv_conflict_817.packaging_24_0.BUILD.bazel b/e2e/cases/snapshots/sdist_build.uv_conflict_817.packaging_24_0.BUILD.bazel new file mode 100644 index 000000000..3889d59f0 --- /dev/null +++ b/e2e/cases/snapshots/sdist_build.uv_conflict_817.packaging_24_0.BUILD.bazel @@ -0,0 +1,23 @@ + +load("@aspect_rules_py//uv/private/pep517_whl:rule.bzl", "pep517_whl") +load("@aspect_rules_py//py:defs.bzl", "py_binary") + +py_binary( + name = "build_tool", + main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", + srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], + deps = ["@@aspect_rules_py++uv+project__ambig//:pyparsing"], +) + +pep517_whl( + name = "whl", + src = "@@aspect_rules_py++uv+sdist__packaging__eb82c5e3e5620907//file:file", + tool = ":build_tool", + version = "24.0", + visibility = ["//visibility:public"], +) + +exports_files( + ["BUILD.bazel"], + visibility = ["//visibility:public"], +) diff --git a/e2e/cases/snapshots/sdist_build.uv_sdist_fallback.cowsay.BUILD.bazel b/e2e/cases/snapshots/sdist_build.uv_sdist_fallback.cowsay.BUILD.bazel index 41bc0ae23..07b09b02a 100644 --- a/e2e/cases/snapshots/sdist_build.uv_sdist_fallback.cowsay.BUILD.bazel +++ b/e2e/cases/snapshots/sdist_build.uv_sdist_fallback.cowsay.BUILD.bazel @@ -1,12 +1,29 @@ load("@aspect_rules_py//uv/private/pep517_whl:rule.bzl", "pep517_native_whl") load("@aspect_rules_py//py:defs.bzl", "py_binary") +load("@aspect_rules_py//uv/private/markers:defs.bzl", "decide_marker") + +decide_marker( + name = "_build_dep_marker_0", + marker = "(sys_platform != 'win32') or (sys_platform != 'imaginary')", +) + +decide_marker( + name = "_build_dep_marker_1", + marker = "sys_platform != 'emscripten'", +) py_binary( name = "build_tool", main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], - deps = ["@aspect_rules_py//uv/private/pep517_whl:memory_monitor", "@@aspect_rules_py++uv+project__uv_sdist_fallback//:packaging", "@@aspect_rules_py++uv+project__uv_sdist_fallback//:build", "@@aspect_rules_py++uv+project__uv_sdist_fallback//:setuptools", "@whl_install__uv_sdist_fallback__setuptools__80_10_2//:install"], + deps = ["@aspect_rules_py//uv/private/pep517_whl:memory_monitor", "@@aspect_rules_py++uv+project__uv_sdist_fallback//:packaging", "@@aspect_rules_py++uv+project__uv_sdist_fallback//:build", "@@aspect_rules_py++uv+project__uv_sdist_fallback//:setuptools", "@whl_install__uv_sdist_fallback__setuptools__80_10_2//:install"] + select({ + ":_build_dep_marker_0": ["@@aspect_rules_py++uv+project__uv_sdist_fallback//:colorama"], + "//conditions:default": [], + }) + select({ + ":_build_dep_marker_1": ["@@aspect_rules_py++uv+project__uv_sdist_fallback//:pyproject_hooks"], + "//conditions:default": [], + }), ) pep517_native_whl( diff --git a/e2e/cases/snapshots/sdist_build.uv_sdist_fallback.tqdm.BUILD.bazel b/e2e/cases/snapshots/sdist_build.uv_sdist_fallback.tqdm.BUILD.bazel new file mode 100644 index 000000000..87dcb79a4 --- /dev/null +++ b/e2e/cases/snapshots/sdist_build.uv_sdist_fallback.tqdm.BUILD.bazel @@ -0,0 +1,32 @@ + +load("@aspect_rules_py//uv/private/pep517_whl:rule.bzl", "pep517_whl") +load("@aspect_rules_py//py:defs.bzl", "py_binary") +load("@aspect_rules_py//uv/private/markers:defs.bzl", "decide_marker") + +decide_marker( + name = "_build_dep_marker_0", + marker = "(sys_platform != 'win32') or (sys_platform != 'imaginary')", +) + +py_binary( + name = "build_tool", + main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", + srcs = ["@aspect_rules_py//uv/private/pep517_whl:build_helper.py"], + deps = ["@@aspect_rules_py++uv+project__uv_sdist_fallback//:urllib3", "@@aspect_rules_py++uv+project__uv_sdist_fallback//:six", "@@aspect_rules_py++uv+project__uv_sdist_fallback//:build", "@@aspect_rules_py++uv+project__uv_sdist_fallback//:setuptools", "@whl_install__uv_sdist_fallback__setuptools__80_10_2//:install"] + select({ + ":_build_dep_marker_0": ["@@aspect_rules_py++uv+project__uv_sdist_fallback//:packaging"], + "//conditions:default": [], + }), +) + +pep517_whl( + name = "whl", + src = "@@aspect_rules_py++uv+sdist__tqdm__18d6a615aedd09ec//file:file", + tool = ":build_tool", + version = "4.52.0", + visibility = ["//visibility:public"], +) + +exports_files( + ["BUILD.bazel"], + visibility = ["//visibility:public"], +) diff --git a/e2e/cases/uv-conflict-817/pyproject.toml b/e2e/cases/uv-conflict-817/pyproject.toml index 31f439d9d..8a662d6c4 100644 --- a/e2e/cases/uv-conflict-817/pyproject.toml +++ b/e2e/cases/uv-conflict-817/pyproject.toml @@ -20,3 +20,6 @@ conflicts = [ { group = "ambig-b" }, ], ] + +[tool.uv.extra-build-dependencies] +packaging = ["pyparsing"] diff --git a/e2e/cases/uv-conflict-817/setup.MODULE.bazel b/e2e/cases/uv-conflict-817/setup.MODULE.bazel index f8c7f6191..a943a2b4e 100644 --- a/e2e/cases/uv-conflict-817/setup.MODULE.bazel +++ b/e2e/cases/uv-conflict-817/setup.MODULE.bazel @@ -5,4 +5,9 @@ uv.project( lock = "//uv-conflict-817:uv.lock", pyproject = "//uv-conflict-817:pyproject.toml", ) -use_repo(uv, "pypi_uv_conflict_817") +use_repo( + uv, + "pypi_uv_conflict_817", + "sdist_build__ambig__packaging__21_3", + "sdist_build__ambig__packaging__24_0", +) diff --git a/e2e/cases/uv-git-source/BUILD.bazel b/e2e/cases/uv-git-source/BUILD.bazel index 730176460..842658dd3 100644 --- a/e2e/cases/uv-git-source/BUILD.bazel +++ b/e2e/cases/uv-git-source/BUILD.bazel @@ -4,7 +4,8 @@ load("@aspect_rules_py//py:defs.bzl", "py_test") # `source = { git = "https://github.com/benjaminp/six?tag=1.17.0#" }`, # exercising parse_git_url on a real uv-generated URL, the GitHub # git-to-http_archive conversion in collect_sdists, and the source build of -# the resulting archive. +# the resulting archive. cowsay is forced to build from source using that +# locked git dependency as an extra build requirement. py_test( name = "test", srcs = ["__test__.py"], @@ -12,6 +13,7 @@ py_test( main = "__test__.py", python_version = "3.11", deps = [ + "@pypi_git_source//cowsay", "@pypi_git_source//six", ], ) diff --git a/e2e/cases/uv-git-source/__test__.py b/e2e/cases/uv-git-source/__test__.py index 143317941..b48854a8e 100644 --- a/e2e/cases/uv-git-source/__test__.py +++ b/e2e/cases/uv-git-source/__test__.py @@ -1,6 +1,8 @@ +import cowsay import six +assert "git build dependency" in cowsay.get_output_string("cow", "git build dependency") assert six.__version__ == "1.17.0", six.__version__ assert six.ensure_str(b"git-source") == "git-source" -print("six", six.__version__, "imported from git source") +print("six", six.__version__, "and cowsay imported from source builds") diff --git a/e2e/cases/uv-git-source/pyproject.toml b/e2e/cases/uv-git-source/pyproject.toml index b1c84a3e5..a7d130e04 100644 --- a/e2e/cases/uv-git-source/pyproject.toml +++ b/e2e/cases/uv-git-source/pyproject.toml @@ -8,6 +8,7 @@ dependencies = [ "build", "setuptools", "six", + "cowsay==6.0", ] # six is resolved from git rather than a registry, exercising the @@ -16,3 +17,9 @@ dependencies = [ # git-to-http_archive conversion for GitHub remotes. [tool.uv.sources] six = { git = "https://github.com/benjaminp/six", tag = "1.17.0" } + +[tool.uv] +no-binary-package = ["cowsay"] + +[tool.uv.extra-build-dependencies] +cowsay = ["six @ git+https://github.com/benjaminp/six@1.17.0"] diff --git a/e2e/cases/uv-git-source/uv.lock b/e2e/cases/uv-git-source/uv.lock index 33fe27704..dbe5ff879 100644 --- a/e2e/cases/uv-git-source/uv.lock +++ b/e2e/cases/uv-git-source/uv.lock @@ -25,6 +25,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "cowsay" +version = "6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/b5/e8e802ddc7f5219417dc7d7953eec81ffe48ad129b793f3040361e4aff89/cowsay-6.0.tar.gz", hash = "sha256:47445cb273684618a1786db8e8d05ec9258455f7eb74893e5d0933daafeb44ba", size = 25228, upload-time = "2023-09-08T17:16:36.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/56/7922bfc226ccd44221befb6b866aaa4da4170bc9d8b036ef0675ce0f1b53/cowsay-6.0-py2.py3-none-any.whl", hash = "sha256:77b07c508af48aa300a90f3b3c5c013a12360a71fc5c87b1efb763fe2803a775", size = 25411, upload-time = "2023-09-08T17:16:34.44Z" }, + { url = "https://files.pythonhosted.org/packages/c7/01/40be54532d7bd37c99d849bbdd590f0135ce13183365df6f0e85c475ca71/cowsay-6.0-py3-none-any.whl", hash = "sha256:011c067841451ea49baf8ff49c355bd7f659d53fc538459e0a84c12ae2b4e027", size = 25409, upload-time = "2023-09-08T17:25:17.505Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -63,6 +73,7 @@ version = "0.0.0" source = { virtual = "." } dependencies = [ { name = "build" }, + { name = "cowsay" }, { name = "setuptools" }, { name = "six" }, ] @@ -70,6 +81,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "build" }, + { name = "cowsay", specifier = "==6.0" }, { name = "setuptools" }, { name = "six", git = "https://github.com/benjaminp/six?tag=1.17.0" }, ] diff --git a/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/BUILD.bazel b/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/BUILD.bazel new file mode 100644 index 000000000..7e22602cf --- /dev/null +++ b/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/BUILD.bazel @@ -0,0 +1 @@ +# Fixture data is consumed by this module's extension. diff --git a/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/MODULE.bazel b/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/MODULE.bazel new file mode 100644 index 000000000..c55455c86 --- /dev/null +++ b/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/MODULE.bazel @@ -0,0 +1,17 @@ +module(name = "missing_extra_build_dep_fixture") + +bazel_dep(name = "aspect_rules_py") +local_path_override( + module_name = "aspect_rules_py", + path = "../../../..", +) + +uv = use_extension("@aspect_rules_py//uv:extensions.bzl", "uv") +uv.declare_hub(hub_name = "missing_extra_build_dep") +uv.project( + default_build_dependencies = [], + hub_name = "missing_extra_build_dep", + lock = "//:uv.lock", + pyproject = "//:pyproject.toml", +) +use_repo(uv, "missing_extra_build_dep") diff --git a/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/pyproject.toml b/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/pyproject.toml new file mode 100644 index 000000000..11654e496 --- /dev/null +++ b/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "missing-extra-build-dep" +version = "0.0.0" +requires-python = ">=3.11" +dependencies = ["cchardet==2.1.7"] + +[tool.uv.extra-build-dependencies] +cchardet = ["cython"] diff --git a/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/uv.lock b/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/uv.lock new file mode 100644 index 000000000..347cea686 --- /dev/null +++ b/e2e/cases/uv-invalid-build-overrides/missing-extra-build-dep/uv.lock @@ -0,0 +1,20 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "cchardet" +version = "2.1.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/5d/090c9f0312b7988a9433246c9cf0b566b1ae1374368cfb8ac897218a4f65/cchardet-2.1.7.tar.gz", hash = "sha256:c428b6336545053c2589f6caf24ea32276c6664cb86db817e03a94c60afa0eaf", size = 653617, upload-time = "2020-10-27T08:36:15.999Z" } + +[[package]] +name = "missing-extra-build-dep" +version = "0.0.0" +source = { virtual = "." } +dependencies = [ + { name = "cchardet" }, +] + +[package.metadata] +requires-dist = [{ name = "cchardet", specifier = "==2.1.7" }] diff --git a/e2e/cases/uv-invalid-build-overrides/test.sh b/e2e/cases/uv-invalid-build-overrides/test.sh index 6e3122beb..30ec26b38 100755 --- a/e2e/cases/uv-invalid-build-overrides/test.sh +++ b/e2e/cases/uv-invalid-build-overrides/test.sh @@ -16,7 +16,8 @@ fail() { expect_failure() { local fixture="$1" local target="$2" - local expected="$3" + local expected + shift 2 if ( cd "$case_dir/$fixture" && @@ -24,16 +25,19 @@ expect_failure() { ) >"$output_log" 2>&1; then fail "$fixture $target unexpectedly succeeded" fi - if ! grep -Fq "$expected" "$output_log"; then - cat "$output_log" >&2 - fail "$fixture $target did not report: $expected" - fi + for expected in "$@"; do + if ! grep -Fq "$expected" "$output_log"; then + cat "$output_log" >&2 + fail "$fixture $target did not report: $expected" + fi + done } expect_failure \ wheel-only \ '@invalid_overrides//:*' \ - 'build-only attributes require a source distribution, but the lock record has only wheels: console_scripts, resource_set, env, monitor_memory, pre_build_patches, pre_build_patch_strip, toolchains' + 'build-only attributes require a source distribution, but the lock record has only wheels:' \ + 'console_scripts, resource_set, env, monitor_memory, pre_build_patches, pre_build_patch_strip, toolchains' expect_failure \ editable-self \ '@invalid_editable_overrides//:*' \ @@ -46,11 +50,18 @@ expect_failure \ unmatched-lock \ '@unmatched_lock_override//:*' \ 'has no uv.project() for that lock' +expect_failure \ + missing-extra-build-dep \ + '@missing_extra_build_dep//:*' \ + 'Unable to resolve extra build dependency `"cython"` for package "cchardet" in @@//:pyproject.toml.' \ + '`uv.lock` does not include packages referenced only by `tool.uv.extra-build-dependencies`' \ + 'Add the dependency as a dependency and regenerate the lock.' expect_failure \ custom-complete \ '@sdist_build__uv_invalid_build_overrides_custom__cowsay__6_0//:*' \ - 'complete `build_file_content`, which bypasses the generated `pep517_*whl(...)` call, so these attributes cannot be applied: resource_set, env, monitor_memory, toolchains' + 'complete `build_file_content`, which bypasses the generated `pep517_*whl(...)` call,' \ + 'so these attributes cannot be applied: resource_set, env, monitor_memory, toolchains' expect_failure \ custom-pure \ '@sdist_build__uv_invalid_build_overrides_custom__cowsay__6_0//:*' \ diff --git a/e2e/cases/uv-sdist-fallback/BUILD.bazel b/e2e/cases/uv-sdist-fallback/BUILD.bazel index bc13737ad..f7ab790f7 100644 --- a/e2e/cases/uv-sdist-fallback/BUILD.bazel +++ b/e2e/cases/uv-sdist-fallback/BUILD.bazel @@ -2,9 +2,9 @@ load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_pex_bin load("@bazel_skylib//rules:build_test.bzl", "build_test") # Regression coverage for the sdist-fallback behavior preserved by the -# "always keep sdist fallbacks when sources exist" fix. cowsay 6.0 has both -# an sdist and a `-none-any.whl`; the pyproject's `no-binary-package` setting -# forces the resolver to build it from source, exercising the sbuild path at +# "always keep sdist fallbacks when sources exist" fix. cowsay and tqdm both +# have an sdist and a `-none-any.whl`; the pyproject's `no-binary-package` +# setting forces both source builds, exercising the two annotation paths at # runtime. py_test( name = "test", @@ -14,6 +14,8 @@ py_test( python_version = "3.11", deps = [ "@pypi_sdist_fallback//cowsay", + "@pypi_sdist_fallback//tqdm", + "@pypi_sdist_fallback//urllib3", ], ) @@ -26,7 +28,11 @@ py_binary( dep_group = "uv_sdist_fallback", main = "__test__.py", python_version = "3.11", - deps = ["@pypi_sdist_fallback//cowsay"], + deps = [ + "@pypi_sdist_fallback//cowsay", + "@pypi_sdist_fallback//tqdm", + "@pypi_sdist_fallback//urllib3", + ], ) py_image_layer( diff --git a/e2e/cases/uv-sdist-fallback/__test__.py b/e2e/cases/uv-sdist-fallback/__test__.py index 84741de10..e06ce9385 100644 --- a/e2e/cases/uv-sdist-fallback/__test__.py +++ b/e2e/cases/uv-sdist-fallback/__test__.py @@ -1,9 +1,8 @@ -"""Smoke test that cowsay built from sdist (via no-binary-package) works. +"""Smoke test that cowsay and tqdm built from sdist work. -cowsay 6.0 ships both an sdist and a `-none-any.whl`. With `no-binary-package -= ["cowsay"]` the resolver must produce a `sdist_build__*` repo so the -package can be built from source. Importing and exercising cowsay verifies -the sdist build pathway end-to-end. +cowsay 6.0 and tqdm 4.52.0 both ship an sdist and a `-none-any.whl`. With +`no-binary-package = ["cowsay", "tqdm"]` the resolver must produce +`sdist_build__*` repos so both packages can be built from source. """ import os @@ -12,9 +11,13 @@ from pathlib import Path import cowsay +import socks +import tqdm output = cowsay.get_output_string("cow", "sdist fallback works!") assert "sdist fallback works!" in output +assert list(tqdm.tqdm(range(2), disable=True)) == [0, 1] +assert hasattr(socks, "socksocket"), "urllib3[socks] build extra was not activated" marker = "declared console script works" wrapper = shutil.which("cowsay") assert wrapper is not None, "cowsay wrapper is absent from PATH" @@ -30,4 +33,4 @@ text=True, ) assert marker in result.stdout -print("cowsay imported from sdist: OK") +print("cowsay and tqdm imported from sdist: OK") diff --git a/e2e/cases/uv-sdist-fallback/pyproject.toml b/e2e/cases/uv-sdist-fallback/pyproject.toml index 9122b6e43..c183e07bd 100644 --- a/e2e/cases/uv-sdist-fallback/pyproject.toml +++ b/e2e/cases/uv-sdist-fallback/pyproject.toml @@ -10,11 +10,40 @@ dependencies = [ "build", "setuptools", "cowsay==6.0", + "tqdm==4.52.0", + # Keep the urllib3 requirement plain; its non-empty extra is build-only. + "urllib3==2.6.3", + # Keep a dedicated locked package for the direct-reference build-dep case. + "six==1.17.0", ] -# cowsay 6.0 ships both an sdist and a `py3-none-any.whl`. Forcing it to -# build from source via `no-binary-package` exercises the source-build -# pathway end-to-end for a package whose lockfile entry has a matching -# anyarch wheel. +[project.optional-dependencies] +# Ensure the lock contains the extra's transitive dependencies while the +# default runtime requirement remains plain. +build-extra-lock = ["urllib3[socks]==2.6.3"] + +# cowsay and tqdm both ship an sdist and a `py3-none-any.whl`. Forcing them to +# build from source exercises both annotation styles end-to-end. [tool.uv] -no-binary-package = ["cowsay"] +no-binary-package = ["cowsay", "tqdm"] + +[tool.uv.extra-build-dependencies] +# `packaging` is intentionally absent from `default_build_dependencies`, so +# the tqdm snapshot catches a broken uv-native annotation path. +cowsay = [ + # Keep two conditional dependencies under distinct markers so composing + # cowsay's legacy annotation does not accidentally discard a marker branch. + "colorama; sys_platform != 'win32'", + "colorama; sys_platform != 'imaginary'", + "pyproject-hooks; sys_platform != 'emscripten'", +] +tqdm = [ + # `socks` is non-empty and must reach the source-build environment even + # though the runtime dependency is plain urllib3. Keep the base + # unconditional and the extra conditional to catch duplicate labels. + "urllib3==2.6.3", + "urllib3[socks]==2.6.3; sys_platform != 'win32'", + "packaging; sys_platform != 'win32'", + "packaging; sys_platform != 'imaginary'", + { requirement = "six @ https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", match-runtime = true }, +] diff --git a/e2e/cases/uv-sdist-fallback/setup.MODULE.bazel b/e2e/cases/uv-sdist-fallback/setup.MODULE.bazel index e2cc1dc5d..5351dca67 100644 --- a/e2e/cases/uv-sdist-fallback/setup.MODULE.bazel +++ b/e2e/cases/uv-sdist-fallback/setup.MODULE.bazel @@ -10,7 +10,8 @@ uv.declare_hub(hub_name = "pypi_sdist_fallback") uv.project( default_build_dependencies = [ "build", - "setuptools", + # Exercise direct-reference resolution for default build dependencies. + "setuptools @ https://files.pythonhosted.org/packages/94/b8/f1f62a5e3c0ad2ff1d189590bfa4c46b4f3b6e49cef6f26c6ee4e575394d/setuptools-80.10.2-py3-none-any.whl", ], hub_name = "pypi_sdist_fallback", lock = "//uv-sdist-fallback:uv.lock", @@ -34,4 +35,5 @@ use_repo( uv, "pypi_sdist_fallback", "sdist_build__uv_sdist_fallback__cowsay__6_0", + "sdist_build__uv_sdist_fallback__tqdm__4_52_0", ) diff --git a/e2e/cases/uv-sdist-fallback/uv.lock b/e2e/cases/uv-sdist-fallback/uv.lock index 6fc4d4468..f3f3d30ee 100644 --- a/e2e/cases/uv-sdist-fallback/uv.lock +++ b/e2e/cases/uv-sdist-fallback/uv.lock @@ -1,6 +1,10 @@ version = 1 revision = 3 requires-python = ">=3.11" +resolution-markers = [ + "python_full_version >= '3.13'", + "python_full_version < '3.13'", +] [[package]] name = "build" @@ -53,6 +57,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, ] +[[package]] +name = "pysocks" +version = "1.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/11/293dd436aea955d45fc4e8a35b6ae7270f5b8e00b53cf6c024c83b657a11/PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0", size = 284429, upload-time = "2019-09-20T02:07:35.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" }, +] + [[package]] name = "setuptools" version = "80.10.2" @@ -62,6 +75,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/94/b8/f1f62a5e3c0ad2ff1d189590bfa4c46b4f3b6e49cef6f26c6ee4e575394d/setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173", size = 1064234, upload-time = "2026-01-25T22:38:15.216Z" }, ] +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "tqdm" +version = "4.52.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/30/8c8015735a54e13444a3d4982a7a9538bde27f8b3bd35203f9e920f0d78c/tqdm-4.52.0.tar.gz", hash = "sha256:18d6a615aedd09ec8456d9524489dab330af4bd5c2a14a76eb3f9a0e14471afe", size = 180391, upload-time = "2020-11-16T14:19:13.746Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/7e/bc84ebdb47cfc1f643d026570cb24dd40c2b7a29e167ea101f1702717658/tqdm-4.52.0-py2.py3-none-any.whl", hash = "sha256:80d9d5165d678dbd027dd102dfb99f71bf05f333b61fb761dbba13b4ab719ead", size = 71328, upload-time = "2020-11-16T14:19:11.069Z" }, +] + +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +] + +[package.optional-dependencies] +socks = [ + { name = "pysocks" }, +] + [[package]] name = "uv-sdist-fallback" version = "0.0.0" @@ -70,6 +115,14 @@ dependencies = [ { name = "build" }, { name = "cowsay" }, { name = "setuptools" }, + { name = "six" }, + { name = "tqdm" }, + { name = "urllib3" }, +] + +[package.optional-dependencies] +build-extra-lock = [ + { name = "urllib3", extra = ["socks"] }, ] [package.metadata] @@ -77,4 +130,9 @@ requires-dist = [ { name = "build" }, { name = "cowsay", specifier = "==6.0" }, { name = "setuptools" }, + { name = "six", specifier = "==1.17.0" }, + { name = "tqdm", specifier = "==4.52.0" }, + { name = "urllib3", specifier = "==2.6.3" }, + { name = "urllib3", extras = ["socks"], marker = "extra == 'build-extra-lock'", specifier = "==2.6.3" }, ] +provides-extras = ["build-extra-lock"] diff --git a/pyproject.toml b/pyproject.toml index 93ed160e0..041dcff0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,3 +61,12 @@ extra-paths = ["py/private/pytest_shard"] [tool.ty.rules] missing-type-argument = "error" possibly-unresolved-reference = "warn" + +# bravado-core 6.1.1. doesn't have a wheel, need to build it. Mark explicitly +# that we need setuptools and build in order to do so; build being the +# standard build tool driver. +[tool.uv.extra-build-dependencies] +bravado-core = [ + "build", + "setuptools", +] diff --git a/uv/private/extension/defs.bzl b/uv/private/extension/defs.bzl index 3c8c9fd9d..e37e2a19b 100644 --- a/uv/private/extension/defs.bzl +++ b/uv/private/extension/defs.bzl @@ -67,6 +67,7 @@ load("//uv/private/tomltool:toml.bzl", "toml") load("//uv/private/uv_hub:repository.bzl", "uv_hub") load("//uv/private/uv_project:repository.bzl", "uv_project") load("//uv/private/whl_install:repository.bzl", "parse_console_script", "whl_install") +load(":git_utils.bzl", "locked_git_requirement_urls") load(":graph_utils.bzl", "activate_extras", "collect_sccs") load(":lockfile.bzl", "build_marker_graph", "collect_bdists", "collect_configurations", "collect_sdists", "normalize_deps", "url_basename") load(":projectfile.bzl", "collate_versions_by_name", "collect_activated_extras", "extract_requirement_marker_pairs") @@ -230,6 +231,7 @@ def _parse_projects(module_ctx, hub_specs): for project in mod.tags.project: project_data = toml.decode_file(module_ctx, project.pyproject) + tool_uv = project_data.get("tool", {}).get("uv", {}) lock_data = toml.decode_file(module_ctx, project.lock) # The stamp derives from the project name: stable across reloads and @@ -245,44 +247,107 @@ def _parse_projects(module_ctx, hub_specs): no_binary_packages = { normalize_name(p): True - for p in project_data.get("tool", {}).get("uv", {}).get("no-binary-package", []) + for p in tool_uv.get("no-binary-package", []) } default_versions, package_versions, lock_data = normalize_deps(project_id, lock_data) - def _resolve(package): - name = normalize_name(package["name"]) - if "version" in package: - return (project_id, name, package["version"], "__base__") + locked_urls = {} + for locked_package in lock_data.get("package", []): + dependency = (project_id, locked_package["name"], locked_package["version"], "__base__") + artifacts = locked_package.get("wheels", []) + [ + locked_package.get("sdist", {}), + locked_package.get("source", {}), + ] + for artifact in artifacts: + url = artifact.get("url") + if url: + locked_urls[(locked_package["name"], url)] = dependency + git = locked_package.get("source", {}).get("git") + if git: + for url in locked_git_requirement_urls(git): + locked_urls[(locked_package["name"], url)] = dependency + + def _resolve(name, version): + name = normalize_name(name) + if version: + return (project_id, name, version, "__base__") elif name in default_versions: return default_versions[name] return None lock_build_dep_anns = {} + lock_conditional_build_dep_anns = {} lock_native_anns = {} + extra_build_dependencies = tool_uv.get("extra-build-dependencies", {}) + for package, extra_deps in extra_build_dependencies.items(): + package_name = normalize_name(package) + targets = [ + (project_id, package_name, version, "__base__") + for version in package_versions.get(package_name, {}) + ] + if not targets: + # Allow a shared annotation file to include entries for other locks. + continue + deps = [] + conditional_deps = {} + for dep in extra_deps: + # TODO(konsti): We currently ignore match-runtime. Since we're already + # using locked dependencies for building, this works as long as there is + # only a single version of the package. + if type(dep) == "dict": + dep = dep["requirement"] + resolved_deps = extract_requirement_marker_pairs( + project.lock, + project_id, + dep, + {}, + package_versions, + locked_urls = locked_urls, + fail_if_missing = False, + ) + if not resolved_deps: + fail(( + "Unable to resolve extra build dependency `{}` for package {} in {}. " + + "`uv.lock` does not include packages referenced only by " + + "`tool.uv.extra-build-dependencies`. Add the dependency as a dependency " + + "and regenerate the lock." + ).format(repr(dep), repr(package), project.pyproject)) + for resolved, marker in resolved_deps: + if marker: + conditional_deps.setdefault(marker, []).append(resolved) + else: + deps.append(resolved) + for target in targets: + lock_build_dep_anns[target] = deps + lock_conditional_build_dep_anns[target] = conditional_deps + + uv_build_dep_anns = dict(lock_build_dep_anns) for ann in mod.tags.unstable_annotate_packages: if ann.lock == project.lock: annotations = toml.decode_file(module_ctx, ann.src) for package in annotations.get("package", []): - k = _resolve(package) - if k == None: + target = _resolve(package["name"], package.get("version")) + if target == None: # Allow a shared annotation file to include entries for other locks. continue if "native" in package: if type(package["native"]) != "bool": fail("Annotation `native` for package {} in {} must be a boolean, got {}".format(package["name"], ann.src, repr(package["native"]))) - lock_native_anns[k] = package["native"] + lock_native_anns[target] = package["native"] if "build-dependencies" in package: deps = [] skip = False for dep in package["build-dependencies"]: - resolved = _resolve(dep) + resolved = _resolve(dep["name"], dep.get("version")) if resolved == None: skip = True break deps.append(resolved) if not skip: - lock_build_dep_anns[k] = deps + # Legacy and uv-native annotations compose, including + # any marker-qualified uv-native dependencies. + lock_build_dep_anns[target] = uv_build_dep_anns.get(target, []) + deps package_overrides = {} package_console_scripts = {} @@ -356,7 +421,23 @@ def _parse_projects(module_ctx, hub_specs): whl_configurations.update(collect_configurations(lock_data)) - configuration_names, activated_extras = collect_activated_extras(project.lock, project_id, project_data, lock_data, default_versions, marker_graph, package_versions) + # Activate extras requested by build requirements using each + # dependency group's selected versions so their transitive + # dependencies reach source-build tools. + build_extra_roots = {} + for deps in lock_build_dep_anns.values(): + for dep_project, dep_name, dep_version, extra in deps: + if extra == "__base__": + continue + build_extra_roots.setdefault((dep_project, dep_name, dep_version, extra), {}).update({"": 1}) + for conditional_deps in lock_conditional_build_dep_anns.values(): + for marker, deps in conditional_deps.items(): + for dep_project, dep_name, dep_version, extra in deps: + if extra == "__base__": + continue + build_extra_roots.setdefault((dep_project, dep_name, dep_version, extra), {}).update({marker: 1}) + + configuration_names, activated_extras = collect_activated_extras(project.lock, project_id, project_data, lock_data, default_versions, marker_graph, package_versions, build_extra_roots, locked_urls = locked_urls) version_activations = collate_versions_by_name(activated_extras) # SCC graph shapes: @@ -486,6 +567,7 @@ def _parse_projects(module_ctx, hub_specs): # could do pyproject.toml introspection. ann_key = (project_id, normalize_name(package["name"]), package["version"], "__base__") build_deps = lock_build_dep_anns.get(ann_key) or [] + conditional_build_deps = lock_conditional_build_dep_anns.get(ann_key) or {} is_native = "auto" if ann_key in lock_native_anns: is_native = "true" if lock_native_anns[ann_key] else "false" @@ -504,11 +586,37 @@ def _parse_projects(module_ctx, hub_specs): lock_build_deps = [ it[0] for req in project.default_build_dependencies - for it in extract_requirement_marker_pairs(project.lock, project_id, req, default_versions, package_versions, fail_if_missing = sbuild_required) + for it in extract_requirement_marker_pairs( + project.lock, + project_id, + req, + default_versions, + package_versions, + locked_urls = locked_urls, + fail_if_missing = sbuild_required, + ) ] build_deps = sets.to_list(sets.make(build_deps + lock_build_deps)) + # A base requirement and its extras resolve through the same + # project package label, so deduplicate after rendering labels. + sbuild_deps = sets.to_list(sets.make([ + "@{0}//:{1}".format(*dep) + for dep in build_deps + ])) + sbuild_conditional_deps = {} + for marker, deps in conditional_build_deps.items(): + for dep in deps: + label = "@{0}//:{1}".format(*dep) + if label in sbuild_deps: + continue + previous = sbuild_conditional_deps.get(label) + if previous: + sbuild_conditional_deps[label] = "({}) or ({})".format(previous, marker) + else: + sbuild_conditional_deps[label] = marker + pre_build_patches = [] pre_build_patch_strip = 0 if pkg_override and pkg_override.pre_build_patches: @@ -530,7 +638,8 @@ def _parse_projects(module_ctx, hub_specs): sbuild_specs[sbuild_id] = struct( src = sdist, - deps = ["@{0}//:{1}".format(*it) for it in build_deps], + deps = sbuild_deps, + conditional_deps = sbuild_conditional_deps, is_native = is_native, version = package["version"], pre_build_patches = pre_build_patches, @@ -745,6 +854,8 @@ def _uv_impl(module_ctx): if sbuild_cfg.available_deps: sbuild_kwargs["available_deps"] = sbuild_cfg.available_deps + if sbuild_cfg.conditional_deps: + sbuild_kwargs["conditional_deps"] = sbuild_cfg.conditional_deps if sbuild_cfg.pre_build_patches: sbuild_kwargs["pre_build_patches"] = sbuild_cfg.pre_build_patches sbuild_kwargs["pre_build_patch_strip"] = sbuild_cfg.pre_build_patch_strip diff --git a/uv/private/extension/git_utils.bzl b/uv/private/extension/git_utils.bzl index 18e217e73..9f986abd1 100644 --- a/uv/private/extension/git_utils.bzl +++ b/uv/private/extension/git_utils.bzl @@ -74,6 +74,30 @@ def parse_git_url(url): return kwargs +def locked_git_requirement_urls(url): + """Returns direct-reference URLs that can identify a locked git source.""" + remote_and_query, _hash_sep, commit = url.partition("#") + remote, _query_sep, query = remote_and_query.partition("?") + remote = remote if remote.startswith("git+") else "git+" + remote + + refs = [] + subdirectory = "" + for param in query.split("&") if query else []: + key, _eq, value = param.partition("=") + value = value.replace("%2F", "/").replace("%2f", "/") + if key in ["tag", "branch", "rev", "ref"] and value: + refs.append(value) + elif key == "subdirectory": + subdirectory = value + + suffix = "#subdirectory=" + subdirectory if subdirectory else "" + result = {remote + suffix: True} + if commit: + result[remote + "@" + commit + suffix] = True + for ref in refs: + result[remote + "@" + ref + suffix] = True + return result.keys() + def try_git_to_http_archive(git_cfg): """Tries to convert a `git_repository` configuration to an `http_archive`. diff --git a/uv/private/extension/projectfile.bzl b/uv/private/extension/projectfile.bzl index 56083a3aa..54d3f00ec 100644 --- a/uv/private/extension/projectfile.bzl +++ b/uv/private/extension/projectfile.bzl @@ -6,7 +6,15 @@ load("//uv/private:normalize_name.bzl", "normalize_name") load("//uv/private/versions:versions.bzl", "find_matching_version") load(":dep_groups.bzl", "resolve_dependency_group_specs") -def extract_requirement_marker_pairs(projectfile, lock_id, req_string, version_map, package_versions = {}, preferred_versions = {}, fail_if_missing = True): +def extract_requirement_marker_pairs( + projectfile, + lock_id, + req_string, + version_map, + package_versions = {}, + preferred_versions = {}, + locked_urls = {}, + fail_if_missing = True): """Parses a requirement string into a list of dependency-marker pairs. This function parses a PEP 508 requirement string (e.g., @@ -23,6 +31,8 @@ def extract_requirement_marker_pairs(projectfile, lock_id, req_string, version_m list — useful for advisory requirements like the project-level `default_build_dependencies`, which may legitimately be absent from lockfiles that ship no sdists needing the build tooling. + locked_urls: A dictionary mapping `(package_name, artifact_url)` to a + locked dependency tuple, used to resolve direct references. Returns: A list of tuples, where each tuple is `(Dependency, Marker)`. @@ -55,6 +65,7 @@ def extract_requirement_marker_pairs(projectfile, lock_id, req_string, version_m "<": 1, "!": 1, "~": 1, + "@": 1, " ": 1, } @@ -85,21 +96,32 @@ def extract_requirement_marker_pairs(projectfile, lock_id, req_string, version_m remainder = remainder[close_idx + 1:] # 4. Look up version - v = preferred_versions.get(pkg_name) - if v == None: - v = version_map.get(pkg_name) - if v == None: - # For multi-version packages (e.g. conflicts), match the version - # specifier against all known versions of this package in the lockfile. - specifier = remainder.strip() - pkg_vers = package_versions.get(pkg_name, {}) - if pkg_vers: - match_spec = specifier if specifier else ">=0" - candidates = { - ver: (lock_id, pkg_name, ver, "__base__") - for ver in pkg_vers.keys() - } - v = find_matching_version(match_spec, candidates) + specifier = remainder.strip() + if specifier.startswith("@"): + # Direct references identify a locked artifact, not a version + # constraint. Never fall back to a different locked version or URL. + url = specifier[1:].strip() + + # Git uses semantically relevant fragments such as `#subdirectory=`. + if not url.startswith("git+"): + # uv stores hashes separately from URLs. + url = url.split("#", 1)[0] + v = locked_urls.get((pkg_name, url)) + else: + v = preferred_versions.get(pkg_name) + if v == None: + v = version_map.get(pkg_name) + if v == None: + # For multi-version packages (e.g. conflicts), match the version + # specifier against all known versions in the lockfile. + pkg_vers = package_versions.get(pkg_name, {}) + if pkg_vers: + match_spec = specifier if specifier else ">=0" + candidates = { + ver: (lock_id, pkg_name, ver, "__base__") + for ver in pkg_vers.keys() + } + v = find_matching_version(match_spec, candidates) if v == None: if not fail_if_missing: return [] @@ -149,7 +171,7 @@ def _extract_lockfile_group_versions(lock_id, lock_data): result.setdefault(group_name, {})[pkg_name] = (lock_id, pkg_name, dep["version"], "__base__") return result -def collect_activated_extras(projectfile, lock_id, project_data, lock_data, default_versions, graph, package_versions = {}): +def collect_activated_extras(projectfile, lock_id, project_data, lock_data, default_versions, graph, package_versions = {}, extra_roots = {}, locked_urls = {}): """Collects the set of transitively activated extras for each configuration. This function determines the full set of extras that are activated for each @@ -162,6 +184,11 @@ def collect_activated_extras(projectfile, lock_id, project_data, lock_data, defa default_versions: A dictionary mapping package names to their default version dependency tuples. graph: The dependency graph, as returned by `build_marker_graph`. + extra_roots: Additional resolved dependencies and their markers to + activate in every dependency group, remapped to each group's + selected package versions, such as build-requirement extras. + locked_urls: A dictionary mapping direct-reference URLs to locked + dependencies. Returns: A tuple containing: @@ -195,13 +222,13 @@ def collect_activated_extras(projectfile, lock_id, project_data, lock_data, defa group_preferences = dict(lockfile_group_versions.get(group_name, {})) for spec in resolved_specs: - for dep, _marker in extract_requirement_marker_pairs(projectfile, lock_id, spec, default_versions, package_versions, group_preferences): + for dep, _marker in extract_requirement_marker_pairs(projectfile, lock_id, spec, default_versions, package_versions, group_preferences, locked_urls = locked_urls): group_preferences[dep[1]] = (dep[0], dep[1], dep[2], "__base__") all_group_preferences[group_name] = group_preferences for spec in resolved_specs: - for dep, marker in extract_requirement_marker_pairs(projectfile, lock_id, spec, default_versions, package_versions, group_preferences): + for dep, marker in extract_requirement_marker_pairs(projectfile, lock_id, spec, default_versions, package_versions, group_preferences, locked_urls = locked_urls): normalized_dep_groups.setdefault(group_name, []).append(dep) # Note that this is the base case for the reach set walk below @@ -209,6 +236,16 @@ def collect_activated_extras(projectfile, lock_id, project_data, lock_data, defa base = (dep[0], dep[1], dep[2], "__base__") activated_extras.setdefault(base, {}).setdefault(group_name, {}).setdefault(dep, {}).update({marker: 1}) + for (dep_project, dep_name, dep_version, extra), markers in extra_roots.items(): + pref = group_preferences.get(dep_name) + if pref: + _pref_project, _pref_name, dep_version, _pref_extra = pref + + target_dep = (dep_project, dep_name, dep_version, extra) + normalized_dep_groups.setdefault(group_name, []).append(target_dep) + base = (dep_project, dep_name, dep_version, "__base__") + activated_extras.setdefault(base, {}).setdefault(group_name, {}).setdefault(target_dep, {}).update(markers) + for group_name, deps in normalized_dep_groups.items(): worklist = list(deps) group_prefs = all_group_preferences.get(group_name, {}) diff --git a/uv/private/extension/test_git_utils.bzl b/uv/private/extension/test_git_utils.bzl index 51c160562..43c60d5d2 100644 --- a/uv/private/extension/test_git_utils.bzl +++ b/uv/private/extension/test_git_utils.bzl @@ -1,5 +1,5 @@ load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") -load(":git_utils.bzl", "ensure_ref", "parse_git_url", "try_git_to_http_archive") +load(":git_utils.bzl", "ensure_ref", "locked_git_requirement_urls", "parse_git_url", "try_git_to_http_archive") def _ensure_ref_test_impl(ctx): env = unittest.begin(ctx) @@ -66,6 +66,25 @@ def _parse_git_url_bare_remote_test_impl(ctx): parse_git_url_bare_remote_test = unittest.make(_parse_git_url_bare_remote_test_impl) +def _locked_git_requirement_urls_test_impl(ctx): + env = unittest.begin(ctx) + result = locked_git_requirement_urls("https://github.com/benjaminp/six?tag=1.17.0#ebd9b3af90247b8858d415a05e96e9ee61e48d07") + asserts.equals(env, [ + "git+https://github.com/benjaminp/six", + "git+https://github.com/benjaminp/six@ebd9b3af90247b8858d415a05e96e9ee61e48d07", + "git+https://github.com/benjaminp/six@1.17.0", + ], result) + + result = locked_git_requirement_urls("git+https://github.com/example/project?branch=release%2F1.x&subdirectory=python%2Fpkg#abc123") + asserts.equals(env, [ + "git+https://github.com/example/project#subdirectory=python/pkg", + "git+https://github.com/example/project@abc123#subdirectory=python/pkg", + "git+https://github.com/example/project@release/1.x#subdirectory=python/pkg", + ], result) + return unittest.end(env) + +locked_git_requirement_urls_test = unittest.make(_locked_git_requirement_urls_test_impl) + def _git_to_http_archive_commit_test_impl(ctx): env = unittest.begin(ctx) result = try_git_to_http_archive({ @@ -133,6 +152,7 @@ def git_utils_test_suite(): parse_git_url_query_ref_test, parse_git_url_fragment_wins_over_query_test, parse_git_url_bare_remote_test, + locked_git_requirement_urls_test, git_to_http_archive_commit_test, git_to_http_archive_ref_test, git_to_http_archive_parsed_ref_url_test, diff --git a/uv/private/extension/test_projectfile.bzl b/uv/private/extension/test_projectfile.bzl index 8ba2e112d..3bb770154 100644 --- a/uv/private/extension/test_projectfile.bzl +++ b/uv/private/extension/test_projectfile.bzl @@ -78,6 +78,50 @@ extract_requirement_marker_pairs_with_extras_test = unittest.make( _extract_requirement_marker_pairs_with_extras_test_impl, ) +def _extract_requirement_marker_pairs_direct_reference_test_impl(ctx): + env = unittest.begin(ctx) + versions = {"build": {"1.3.0": 1, "2.0.0": 1}} + locked_urls = { + ("build", "https://example.invalid/build-1.3.0-py3-none-any.whl"): ("proj", "build", "1.3.0", "__base__"), + } + result = extract_requirement_marker_pairs( + "//:pyproject.toml", + "proj", + 'build[extra] @ https://example.invalid/build-1.3.0-py3-none-any.whl ; python_version >= "3.9"', + {}, + versions, + locked_urls = locked_urls, + ) + asserts.equals(env, 2, len(result)) + asserts.equals(env, (("proj", "build", "1.3.0", "__base__"), 'python_version >= "3.9"'), result[0]) + asserts.equals(env, (("proj", "build", "1.3.0", "extra"), 'python_version >= "3.9"'), result[1]) + + hashed = extract_requirement_marker_pairs( + "//:pyproject.toml", + "proj", + "build @ https://example.invalid/build-1.3.0-py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + {}, + versions, + locked_urls = locked_urls, + ) + asserts.equals(env, [(("proj", "build", "1.3.0", "__base__"), "")], hashed) + + missing = extract_requirement_marker_pairs( + "//:pyproject.toml", + "proj", + "build@https://example.invalid/build-2.0.0-py3-none-any.whl", + {}, + versions, + fail_if_missing = False, + locked_urls = locked_urls, + ) + asserts.equals(env, [], missing) + return unittest.end(env) + +extract_requirement_marker_pairs_direct_reference_test = unittest.make( + _extract_requirement_marker_pairs_direct_reference_test_impl, +) + def _extract_requirement_marker_pairs_preferred_overrides_version_map_test_impl(ctx): env = unittest.begin(ctx) version_map = {"build": ("proj", "build", "1.2.0", "__base__")} @@ -119,6 +163,96 @@ extract_requirement_marker_pairs_preferred_overrides_multi_version_test = unitte _extract_requirement_marker_pairs_preferred_overrides_multi_version_test_impl, ) +def _collect_activated_extras_build_extra_test_impl(ctx): + env = unittest.begin(ctx) + urllib3 = ("lock", "urllib3", "2.6.3", "__base__") + urllib3_brotli = ("lock", "urllib3", "2.6.3", "brotli") + brotli = ("lock", "brotli", "1.2.0", "__base__") + graph = { + urllib3: {}, + urllib3_brotli: {brotli: {"platform_python_implementation == 'CPython'": 1}}, + brotli: {}, + } + + _cfg_names, activated_extras = collect_activated_extras( + "//:pyproject.toml", + "lock", + { + "project": {"name": "test_project"}, + "dependency-groups": {"runtime": ["urllib3"]}, + }, + {}, + {"urllib3": urllib3, "brotli": brotli}, + graph, + {"urllib3": {"2.6.3": 1}, "brotli": {"1.2.0": 1}}, + {urllib3_brotli: {"sys_platform != 'win32'": 1}}, + ) + + asserts.equals(env, {"sys_platform != 'win32'": 1}, activated_extras[urllib3]["runtime"][urllib3_brotli]) + asserts.equals(env, {"platform_python_implementation == 'CPython'": 1}, activated_extras[brotli]["runtime"][brotli]) + return unittest.end(env) + +collect_activated_extras_build_extra_test = unittest.make( + _collect_activated_extras_build_extra_test_impl, +) + +def _collect_activated_extras_build_extra_conflicting_versions_test_impl(ctx): + env = unittest.begin(ctx) + requests_228 = ("lock", "requests", "2.28.2", "__base__") + requests_228_socks = ("lock", "requests", "2.28.2", "socks") + requests_232 = ("lock", "requests", "2.32.5", "__base__") + requests_232_socks = ("lock", "requests", "2.32.5", "socks") + pysocks = ("lock", "pysocks", "1.7.1", "__base__") + graph = { + requests_228: {}, + requests_228_socks: {pysocks: {"": 1}}, + requests_232: {}, + requests_232_socks: {pysocks: {"": 1}}, + pysocks: {}, + } + + _cfg_names, activated_extras = collect_activated_extras( + "//:pyproject.toml", + "lock", + { + "project": {"name": "test_project"}, + "dependency-groups": { + "old": ["requests==2.28.2"], + "new": ["requests==2.32.5"], + }, + }, + { + "package": [{ + "name": "test_project", + "version": "0.0.0", + "source": {"virtual": "."}, + "dev-dependencies": { + "old": [{"name": "requests", "version": "2.28.2"}], + "new": [{"name": "requests", "version": "2.32.5"}], + }, + }], + }, + {"pysocks": pysocks}, + graph, + { + "requests": {"2.28.2": 1, "2.32.5": 1}, + "pysocks": {"1.7.1": 1}, + }, + {requests_232_socks: {"": 1}}, + ) + + asserts.equals(env, {requests_228: {"": 1}, requests_228_socks: {"": 1}}, activated_extras[requests_228]["old"]) + asserts.false(env, "old" in activated_extras.get(requests_232, {})) + asserts.equals(env, {requests_232: {"": 1}, requests_232_socks: {"": 1}}, activated_extras[requests_232]["new"]) + asserts.false(env, "new" in activated_extras.get(requests_228, {})) + asserts.equals(env, {pysocks: {"": 1}}, activated_extras[pysocks]["old"]) + asserts.equals(env, {pysocks: {"": 1}}, activated_extras[pysocks]["new"]) + return unittest.end(env) + +collect_activated_extras_build_extra_conflicting_versions_test = unittest.make( + _collect_activated_extras_build_extra_conflicting_versions_test_impl, +) + def _collect_activated_extras_transitive_remap_test_impl(ctx): env = unittest.begin(ctx) project_data = { @@ -207,7 +341,10 @@ def projectfile_test_suite(): extract_requirement_marker_pairs_multi_version_with_specifier_test, extract_requirement_marker_pairs_single_version_via_map_test, extract_requirement_marker_pairs_with_extras_test, + extract_requirement_marker_pairs_direct_reference_test, extract_requirement_marker_pairs_preferred_overrides_version_map_test, extract_requirement_marker_pairs_preferred_overrides_multi_version_test, + collect_activated_extras_build_extra_test, + collect_activated_extras_build_extra_conflicting_versions_test, collect_activated_extras_transitive_remap_test, ) diff --git a/uv/private/sdist_build/repository.bzl b/uv/private/sdist_build/repository.bzl index c07ae3bf1..2ed8c5968 100644 --- a/uv/private/sdist_build/repository.bzl +++ b/uv/private/sdist_build/repository.bzl @@ -235,6 +235,10 @@ def _sdist_build_impl(repository_ctx): # Merge explicit deps with auto-discovered deps all_deps = [str(d) for d in repository_ctx.attr.deps] + extra_dep_labels + conditional_deps = {} + for dep, marker in repository_ctx.attr.conditional_deps.items(): + conditional_deps.setdefault(marker, []).append(str(dep)) + monitor_memory_attr = "" if repository_ctx.attr.monitor_memory: all_deps = [ @@ -280,12 +284,35 @@ def _sdist_build_impl(repository_ctx): if repository_ctx.attr.resource_set != "default": resource_set_attr = "\n resource_set = \"{}\",".format(repository_ctx.attr.resource_set) + rule = "pep517_native_whl" if is_native else "pep517_whl" + loads = [ + 'load("@aspect_rules_py//uv/private/pep517_whl:rule.bzl", "{}")'.format(rule), + 'load("@aspect_rules_py//py:defs.bzl", "py_binary")', + ] + if conditional_deps: + loads.append('load("@aspect_rules_py//uv/private/markers:defs.bzl", "decide_marker")') + marker_rules = "" + deps_expr = repr(all_deps) + for idx, (marker, deps) in enumerate(conditional_deps.items()): + if not deps: + continue + marker_name = "_build_dep_marker_{}".format(idx) + marker_rules += """ +decide_marker( + name = "{name}", + marker = {marker}, +) +""".format(name = marker_name, marker = repr(marker)) + deps_expr += """ + select({{ + ":{marker}": {deps}, + "//conditions:default": [], + }})""".format(marker = marker_name, deps = repr(deps)) + # Leave args unset: the pure rule validates anyarch wheels by default, # while the native rule defaults to no validation. repository_ctx.file("BUILD.bazel", content = """ -load("@aspect_rules_py//uv/private/pep517_whl:rule.bzl", "{rule}") -load("@aspect_rules_py//py:defs.bzl", "py_binary") - +{loads} +{marker_rules} py_binary( name = "build_tool", main = "@aspect_rules_py//uv/private/pep517_whl:build_helper.py", @@ -307,9 +334,11 @@ exports_files( ) """.format( src = repository_ctx.attr.src, - deps = repr(all_deps), + deps = deps_expr, + loads = "\n".join(loads), + marker_rules = marker_rules, monitor_memory_attr = monitor_memory_attr, - rule = "pep517_native_whl" if is_native else "pep517_whl", + rule = rule, version = repository_ctx.attr.version, resource_set_attr = resource_set_attr, patch_attrs = patch_attrs, @@ -321,6 +350,12 @@ sdist_build = repository_rule( attrs = { "src": attr.label(), "deps": attr.label_list(), + "conditional_deps": attr.label_keyed_string_dict( + default = {}, + doc = "Dict mapping build dependency labels to PEP 508 marker " + + "expressions. The generated build tool adds " + + "these dependencies only when their markers match.", + ), "available_deps": attr.string_dict( doc = "Dict mapping normalized package names to install labels. " + "Passed from the uv extension; used to resolve deps " +