diff --git a/e2e/cases/BUILD.bazel b/e2e/cases/BUILD.bazel index 1a301bf9a..314ebc619 100644 --- a/e2e/cases/BUILD.bazel +++ b/e2e/cases/BUILD.bazel @@ -77,6 +77,19 @@ write_source_files( # decide_marker emitted — is already pinned by project_single above.) "snapshots/extra_marker.private.sccs.BUILD.bazel": "@project__extra_marker//private/sccs:BUILD.bazel", + # @pypi_dedup (dedup-installs: three lock universes over a shared set of + # pure-bdist packages, with varying deps and exclude_glob overrides). + # Removal proof: this genquery lists every wheel-install repo reachable + # from all three projects' targets. Without dedup it names 26 repos + # (canonical 9 + duplicate 9 + triplicate 8); the pinned file has 12 — + # the identities that survive after matching (wheels, exclude_glob) + # installs collapse. The repo names encode which projects shared: e.g. + # both a `_canonical_ pyflakes` (excluded) and a `_duplicate_ pyflakes` + # (unexcluded, shared with triplicate) survive, while pycodestyle and + # toml each collapse their two same-exclusion copies into one and leave + # the odd-one-out separate. See dedup-installs/setup.MODULE.bazel. + "snapshots/dedup_installs.shared_install_repos.query": "//dedup-installs:shared_install_repos", + # @whl_install for grpcio with bare linux_* wheels # ---------------------------------------------------------------- # grpcio 1.80.0 ships bare linux_armv7l wheels alongside manylinux/ diff --git a/e2e/cases/MODULE.bazel b/e2e/cases/MODULE.bazel index b9d1eda0e..2d0137ca4 100644 --- a/e2e/cases/MODULE.bazel +++ b/e2e/cases/MODULE.bazel @@ -107,6 +107,7 @@ include("//azure-namespace-sibling:setup.MODULE.bazel") include("//coverage-drivers:setup.MODULE.bazel") include("//cross-repo-610:setup.MODULE.bazel") include("//current-py-toolchain-bazel-env-896:setup.MODULE.bazel") +include("//dedup-installs:setup.MODULE.bazel") include("//firebase-admin-import:setup.MODULE.bazel") include("//freethreaded-805:setup.MODULE.bazel") include("//google-native-overlay:setup.MODULE.bazel") diff --git a/e2e/cases/dedup-installs/BUILD.bazel b/e2e/cases/dedup-installs/BUILD.bazel new file mode 100644 index 000000000..8e40d934d --- /dev/null +++ b/e2e/cases/dedup-installs/BUILD.bazel @@ -0,0 +1,66 @@ +# Three lock universes (`canonical`, `duplicate`, `triplicate`) over a shared +# set of pure-bdist packages, with slightly varying deps and exclude_glob +# overrides (see setup.MODULE.bazel). Each test builds+runs against the shared +# deduplicated install repos, proving the reference rewrite leaves no dangling +# label. The //snapshots dedup-installs entry pins the dedup signature. +load("@aspect_rules_py//py:defs.bzl", "py_test") + +# The seven packages every project shares. canonical/duplicate add mccabe. +_COMMON_PACKAGES = [ + "@pypi_dedup//cachetools", + "@pypi_dedup//decorator", + "@pypi_dedup//distlib", + "@pypi_dedup//inflection", + "@pypi_dedup//pycodestyle", + "@pypi_dedup//pyflakes", + "@pypi_dedup//sortedcontainers", + "@pypi_dedup//toml", +] + +# The dedup proof: enumerate every wheel-install repo reachable from all three +# lock universes. The query is unconfigured, so it walks every select() arm of +# all three projects. Without the dedup this lists 26 repos (canonical 9 + +# duplicate 9 + triplicate 8); the pinned snapshot has 12 — the surviving +# identities after collapsing matching (wheels, exclude_glob) installs. +genquery( + name = "shared_install_repos", + expression = "filter(\"whl_install__shared_install_.*//:install$\", deps(//dedup-installs:canonical) + deps(//dedup-installs:duplicate) + deps(//dedup-installs:triplicate))", + scope = [ + "//dedup-installs:canonical", + "//dedup-installs:duplicate", + "//dedup-installs:triplicate", + ], + visibility = ["//:__pkg__"], +) + +# Each test resolves its packages through its own project's dep_group config, +# so the three lock universes are exercised independently while landing on the +# shared install repos. The smoke test imports only the common packages so it +# runs unchanged against triplicate (which omits mccabe); pyflakes and +# pycodestyle are imported to prove the exclude_glob installs still work. +py_test( + name = "canonical", + srcs = ["__test__.py"], + dep_group = "shared-install-canonical", + main = "__test__.py", + python_version = "3.11", + deps = _COMMON_PACKAGES + ["@pypi_dedup//mccabe"], +) + +py_test( + name = "duplicate", + srcs = ["__test__.py"], + dep_group = "shared-install-duplicate", + main = "__test__.py", + python_version = "3.11", + deps = _COMMON_PACKAGES + ["@pypi_dedup//mccabe"], +) + +py_test( + name = "triplicate", + srcs = ["__test__.py"], + dep_group = "shared-install-triplicate", + main = "__test__.py", + python_version = "3.11", + deps = _COMMON_PACKAGES, +) diff --git a/e2e/cases/dedup-installs/__test__.py b/e2e/cases/dedup-installs/__test__.py new file mode 100644 index 000000000..538e09219 --- /dev/null +++ b/e2e/cases/dedup-installs/__test__.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Smoke test that every lock universe resolves the shared package set, +including the packages that carry an exclude_glob override.""" + +import cachetools +import decorator +import distlib +import inflection +import pycodestyle +import pyflakes +import sortedcontainers +import toml + +assert toml.loads("k = 1")["k"] == 1 +assert inflection.camelize("dedup_installs") == "DedupInstalls" +assert sortedcontainers.SortedList([3, 1, 2]) == [1, 2, 3] diff --git a/e2e/cases/dedup-installs/canonical/BUILD.bazel b/e2e/cases/dedup-installs/canonical/BUILD.bazel new file mode 100644 index 000000000..7154579d2 --- /dev/null +++ b/e2e/cases/dedup-installs/canonical/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files([ + "pyproject.toml", + "uv.lock", +]) diff --git a/e2e/cases/dedup-installs/canonical/pyproject.toml b/e2e/cases/dedup-installs/canonical/pyproject.toml new file mode 100644 index 000000000..1301c69f9 --- /dev/null +++ b/e2e/cases/dedup-installs/canonical/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "shared-install-canonical" +version = "0.0.0" +authors = [] +requires-python = ">=3.11" +dependencies = [ + "cachetools", + "decorator", + "distlib", + "inflection", + "mccabe", + "pycodestyle", + "pyflakes", + "sortedcontainers", + "toml", +] diff --git a/e2e/cases/dedup-installs/canonical/uv.lock b/e2e/cases/dedup-installs/canonical/uv.lock new file mode 100644 index 000000000..fa644dfc0 --- /dev/null +++ b/e2e/cases/dedup-installs/canonical/uv.lock @@ -0,0 +1,94 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "shared-install-canonical" +version = "0.0.0" +source = { virtual = "." } +dependencies = [ + { name = "cachetools" }, + { name = "decorator" }, + { name = "distlib" }, + { name = "inflection" }, + { name = "mccabe" }, + { name = "pycodestyle" }, + { name = "pyflakes" }, + { name = "sortedcontainers" }, + { name = "toml" }, +] + +[package.metadata] +requires-dist = [{ name = "cachetools" }, { name = "decorator" }, { name = "distlib" }, { name = "inflection" }, { name = "mccabe" }, { name = "pycodestyle" }, { name = "pyflakes" }, { name = "sortedcontainers" }, { name = "toml" }] + +[[package]] +name = "cachetools" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080 }, +] + +[[package]] +name = "decorator" +version = "5.1.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/50/83c593b07763e1161326b3b8c6686f0f4b0f24d5526546bee538c89837d6/decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186", size = 9073 }, +] + +[[package]] +name = "distlib" +version = "0.3.9" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/a1/cf2472db20f7ce4a6be1253a81cfdf85ad9c7885ffbed7047fb72c24cf87/distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87", size = 468973 }, +] + +[[package]] +name = "inflection" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/91/aa6bde563e0085a02a435aa99b49ef75b0a4b062635e606dab23ce18d720/inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2", size = 9454 }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350 }, +] + +[[package]] +name = "pycodestyle" +version = "2.12.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/d8/a211b3f85e99a0daa2ddec96c949cac6824bd305b040571b82a03dd62636/pycodestyle-2.12.1-py2.py3-none-any.whl", hash = "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3", size = 31284 }, +] + +[[package]] +name = "pyflakes" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/d7/f1b7db88d8e4417c5d47adad627a93547f44bdc9028372dbd2313f34a855/pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a", size = 62725 }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575 }, +] + +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588 }, +] diff --git a/e2e/cases/dedup-installs/duplicate/BUILD.bazel b/e2e/cases/dedup-installs/duplicate/BUILD.bazel new file mode 100644 index 000000000..7154579d2 --- /dev/null +++ b/e2e/cases/dedup-installs/duplicate/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files([ + "pyproject.toml", + "uv.lock", +]) diff --git a/e2e/cases/dedup-installs/duplicate/pyproject.toml b/e2e/cases/dedup-installs/duplicate/pyproject.toml new file mode 100644 index 000000000..f40797401 --- /dev/null +++ b/e2e/cases/dedup-installs/duplicate/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "shared-install-duplicate" +version = "0.0.0" +authors = [] +requires-python = ">=3.11" +dependencies = [ + "cachetools", + "decorator", + "distlib", + "inflection", + "mccabe", + "pycodestyle", + "pyflakes", + "sortedcontainers", + "toml", +] diff --git a/e2e/cases/dedup-installs/duplicate/uv.lock b/e2e/cases/dedup-installs/duplicate/uv.lock new file mode 100644 index 000000000..ec4998637 --- /dev/null +++ b/e2e/cases/dedup-installs/duplicate/uv.lock @@ -0,0 +1,94 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "shared-install-duplicate" +version = "0.0.0" +source = { virtual = "." } +dependencies = [ + { name = "cachetools" }, + { name = "decorator" }, + { name = "distlib" }, + { name = "inflection" }, + { name = "mccabe" }, + { name = "pycodestyle" }, + { name = "pyflakes" }, + { name = "sortedcontainers" }, + { name = "toml" }, +] + +[package.metadata] +requires-dist = [{ name = "cachetools" }, { name = "decorator" }, { name = "distlib" }, { name = "inflection" }, { name = "mccabe" }, { name = "pycodestyle" }, { name = "pyflakes" }, { name = "sortedcontainers" }, { name = "toml" }] + +[[package]] +name = "cachetools" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080 }, +] + +[[package]] +name = "decorator" +version = "5.1.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/50/83c593b07763e1161326b3b8c6686f0f4b0f24d5526546bee538c89837d6/decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186", size = 9073 }, +] + +[[package]] +name = "distlib" +version = "0.3.9" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/a1/cf2472db20f7ce4a6be1253a81cfdf85ad9c7885ffbed7047fb72c24cf87/distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87", size = 468973 }, +] + +[[package]] +name = "inflection" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/91/aa6bde563e0085a02a435aa99b49ef75b0a4b062635e606dab23ce18d720/inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2", size = 9454 }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350 }, +] + +[[package]] +name = "pycodestyle" +version = "2.12.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/d8/a211b3f85e99a0daa2ddec96c949cac6824bd305b040571b82a03dd62636/pycodestyle-2.12.1-py2.py3-none-any.whl", hash = "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3", size = 31284 }, +] + +[[package]] +name = "pyflakes" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/d7/f1b7db88d8e4417c5d47adad627a93547f44bdc9028372dbd2313f34a855/pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a", size = 62725 }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575 }, +] + +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588 }, +] diff --git a/e2e/cases/dedup-installs/setup.MODULE.bazel b/e2e/cases/dedup-installs/setup.MODULE.bazel new file mode 100644 index 000000000..fdac1fc30 --- /dev/null +++ b/e2e/cases/dedup-installs/setup.MODULE.bazel @@ -0,0 +1,82 @@ +"""Three `uv.project()` lock universes over a shared set of pure-bdist packages, +arranged so the //snapshots dedup entry exercises every branch of the +`shared_install_key` identity: + + * cachetools, decorator, distlib, inflection, sortedcontainers — depended on + by all three projects with no override, so each collapses 3 -> 1. + * mccabe — depended on by canonical and duplicate only (triplicate's deps + vary), so it collapses 2 -> 1 and triplicate contributes nothing. + * pyflakes — canonical applies an exclude_glob, the other two don't. The + differing exclusion splits canonical's install off; duplicate and + triplicate (identical, no exclusion) still share. So pyflakes -> 2 repos. + * pycodestyle — canonical AND duplicate apply the SAME exclude_glob; + triplicate has none. The matching exclusions dedup canonical+duplicate + together, leaving triplicate separate. So pycodestyle -> 2 repos. + * toml — depended on by all three, and ALL THREE exclude, but only canonical + and duplicate use the same glob. Dedup keys on the exclusion value, not its + mere presence, so canonical+duplicate collapse and triplicate's different + glob stays separate. So toml -> 2 repos. + +12 install repos survive where canonical 9 + duplicate 9 + triplicate 8 +(triplicate drops mccabe) = 26 would exist without the dedup. The packages are used by no other case, so the +canonical repo of each identity is deterministic across the shared extension. +""" + +uv = use_extension("@aspect_rules_py//uv:extensions.bzl", "uv") +uv.declare_hub(hub_name = "pypi_dedup") +uv.project( + hub_name = "pypi_dedup", + lock = "//dedup-installs/canonical:uv.lock", + pyproject = "//dedup-installs/canonical:pyproject.toml", +) +uv.project( + hub_name = "pypi_dedup", + lock = "//dedup-installs/duplicate:uv.lock", + pyproject = "//dedup-installs/duplicate:pyproject.toml", +) +uv.project( + hub_name = "pypi_dedup", + lock = "//dedup-installs/triplicate:uv.lock", + pyproject = "//dedup-installs/triplicate:pyproject.toml", +) + +# Exclusion on ONE project only — its pyflakes install cannot share with the +# unexcluded copies in duplicate/triplicate. +uv.override_package( + name = "pyflakes", + exclude_glob = ["**/*.pyi"], + lock = "//dedup-installs/canonical:uv.lock", +) + +# The SAME exclusion on TWO projects — canonical and duplicate keep matching +# pycodestyle identities and dedup together; triplicate (no exclusion) does not. +uv.override_package( + name = "pycodestyle", + exclude_glob = ["**/py.typed"], + lock = "//dedup-installs/canonical:uv.lock", +) +uv.override_package( + name = "pycodestyle", + exclude_glob = ["**/py.typed"], + lock = "//dedup-installs/duplicate:uv.lock", +) + +# toml is depended on by all three, and all three exclude — but only canonical +# and duplicate share the SAME glob. Dedup keys on the exclusion value, so +# those two collapse while triplicate's different glob keeps it separate. +uv.override_package( + name = "toml", + exclude_glob = ["**/*.md"], + lock = "//dedup-installs/canonical:uv.lock", +) +uv.override_package( + name = "toml", + exclude_glob = ["**/*.md"], + lock = "//dedup-installs/duplicate:uv.lock", +) +uv.override_package( + name = "toml", + exclude_glob = ["**/*.rst"], + lock = "//dedup-installs/triplicate:uv.lock", +) +use_repo(uv, "pypi_dedup") diff --git a/e2e/cases/dedup-installs/triplicate/BUILD.bazel b/e2e/cases/dedup-installs/triplicate/BUILD.bazel new file mode 100644 index 000000000..7154579d2 --- /dev/null +++ b/e2e/cases/dedup-installs/triplicate/BUILD.bazel @@ -0,0 +1,4 @@ +exports_files([ + "pyproject.toml", + "uv.lock", +]) diff --git a/e2e/cases/dedup-installs/triplicate/pyproject.toml b/e2e/cases/dedup-installs/triplicate/pyproject.toml new file mode 100644 index 000000000..06dcc989f --- /dev/null +++ b/e2e/cases/dedup-installs/triplicate/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "shared-install-triplicate" +version = "0.0.0" +authors = [] +requires-python = ">=3.11" +dependencies = [ + "cachetools", + "decorator", + "distlib", + "inflection", + "pycodestyle", + "pyflakes", + "sortedcontainers", + "toml", +] diff --git a/e2e/cases/dedup-installs/triplicate/uv.lock b/e2e/cases/dedup-installs/triplicate/uv.lock new file mode 100644 index 000000000..b2265a27b --- /dev/null +++ b/e2e/cases/dedup-installs/triplicate/uv.lock @@ -0,0 +1,85 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "shared-install-triplicate" +version = "0.0.0" +source = { virtual = "." } +dependencies = [ + { name = "cachetools" }, + { name = "decorator" }, + { name = "distlib" }, + { name = "inflection" }, + { name = "pycodestyle" }, + { name = "pyflakes" }, + { name = "sortedcontainers" }, + { name = "toml" }, +] + +[package.metadata] +requires-dist = [{ name = "cachetools" }, { name = "decorator" }, { name = "distlib" }, { name = "inflection" }, { name = "pycodestyle" }, { name = "pyflakes" }, { name = "sortedcontainers" }, { name = "toml" }] + +[[package]] +name = "cachetools" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/72/76/20fa66124dbe6be5cafeb312ece67de6b61dd91a0247d1ea13db4ebb33c2/cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a", size = 10080 }, +] + +[[package]] +name = "decorator" +version = "5.1.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/50/83c593b07763e1161326b3b8c6686f0f4b0f24d5526546bee538c89837d6/decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186", size = 9073 }, +] + +[[package]] +name = "distlib" +version = "0.3.9" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/a1/cf2472db20f7ce4a6be1253a81cfdf85ad9c7885ffbed7047fb72c24cf87/distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87", size = 468973 }, +] + +[[package]] +name = "inflection" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/91/aa6bde563e0085a02a435aa99b49ef75b0a4b062635e606dab23ce18d720/inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2", size = 9454 }, +] + +[[package]] +name = "pycodestyle" +version = "2.12.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/d8/a211b3f85e99a0daa2ddec96c949cac6824bd305b040571b82a03dd62636/pycodestyle-2.12.1-py2.py3-none-any.whl", hash = "sha256:46f0fb92069a7c28ab7bb558f05bfc0110dac69a0cd23c61ea0040283a9d78b3", size = 31284 }, +] + +[[package]] +name = "pyflakes" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/d7/f1b7db88d8e4417c5d47adad627a93547f44bdc9028372dbd2313f34a855/pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a", size = 62725 }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575 }, +] + +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588 }, +] diff --git a/e2e/cases/snapshots/dedup_installs.shared_install_repos.query b/e2e/cases/snapshots/dedup_installs.shared_install_repos.query new file mode 100644 index 000000000..67baa6ce0 --- /dev/null +++ b/e2e/cases/snapshots/dedup_installs.shared_install_repos.query @@ -0,0 +1,12 @@ +@@aspect_rules_py++uv+whl_install__shared_install_canonical__cachetools__5_5_2//:install +@@aspect_rules_py++uv+whl_install__shared_install_canonical__decorator__5_1_1//:install +@@aspect_rules_py++uv+whl_install__shared_install_canonical__distlib__0_3_9//:install +@@aspect_rules_py++uv+whl_install__shared_install_canonical__inflection__0_5_1//:install +@@aspect_rules_py++uv+whl_install__shared_install_canonical__mccabe__0_7_0//:install +@@aspect_rules_py++uv+whl_install__shared_install_canonical__pycodestyle__2_12_1//:install +@@aspect_rules_py++uv+whl_install__shared_install_canonical__pyflakes__3_2_0//:install +@@aspect_rules_py++uv+whl_install__shared_install_canonical__sortedcontainers__2_4_0//:install +@@aspect_rules_py++uv+whl_install__shared_install_canonical__toml__0_10_2//:install +@@aspect_rules_py++uv+whl_install__shared_install_duplicate__pyflakes__3_2_0//:install +@@aspect_rules_py++uv+whl_install__shared_install_triplicate__pycodestyle__2_12_1//:install +@@aspect_rules_py++uv+whl_install__shared_install_triplicate__toml__0_10_2//:install diff --git a/e2e/cases/snapshots/project_single.BUILD.bazel b/e2e/cases/snapshots/project_single.BUILD.bazel index 11699bd7c..e5fe000de 100644 --- a/e2e/cases/snapshots/project_single.BUILD.bazel +++ b/e2e/cases/snapshots/project_single.BUILD.bazel @@ -30,7 +30,7 @@ alias( alias( name = "_package_cowsay_single_project_hub_whl", actual = select({ - "//conditions:default": "@whl_install__single_project_hub__cowsay__6_1//:whl", + "//conditions:default": "@whl_install__arch_alias_marker__cowsay__6_1//:whl", }), visibility = ["//visibility:private"], ) @@ -113,7 +113,7 @@ filegroup( filegroup( name = "gazelle_index_whls", srcs = [ - "@whl_install__single_project_hub__cowsay__6_1//:gazelle_index_whl", + "@whl_install__arch_alias_marker__cowsay__6_1//:gazelle_index_whl", ], visibility = ["//visibility:public"], ) diff --git a/e2e/cases/snapshots/project_single.private.sccs.BUILD.bazel b/e2e/cases/snapshots/project_single.private.sccs.BUILD.bazel index e0b2a522d..05f431cbd 100644 --- a/e2e/cases/snapshots/project_single.private.sccs.BUILD.bazel +++ b/e2e/cases/snapshots/project_single.private.sccs.BUILD.bazel @@ -12,7 +12,7 @@ py_library( # scc: cowsay__6_1 # members: # { -# "@whl_install__single_project_hub__cowsay__6_1//:install": { +# "@whl_install__arch_alias_marker__cowsay__6_1//:install": { # "": 1, # }, # } @@ -23,7 +23,7 @@ py_library( py_library( name = "cowsay__6_1", deps = [ - "@whl_install__single_project_hub__cowsay__6_1//:install", + "@whl_install__arch_alias_marker__cowsay__6_1//:install", ], visibility = ["//:__subpackages__"], ) diff --git a/uv/private/extension/defs.bzl b/uv/private/extension/defs.bzl index faa34ee6c..a7140664d 100644 --- a/uv/private/extension/defs.bzl +++ b/uv/private/extension/defs.bzl @@ -53,6 +53,7 @@ resolved dependencies available in the `@uv` repository. load("@bazel_lib//lib:resource_sets.bzl", "resource_set_values") load("@bazel_skylib//lib:sets.bzl", "sets") +load("@bazel_skylib//lib:structs.bzl", "structs") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") load("//py/private/interpreter:resolve.bzl", "resolve_host_interpreter_label") load("//uv/private:normalize_name.bzl", "normalize_name") @@ -82,6 +83,55 @@ def _dist_sha256(dist): hash = dist.get("hash", "") return hash[len("sha256:"):] if hash.startswith("sha256:") else None +def shared_install_key(install_cfg): + """Order-sensitive identity for a wheel install shareable across lock universes. + + Two lockfiles that pin the same package version resolve to the same + content-addressed `whl_dist` labels, so their `whl_install` selector repos + generate byte-identical BUILD files and can collapse to one. Returns None + for project-local installs (source builds, patches, extra deps/data), whose + inputs are keyed per-lockfile and never interchangeable. + """ + if install_cfg.sbuild or install_cfg.post_install_patches or install_cfg.extra_deps or install_cfg.extra_data: + return None + return json.encode([ + # Wheel selection keeps the first candidate on a specificity tie, so + # ordering is significant and the pairs stay as an ordered list. + install_cfg.whls.items(), + install_cfg.exclude_glob, + ]) + +def dedupe_shared_installs(install_cfgs): + """Drop duplicate shareable installs, returning `{dropped_id: canonical_id}`. + + Mutates `install_cfgs` in place, keeping the first install of each identity + as the canonical repo. Callers rewrite references to dropped ids through the + returned remap so no dangling `@dropped//:install` label survives. + """ + seen = {} + id_remap = {} + for install_id in install_cfgs.keys(): + key = shared_install_key(install_cfgs[install_id]) + if key == None: + continue + canonical = seen.get(key) + if canonical == None: + seen[key] = install_id + else: + id_remap[install_id] = canonical + for dropped in id_remap: + install_cfgs.pop(dropped) + return id_remap + +def _rewrite_available_deps(sbuild_cfg, target_remap): + """Rebuild an sbuild spec with its available_deps routed through the remap.""" + fields = structs.to_dict(sbuild_cfg) + fields["available_deps"] = { + pkg: target_remap.get(target, target) + for pkg, target in sbuild_cfg.available_deps.items() + } + return struct(**fields) + def parse_declared_console_script(name, entry_point): """Canonicalize one override_package console-script declaration. @@ -626,6 +676,35 @@ def _parse_projects(module_ctx, hub_specs): fail("uv.override_package() for '{}=={}' matches no uv.project() locks in module '{}'.".format(override.name, override.version, mod.name)) fail("uv.override_package() for '{}' matches no uv.project() locks in module '{}'.".format(override.name, mod.name)) + # Collapse installs that resolve identically across lock universes into one + # repo, then rewrite the two carriers of `@id//:install` labels — SCC graph + # keys and each sbuild's available_deps — so dropped ids leave no dangling + # reference. + id_remap = dedupe_shared_installs(install_cfgs) + if id_remap: + target_remap = { + "@{}//:install".format(dropped): "@{}//:install".format(canonical) + for dropped, canonical in id_remap.items() + } + project_cfgs = { + project_id: struct( + dep_to_scc = pc.dep_to_scc, + scc_deps = pc.scc_deps, + scc_graph = { + scc_id: { + target_remap.get(target, target): markers + for target, markers in members.items() + } + for scc_id, members in pc.scc_graph.items() + }, + ) + for project_id, pc in project_cfgs.items() + } + sbuild_specs = { + sbuild_id: _rewrite_available_deps(sc, target_remap) + for sbuild_id, sc in sbuild_specs.items() + } + return struct( project_cfgs = project_cfgs, hub_cfgs = hub_cfgs, diff --git a/uv/private/extension/test_defs.bzl b/uv/private/extension/test_defs.bzl index 05ae3bff3..502c1a581 100644 --- a/uv/private/extension/test_defs.bzl +++ b/uv/private/extension/test_defs.bzl @@ -1,7 +1,7 @@ """Unit tests for helpers in defs.bzl""" load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") -load(":defs.bzl", "parse_declared_console_script") +load(":defs.bzl", "dedupe_shared_installs", "parse_declared_console_script", "shared_install_key") load(":lockfile.bzl", "url_basename") def _url_basename_test_impl(ctx): @@ -72,6 +72,91 @@ def _declared_console_script_test_impl(ctx): declared_console_script_test = unittest.make(_declared_console_script_test_impl) +def _install_cfg( + whls, + exclude_glob = [], + sbuild = None, + post_install_patches = [], + extra_deps = [], + extra_data = []): + return struct( + whls = whls, + exclude_glob = exclude_glob, + sbuild = sbuild, + post_install_patches = post_install_patches, + extra_deps = extra_deps, + extra_data = extra_data, + ) + +def _shared_install_key_test_impl(ctx): + env = unittest.begin(ctx) + + ordered = { + "demo-1.0-cp311-cp311-manylinux.whl": "@whl__demo__aaa//:whl", + "demo-1.0-py3-none-any.whl": "@whl__demo__bbb//:whl", + } + reversed_order = { + "demo-1.0-py3-none-any.whl": "@whl__demo__bbb//:whl", + "demo-1.0-cp311-cp311-manylinux.whl": "@whl__demo__aaa//:whl", + } + + key = shared_install_key(_install_cfg(ordered)) + + # Same content across lock universes collapses to one repo. + asserts.equals(env, key, shared_install_key(_install_cfg(ordered))) + + # Wheel order is significant; reversed candidates are not interchangeable. + asserts.equals(env, False, key == shared_install_key(_install_cfg(reversed_order))) + + # exclude_glob changes the generated install and must split the identity. + asserts.equals( + env, + False, + key == shared_install_key(_install_cfg(ordered, exclude_glob = ["**/*.pyi"])), + ) + + # Project-local inputs are never shareable. + asserts.equals(env, None, shared_install_key(_install_cfg(ordered, sbuild = "@sdist_build__x//:whl"))) + asserts.equals(env, None, shared_install_key(_install_cfg(ordered, post_install_patches = ["//:demo.patch"]))) + asserts.equals(env, None, shared_install_key(_install_cfg(ordered, extra_deps = ["//:extra"]))) + asserts.equals(env, None, shared_install_key(_install_cfg(ordered, extra_data = ["//:data.txt"]))) + + return unittest.end(env) + +shared_install_key_test = unittest.make(_shared_install_key_test_impl) + +def _dedupe_shared_installs_test_impl(ctx): + env = unittest.begin(ctx) + + cowsay = {"cowsay-6.1-py3-none-any.whl": "@whl__cowsay__x//:whl"} + numpy = {"numpy-2.0-cp311-cp311-manylinux.whl": "@whl__numpy__y//:whl"} + install_cfgs = { + # First lock universe: canonical for cowsay. + "whl_install__alpha__cowsay__6_1": _install_cfg(cowsay), + # Second lock universe: identical cowsay collapses onto alpha. + "whl_install__beta__cowsay__6_1": _install_cfg(cowsay), + # Distinct package survives untouched. + "whl_install__beta__numpy__2_0": _install_cfg(numpy), + # Source-built install is never shareable, even with matching wheels. + "whl_install__gamma__cowsay__6_1": _install_cfg(cowsay, sbuild = "@sdist_build__gamma//:whl"), + } + + remap = dedupe_shared_installs(install_cfgs) + + # Only the duplicate shareable install is remapped, onto the first seen. + asserts.equals(env, {"whl_install__beta__cowsay__6_1": "whl_install__alpha__cowsay__6_1"}, remap) + + # The dropped repo is gone; canonical, distinct, and project-local stay. + asserts.equals( + env, + ["whl_install__alpha__cowsay__6_1", "whl_install__beta__numpy__2_0", "whl_install__gamma__cowsay__6_1"], + sorted(install_cfgs.keys()), + ) + + return unittest.end(env) + +dedupe_shared_installs_test = unittest.make(_dedupe_shared_installs_test_impl) + def defs_test_suite(): unittest.suite( "url_basename_tests", @@ -81,3 +166,11 @@ def defs_test_suite(): "declared_console_script_tests", declared_console_script_test, ) + unittest.suite( + "shared_install_key_tests", + shared_install_key_test, + ) + unittest.suite( + "dedupe_shared_installs_tests", + dedupe_shared_installs_test, + )