Skip to content

Commit bfcc7b3

Browse files
jbedardjbedard
authored
perf(uv): deduplicate wheel installs (#1370)
Close #1352 ### Changes are visible to end-users: yes - Searched for relevant documentation and updated as needed: yes - Breaking change (forces users to change their own code or config): no - Suggested release notes appear below: yes Performance: share wheel installs across lockfiles when equivelent ### Test plan - Covered by existing test cases - New test cases added
1 parent 421da95 commit bfcc7b3

19 files changed

Lines changed: 699 additions & 5 deletions

‎e2e/cases/BUILD.bazel‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,19 @@ write_source_files(
7777
# decide_marker emitted — is already pinned by project_single above.)
7878
"snapshots/extra_marker.private.sccs.BUILD.bazel": "@project__extra_marker//private/sccs:BUILD.bazel",
7979

80+
# @pypi_dedup (dedup-installs: three lock universes over a shared set of
81+
# pure-bdist packages, with varying deps and exclude_glob overrides).
82+
# Removal proof: this genquery lists every wheel-install repo reachable
83+
# from all three projects' targets. Without dedup it names 26 repos
84+
# (canonical 9 + duplicate 9 + triplicate 8); the pinned file has 12 —
85+
# the identities that survive after matching (wheels, exclude_glob)
86+
# installs collapse. The repo names encode which projects shared: e.g.
87+
# both a `_canonical_ pyflakes` (excluded) and a `_duplicate_ pyflakes`
88+
# (unexcluded, shared with triplicate) survive, while pycodestyle and
89+
# toml each collapse their two same-exclusion copies into one and leave
90+
# the odd-one-out separate. See dedup-installs/setup.MODULE.bazel.
91+
"snapshots/dedup_installs.shared_install_repos.query": "//dedup-installs:shared_install_repos",
92+
8093
# @whl_install for grpcio with bare linux_* wheels
8194
# ----------------------------------------------------------------
8295
# grpcio 1.80.0 ships bare linux_armv7l wheels alongside manylinux/

‎e2e/cases/MODULE.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ include("//azure-namespace-sibling:setup.MODULE.bazel")
107107
include("//coverage-drivers:setup.MODULE.bazel")
108108
include("//cross-repo-610:setup.MODULE.bazel")
109109
include("//current-py-toolchain-bazel-env-896:setup.MODULE.bazel")
110+
include("//dedup-installs:setup.MODULE.bazel")
110111
include("//firebase-admin-import:setup.MODULE.bazel")
111112
include("//freethreaded-805:setup.MODULE.bazel")
112113
include("//google-native-overlay:setup.MODULE.bazel")

‎e2e/cases/dedup-installs/BUILD.bazel‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
# Three lock universes (`canonical`, `duplicate`, `triplicate`) over a shared
2+
# set of pure-bdist packages, with slightly varying deps and exclude_glob
3+
# overrides (see setup.MODULE.bazel). Each test builds+runs against the shared
4+
# deduplicated install repos, proving the reference rewrite leaves no dangling
5+
# label. The //snapshots dedup-installs entry pins the dedup signature.
6+
load("@aspect_rules_py//py:defs.bzl", "py_test")
7+
8+
# The seven packages every project shares. canonical/duplicate add mccabe.
9+
_COMMON_PACKAGES = [
10+
"@pypi_dedup//cachetools",
11+
"@pypi_dedup//decorator",
12+
"@pypi_dedup//distlib",
13+
"@pypi_dedup//inflection",
14+
"@pypi_dedup//pycodestyle",
15+
"@pypi_dedup//pyflakes",
16+
"@pypi_dedup//sortedcontainers",
17+
"@pypi_dedup//toml",
18+
]
19+
20+
# The dedup proof: enumerate every wheel-install repo reachable from all three
21+
# lock universes. The query is unconfigured, so it walks every select() arm of
22+
# all three projects. Without the dedup this lists 26 repos (canonical 9 +
23+
# duplicate 9 + triplicate 8); the pinned snapshot has 12 — the surviving
24+
# identities after collapsing matching (wheels, exclude_glob) installs.
25+
genquery(
26+
name = "shared_install_repos",
27+
expression = "filter(\"whl_install__shared_install_.*//:install$\", deps(//dedup-installs:canonical) + deps(//dedup-installs:duplicate) + deps(//dedup-installs:triplicate))",
28+
scope = [
29+
"//dedup-installs:canonical",
30+
"//dedup-installs:duplicate",
31+
"//dedup-installs:triplicate",
32+
],
33+
visibility = ["//:__pkg__"],
34+
)
35+
36+
# Each test resolves its packages through its own project's dep_group config,
37+
# so the three lock universes are exercised independently while landing on the
38+
# shared install repos. The smoke test imports only the common packages so it
39+
# runs unchanged against triplicate (which omits mccabe); pyflakes and
40+
# pycodestyle are imported to prove the exclude_glob installs still work.
41+
py_test(
42+
name = "canonical",
43+
srcs = ["__test__.py"],
44+
dep_group = "shared-install-canonical",
45+
main = "__test__.py",
46+
python_version = "3.11",
47+
deps = _COMMON_PACKAGES + ["@pypi_dedup//mccabe"],
48+
)
49+
50+
py_test(
51+
name = "duplicate",
52+
srcs = ["__test__.py"],
53+
dep_group = "shared-install-duplicate",
54+
main = "__test__.py",
55+
python_version = "3.11",
56+
deps = _COMMON_PACKAGES + ["@pypi_dedup//mccabe"],
57+
)
58+
59+
py_test(
60+
name = "triplicate",
61+
srcs = ["__test__.py"],
62+
dep_group = "shared-install-triplicate",
63+
main = "__test__.py",
64+
python_version = "3.11",
65+
deps = _COMMON_PACKAGES,
66+
)

‎e2e/cases/dedup-installs/__test__.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env python3
2+
"""Smoke test that every lock universe resolves the shared package set,
3+
including the packages that carry an exclude_glob override."""
4+
5+
import cachetools
6+
import decorator
7+
import distlib
8+
import inflection
9+
import pycodestyle
10+
import pyflakes
11+
import sortedcontainers
12+
import toml
13+
14+
assert toml.loads("k = 1")["k"] == 1
15+
assert inflection.camelize("dedup_installs") == "DedupInstalls"
16+
assert sortedcontainers.SortedList([3, 1, 2]) == [1, 2, 3]

‎e2e/cases/dedup-installs/canonical/BUILD.bazel‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
exports_files([
2+
"pyproject.toml",
3+
"uv.lock",
4+
])

‎e2e/cases/dedup-installs/canonical/pyproject.toml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
[project]
2+
name = "shared-install-canonical"
3+
version = "0.0.0"
4+
authors = []
5+
requires-python = ">=3.11"
6+
dependencies = [
7+
"cachetools",
8+
"decorator",
9+
"distlib",
10+
"inflection",
11+
"mccabe",
12+
"pycodestyle",
13+
"pyflakes",
14+
"sortedcontainers",
15+
"toml",
16+
]

‎e2e/cases/dedup-installs/canonical/uv.lock‎

Lines changed: 94 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎e2e/cases/dedup-installs/duplicate/BUILD.bazel‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
exports_files([
2+
"pyproject.toml",
3+
"uv.lock",
4+
])

‎e2e/cases/dedup-installs/duplicate/pyproject.toml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
[project]
2+
name = "shared-install-duplicate"
3+
version = "0.0.0"
4+
authors = []
5+
requires-python = ">=3.11"
6+
dependencies = [
7+
"cachetools",
8+
"decorator",
9+
"distlib",
10+
"inflection",
11+
"mccabe",
12+
"pycodestyle",
13+
"pyflakes",
14+
"sortedcontainers",
15+
"toml",
16+
]

‎e2e/cases/dedup-installs/duplicate/uv.lock‎

Lines changed: 94 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)