Skip to content

Commit 1917736

Browse files
iliakurclaude
iliakur
and
claude
authored
Patch ddev to fully support the new lockfile format (DataDog#23357)
* Patch ddev to fully support the new lockfile format * Address PR feedback: make --wheels-storage required Click's `required=True` replaces the runtime ValueError in resolve_wheel_url and lets the typed signatures drop Optional[str] throughout. measure-disk-usage.yml gets a job-level INTEGRATIONS_WHEELS_STORAGE=stable env var that binds to the option. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 8ba656b commit 1917736

10 files changed

Lines changed: 83 additions & 16 deletions

File tree

‎.github/workflows/measure-disk-usage.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ on:
88
- completed
99
env:
1010
PYTHON_VERSION: "3.13"
11+
INTEGRATIONS_WHEELS_STORAGE: "stable"
1112

1213
jobs:
1314

‎ddev/changelog.d/23063.added‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
Support size computation for lockfiles in both new and old formats.
1+
- Support size computation for lockfiles in both new and old formats. New CLI param (and envvar) to switch between storage locations.
2+
- Command to trigger the promotion of wheels for PRs that bump dependencies.

‎ddev/src/ddev/cli/size/diff.py‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ def diff(
4949
compressed: bool,
5050
format: list[str],
5151
show_gui: bool,
52+
wheels_storage: str,
5253
) -> None:
5354
"""
5455
Compare the size of integrations and dependencies between two commits.
@@ -107,6 +108,7 @@ def diff(
107108
"compressed": compressed,
108109
"format": format,
109110
"show_gui": show_gui,
111+
"wheels_storage": wheels_storage,
110112
}
111113
modules_plat_ver.extend(
112114
diff_mode(
@@ -133,7 +135,14 @@ def diff_mode(
133135
progress: Progress,
134136
) -> list[FileDataEntryPlatformVersion]:
135137
files_b, dependencies_b, files_a, dependencies_a = get_repo_info(
136-
gitRepo, params["platform"], params["version"], first_commit, second_commit, params["compressed"], progress
138+
gitRepo,
139+
params["platform"],
140+
params["version"],
141+
first_commit,
142+
second_commit,
143+
params["compressed"],
144+
params["wheels_storage"],
145+
progress,
137146
)
138147

139148
integrations = get_diff(files_b, files_a, "Integration")
@@ -178,6 +187,7 @@ def get_repo_info(
178187
first_commit: str,
179188
second_commit: str,
180189
compressed: bool,
190+
wheels_storage: str,
181191
progress: Progress,
182192
) -> tuple[list[FileDataEntry], list[FileDataEntry], list[FileDataEntry], list[FileDataEntry]]:
183193
with progress:
@@ -205,13 +215,13 @@ def get_repo_info(
205215
task = progress.add_task("[cyan]Calculating sizes for the first commit...", total=None)
206216
gitRepo.checkout_commit(first_commit)
207217
files_b = get_files(repo, compressed, version)
208-
dependencies_b = get_dependencies(repo, platform, version, compressed)
218+
dependencies_b = get_dependencies(repo, platform, version, compressed, wheels_storage)
209219
progress.remove_task(task)
210220

211221
task = progress.add_task("[cyan]Calculating sizes for the second commit...", total=None)
212222
gitRepo.checkout_commit(second_commit)
213223
files_a = get_files(repo, compressed, version)
214-
dependencies_a = get_dependencies(repo, platform, version, compressed)
224+
dependencies_a = get_dependencies(repo, platform, version, compressed, wheels_storage)
215225
progress.remove_task(task)
216226

217227
return files_b, dependencies_b, files_a, dependencies_a

‎ddev/src/ddev/cli/size/status.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ def status(
3535
compressed: bool,
3636
format: list[str],
3737
show_gui: bool,
38+
wheels_storage: str,
3839
to_dd_org: str | None,
3940
to_dd_key: str | None,
4041
dependency_sizes: Path | None,
@@ -91,6 +92,7 @@ def status(
9192
"compressed": compressed,
9293
"format": format,
9394
"show_gui": show_gui,
95+
"wheels_storage": wheels_storage,
9496
}
9597
modules_plat_ver.extend(
9698
status_mode(
@@ -179,7 +181,7 @@ def status_mode(
179181
f"Getting dependencies from lockfiles for {params['platform']} {params['version']}"
180182
)
181183
modules = get_files(repo_path, params["compressed"], params["version"]) + get_dependencies(
182-
repo_path, params["platform"], params["version"], params["compressed"]
184+
repo_path, params["platform"], params["version"], params["compressed"], params["wheels_storage"]
183185
)
184186

185187
formatted_modules = format_modules(modules, params["platform"], params["version"])

‎ddev/src/ddev/cli/size/timeline.py‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
is_correct_dependency,
3131
is_valid_integration_file,
3232
print_table,
33+
resolve_wheel_url,
3334
save_csv,
3435
save_json,
3536
save_markdown,
@@ -71,6 +72,7 @@ def timeline(
7172
compressed: bool,
7273
format: Optional[list[str]],
7374
show_gui: bool,
75+
wheels_storage: str,
7476
) -> None:
7577
"""
7678
Show the size evolution of a module (integration or dependency) over time.
@@ -174,6 +176,7 @@ def timeline(
174176
"show_gui": show_gui,
175177
"first_commit": None,
176178
"platform": plat,
179+
"wheels_storage": wheels_storage,
177180
}
178181

179182
modules_plat.extend(
@@ -196,6 +199,7 @@ def timeline(
196199
"show_gui": show_gui,
197200
"first_commit": None,
198201
"platform": platform,
202+
"wheels_storage": wheels_storage,
199203
}
200204
modules_plat.extend(
201205
timeline_mode(
@@ -220,6 +224,7 @@ def timeline(
220224
"show_gui": show_gui,
221225
"first_commit": first_commit,
222226
"platform": None,
227+
"wheels_storage": wheels_storage,
223228
}
224229
progress.remove_task(task)
225230
modules.extend(
@@ -388,6 +393,7 @@ def process_commits(
388393
author,
389394
message,
390395
params["compressed"],
396+
params["wheels_storage"],
391397
)
392398
if result:
393399
file_data.append(result)
@@ -492,6 +498,7 @@ def get_dependencies(
492498
author: str,
493499
message: str,
494500
compressed: bool,
501+
wheels_storage: str,
495502
) -> Optional[CommitEntry]:
496503
"""
497504
Returns the size and metadata of a dependency for a given commit and platform.
@@ -515,7 +522,7 @@ def get_dependencies(
515522
for filename in paths:
516523
file_path = os.path.join(resolved_path, filename)
517524
if os.path.isfile(file_path) and is_correct_dependency(platform, version, filename):
518-
download_url, dep_version = get_dependency_data(file_path, module)
525+
download_url, dep_version = get_dependency_data(file_path, module, wheels_storage)
519526
return (
520527
get_dependency_size(download_url, dep_version, commit, date, author, message, compressed)
521528
if download_url and dep_version is not None
@@ -524,7 +531,7 @@ def get_dependencies(
524531
return None
525532

526533

527-
def get_dependency_data(file_path: str, module: str) -> tuple[Optional[str], Optional[str]]:
534+
def get_dependency_data(file_path: str, module: str, wheels_storage: str) -> tuple[Optional[str], Optional[str]]:
528535
"""
529536
Parses a dependency file and extracts the dependency name, download URL, and version.
530537
@@ -545,7 +552,7 @@ def get_dependency_data(file_path: str, module: str) -> tuple[Optional[str], Opt
545552
raise WrongDependencyFormat("The dependency format 'name @ link' is no longer supported.")
546553
name, url = match.groups()
547554
if name == module:
548-
url = url.replace("${INTEGRATIONS_WHEELS_STORAGE}", "stable")
555+
url = resolve_wheel_url(url, wheels_storage)
549556
version_match = re.search(rf"{re.escape(name)}/[^/]+?-([0-9]+(?:\.[0-9]+)*)-", url)
550557
version = version_match.group(1) if version_match else ""
551558
return url, version

‎ddev/src/ddev/cli/size/utils/common_funcs.py‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ class CLIParameters(TypedDict):
7373
compressed: bool # Whether to analyze compressed file sizes
7474
format: Optional[list[str]] # Output format options (png, csv, markdown, json)
7575
show_gui: bool # Whether to display interactive visualization
76+
wheels_storage: str # Storage tier (dev/stable) for new-style lockfile URLs
7677

7778

7879
class CLIParametersTimeline(TypedDict):
@@ -82,6 +83,7 @@ class CLIParametersTimeline(TypedDict):
8283
compressed: bool # Whether to analyze compressed file sizes
8384
format: Optional[list[str]] # Output format options (png, csv, markdown, json)
8485
show_gui: bool # Whether to display interactive visualization
86+
wheels_storage: str # Storage tier (dev/stable) for new-style lockfile URLs
8587

8688

8789
class InitialParametersTimelineIntegration(CLIParametersTimeline):
@@ -300,7 +302,17 @@ def extract_version_from_about_py(path: str) -> str:
300302
return ""
301303

302304

303-
def get_dependencies(repo_path: str | Path, platform: str, version: str, compressed: bool) -> list[FileDataEntry]:
305+
WHEELS_STORAGE_PLACEHOLDER = "${INTEGRATIONS_WHEELS_STORAGE}"
306+
307+
308+
def resolve_wheel_url(url: str, wheels_storage: str) -> str:
309+
"""Substitute the wheels storage tier into a lockfile URL."""
310+
return url.replace(WHEELS_STORAGE_PLACEHOLDER, wheels_storage)
311+
312+
313+
def get_dependencies(
314+
repo_path: str | Path, platform: str, version: str, compressed: bool, wheels_storage: str
315+
) -> list[FileDataEntry]:
304316
"""
305317
Gets the list of dependencies for a given platform and Python version and returns a FileDataEntry that includes:
306318
Name, Version, Size_Bytes, Size, and Type.
@@ -311,12 +323,12 @@ def get_dependencies(repo_path: str | Path, platform: str, version: str, compres
311323
file_path = os.path.join(resolved_path, filename)
312324

313325
if os.path.isfile(file_path) and is_correct_dependency(platform, version, filename):
314-
deps, download_urls, versions = get_dependencies_list(file_path)
326+
deps, download_urls, versions = get_dependencies_list(file_path, wheels_storage)
315327
return get_dependencies_sizes(deps, download_urls, versions, compressed)
316328
return []
317329

318330

319-
def get_dependencies_list(file_path: str) -> tuple[list[str], list[str], list[str]]:
331+
def get_dependencies_list(file_path: str, wheels_storage: str) -> tuple[list[str], list[str], list[str]]:
320332
"""
321333
Parses a dependency file and extracts the dependency names, download URLs, and versions.
322334
"""
@@ -331,7 +343,7 @@ def get_dependencies_list(file_path: str) -> tuple[list[str], list[str], list[st
331343
if not match:
332344
raise WrongDependencyFormat("The dependency format 'name @ link' is no longer supported.")
333345
name = match.group(1)
334-
url = match.group(2).replace("${INTEGRATIONS_WHEELS_STORAGE}", "stable")
346+
url = resolve_wheel_url(match.group(2), wheels_storage)
335347

336348
deps.append(name)
337349
download_urls.append(url)

‎ddev/src/ddev/cli/size/utils/common_params.py‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,35 @@ def common_params(func: Callable) -> Callable:
2121
is_flag=True,
2222
help="Display a pop-up window with a treemap showing the current size distribution of modules.",
2323
)
24+
# An option rather than a positional argument so it can bind to the
25+
# INTEGRATIONS_WHEELS_STORAGE env var (Click only supports envvar on options).
26+
# This keeps CI invocations aligned with the GitLab variable of the same name.
27+
@click.option(
28+
"--wheels-storage",
29+
type=click.Choice(["dev", "stable"]),
30+
required=True,
31+
envvar="INTEGRATIONS_WHEELS_STORAGE",
32+
help=(
33+
"Which wheel storage tier to resolve dependency URLs against. "
34+
"Can also be set via the INTEGRATIONS_WHEELS_STORAGE env var."
35+
),
36+
)
2437
@click.pass_context
2538
def wrapper(
26-
ctx: click.Context, platform: str, compressed: bool, format: list[str], show_gui: bool, *args, **kwargs
39+
ctx: click.Context,
40+
platform: str,
41+
compressed: bool,
42+
format: list[str],
43+
show_gui: bool,
44+
wheels_storage: str,
45+
*args,
46+
**kwargs,
2747
):
2848
kwargs["platform"] = platform
2949
kwargs["compressed"] = compressed
3050
kwargs["format"] = format
3151
kwargs["show_gui"] = show_gui
52+
kwargs["wheels_storage"] = wheels_storage
3253
return ctx.invoke(func, *args, **kwargs)
3354

3455
return wrapper

‎ddev/tests/cli/size/conftest.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,8 @@ def mock_matplotlib_globally(mocker: pytest_mock.MockerFixture):
88
mocker.patch("matplotlib.pyplot.show")
99
mocker.patch("matplotlib.pyplot.savefig")
1010
mocker.patch("matplotlib.pyplot.figure")
11+
12+
13+
@pytest.fixture(autouse=True)
14+
def default_wheels_storage(monkeypatch):
15+
monkeypatch.setenv("INTEGRATIONS_WHEELS_STORAGE", "stable")

‎ddev/tests/size/test_common.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ def test_get_dependencies_list():
148148
file_content = "dependency1 @ https://example.com/dependency1/dependency1-1.1.1-.whl\ndependency2 @ https://example.com/dependency2/dependency2-1.1.1-.whl"
149149
mock_open_obj = mock_open(read_data=file_content)
150150
with patch("builtins.open", mock_open_obj):
151-
deps, urls, versions = get_dependencies_list("fake_path")
151+
deps, urls, versions = get_dependencies_list("fake_path", "stable")
152152
assert deps == ["dependency1", "dependency2"]
153153
assert urls == [
154154
"https://example.com/dependency1/dependency1-1.1.1-.whl",

‎ddev/tests/size/test_timeline.py‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ def test_get_dependency():
108108
content = """dep1 @ https://example.com/dep1/dep1-1.1.1-.whl
109109
dep2 @ https://example.com/dep2/dep2-1.1.2-.whl"""
110110
with patch("ddev.cli.size.timeline.open", mock_open(read_data=content)):
111-
url, version = get_dependency_data(Path("some") / "path" / "file.txt", "dep2")
111+
url, version = get_dependency_data(Path("some") / "path" / "file.txt", "dep2", "stable")
112112
assert (url, version) == ("https://example.com/dep2/dep2-1.1.2-.whl", "1.1.2")
113113

114114

@@ -155,7 +155,15 @@ def test_get_compressed_dependencies():
155155
patch("ddev.cli.size.timeline.requests.head", return_value=make_mock_response("12345")),
156156
):
157157
result = get_dependencies(
158-
"fake_repo", "dep1", "linux-x86_64", "abc1234", datetime(2025, 4, 4).date(), "auth", "Added dep1", True
158+
"fake_repo",
159+
"dep1",
160+
"linux-x86_64",
161+
"abc1234",
162+
datetime(2025, 4, 4).date(),
163+
"auth",
164+
"Added dep1",
165+
True,
166+
"stable",
159167
)
160168
assert result == {
161169
"Size_Bytes": 12345,

0 commit comments

Comments
 (0)