Skip to content

Commit 828a2ee

Browse files
lucia-sbAAraKKealopezz
authored
[AI-5800] Measure size of dependencies before PRs are merged (DataDog#21331)
* create dependency json * typing * order sizes * send metrics * send metrics * change on * change on * change on * change on * change on * change on * change on * change on * check files * check files * check files * check files * check files * check files * check files * check files * check files * measure disk usage * fix * repo * check wf * check wf * permissions * run_id * ddev * run_id * permissions * permissions * platforms * permissions * run_id * compressed * merge base * remove file * test run * uncomment * changelog * build deps folder * comment build * change * uncomment * workflow_run * default * sha * test * dep files * comment * tests * comment * spaces * test * change metric name * Review Changes * Review Changes * Review Changes * Review Changes * Review Changes * Review Changes * typing * Review Changes * Fix * Fix * only one platform * change git merge-base master to origin/master * check changed files * check for file changes * check no changes * delete previous code * docstrings * project sizes variable * improve docstring * review changes * review changes * Review changes * Review changes * Review changes * Review changes * Review changes * Review changes * Update .github/workflows/resolve-build-deps.yaml Co-authored-by: Alex Lopez <alex.lopez.zorzano@gmail.com> --------- Co-authored-by: Juanpe Araque <juanpedro.araque@datadoghq.com> Co-authored-by: Alex Lopez <alex.lopez.zorzano@gmail.com>
1 parent 3bc447b commit 828a2ee

10 files changed

Lines changed: 836 additions & 378 deletions

File tree

‎.builders/build.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from __future__ import annotations
22

33
import argparse
4+
import json
45
import os
56
import shutil
67
import subprocess
@@ -153,6 +154,10 @@ def build_macos():
153154
final_requirements = mount_dir / 'frozen.txt'
154155
shutil.move(final_requirements, output_dir)
155156

157+
# Move the dependency sizes to the output directory
158+
dependency_sizes_dir = mount_dir / 'sizes.json'
159+
shutil.move(dependency_sizes_dir, output_dir)
160+
156161

157162
def build_image():
158163
parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False)
@@ -246,6 +251,9 @@ def build_image():
246251
# Move the final requirements file to the output directory
247252
shutil.move(final_requirements, output_dir)
248253

254+
# Move the dependency sizes to the output directory
255+
shutil.move(mount_dir / 'sizes.json', output_dir)
256+
249257

250258
def main():
251259
if sys.platform == 'darwin':

‎.builders/scripts/build_wheels.py‎

Lines changed: 46 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
from __future__ import annotations
22

33
import argparse
4+
import json
45
import os
56
import subprocess
67
import sys
78
from pathlib import Path
89
from tempfile import TemporaryDirectory
10+
from typing import TypedDict
11+
from zipfile import ZipFile
912

1013
from dotenv import dotenv_values
1114
from utils import extract_metadata, normalize_project_name
@@ -14,6 +17,10 @@
1417
CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external'
1518
CUSTOM_BUILT_INDEX = f'{INDEX_BASE_URL}/built'
1619

20+
class WheelSizes(TypedDict):
21+
compressed: int
22+
uncompressed: int
23+
1724
if sys.platform == 'win32':
1825
PY3_PATH = Path('C:\\py3\\Scripts\\python.exe')
1926
PY2_PATH = Path('C:\\py2\\Scripts\\python.exe')
@@ -55,6 +62,13 @@ def check_process(*args, **kwargs) -> subprocess.CompletedProcess:
5562
return process
5663

5764

65+
def calculate_wheel_sizes(wheel_path: Path) -> WheelSizes:
66+
compressed_size = wheel_path.stat(follow_symlinks=True).st_size
67+
with ZipFile(wheel_path) as zf:
68+
uncompressed_size = sum(zinfo.file_size for zinfo in zf.infolist())
69+
return {'compressed': compressed_size, 'uncompressed': uncompressed_size}
70+
71+
5872
def main():
5973
parser = argparse.ArgumentParser(prog='wheel-builder', allow_abbrev=False)
6074
parser.add_argument('--python', required=True)
@@ -109,9 +123,14 @@ def main():
109123

110124
# Fetch or build wheels
111125
command_args = [
112-
str(python_path), '-m', 'pip', 'wheel',
113-
'-r', str(MOUNT_DIR / 'requirements.in'),
114-
'--wheel-dir', str(staged_wheel_dir),
126+
str(python_path),
127+
'-m',
128+
'pip',
129+
'wheel',
130+
'-r',
131+
str(MOUNT_DIR / 'requirements.in'),
132+
'--wheel-dir',
133+
str(staged_wheel_dir),
115134
# Temporarily removing extra index urls. See below.
116135
# '--extra-index-url', CUSTOM_EXTERNAL_INDEX,
117136
]
@@ -124,21 +143,37 @@ def main():
124143
check_process(command_args, env=env_vars)
125144

126145
# Repair wheels
127-
check_process([
128-
sys.executable, '-u', str(MOUNT_DIR / 'scripts' / 'repair_wheels.py'),
129-
'--source-dir', str(staged_wheel_dir),
130-
'--built-dir', str(built_wheels_dir),
131-
'--external-dir', str(external_wheels_dir),
132-
])
146+
check_process(
147+
[
148+
sys.executable,
149+
'-u',
150+
str(MOUNT_DIR / 'scripts' / 'repair_wheels.py'),
151+
'--source-dir',
152+
str(staged_wheel_dir),
153+
'--built-dir',
154+
str(built_wheels_dir),
155+
'--external-dir',
156+
str(external_wheels_dir),
157+
]
158+
)
133159

134160
dependencies: dict[str, tuple[str, str]] = {}
161+
sizes: dict[str, WheelSizes] = {}
162+
135163
for wheel_dir in wheels_dir.iterdir():
136-
for entry in wheel_dir.iterdir():
137-
project_metadata = extract_metadata(entry)
164+
for wheel in wheel_dir.iterdir():
165+
project_metadata = extract_metadata(wheel)
138166
project_name = normalize_project_name(project_metadata['Name'])
139167
project_version = project_metadata['Version']
140168
dependencies[project_name] = project_version
141169

170+
171+
sizes[project_name] = {'version': project_version, **calculate_wheel_sizes(wheel)}
172+
173+
output_path = MOUNT_DIR / 'sizes.json'
174+
with output_path.open('w', encoding='utf-8') as fp:
175+
json.dump(sizes, fp, indent=2, sort_keys=True)
176+
142177
final_requirements = MOUNT_DIR / 'frozen.txt'
143178
with final_requirements.open('w', encoding='utf-8') as f:
144179
for project_name, project_version in sorted(dependencies.items()):

‎.github/workflows/measure-disk-usage.yml‎

Lines changed: 31 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -1,113 +1,66 @@
11
name: Measure Disk Usage
22

33
on:
4-
push:
5-
branches:
6-
- master
4+
workflow_run:
5+
workflows: ['Resolve Dependencies and Build Wheels']
6+
types:
7+
- completed
78
env:
89
PYTHON_VERSION: "3.13"
910

1011
jobs:
12+
1113
measure-disk-usage:
1214
runs-on: ubuntu-22.04
15+
permissions:
16+
contents: read
17+
actions: read
1318
steps:
1419
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
1520
with:
1621
fetch-depth: 0
22+
1723
- name: Set up Python ${{ env.PYTHON_VERSION }}
1824
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
1925
with:
2026
python-version: ${{ env.PYTHON_VERSION }}
27+
2128
- name: Install ddev
2229
run: |
2330
pip install -e ./datadog_checks_dev[cli]
2431
pip install -e ./ddev
2532
2633
- name: Configure ddev
2734
run: |
28-
ddev config set repos.core .
29-
ddev config set repo core
30-
- name: Measure disk usage (uncompressed)
31-
run: |
32-
ddev size status --to-dd-key ${{secrets.DD_API_KEY}} > size-uncompressed.txt
33-
ddev size status --format png,csv,markdown
34-
cat size-uncompressed.txt
35-
echo "# Size (uncompressed)" >> $GITHUB_STEP_SUMMARY
36-
cat uncompressed_status.md >> $GITHUB_STEP_SUMMARY
37-
38-
- name: Measure disk usage (compressed)
39-
run: |
40-
ddev size status --compressed --to-dd-key ${{secrets.DD_API_KEY}} > size-compressed.txt
41-
ddev size status --compressed --format png,csv,markdown
42-
cat size-compressed.txt
43-
echo "# Size (compressed)" >> $GITHUB_STEP_SUMMARY
44-
cat compressed_status.md >> $GITHUB_STEP_SUMMARY
45-
35+
ddev config override
4636
37+
- name: Measure disk usage
38+
env:
39+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
40+
run: |
41+
cmd="ddev size status \
42+
--commit ${{ github.event.workflow_run.head_sha }} \
43+
--format json"
4744
48-
- name: Measure disk usage differences from last commit (uncompressed)
49-
run: |
50-
BEFORE=$(git rev-parse HEAD^)
51-
AFTER=$(git rev-parse HEAD)
52-
ddev size diff $BEFORE $AFTER > diff-uncompressed.txt
53-
ddev size diff $BEFORE $AFTER --format png,csv,markdown
54-
cat diff-uncompressed.txt
55-
echo "# Size diff (uncompressed)" >> $GITHUB_STEP_SUMMARY
56-
if [ -f uncompressed_diff.md ]; then
57-
cat uncompressed_diff.md >> $GITHUB_STEP_SUMMARY
58-
fi
59-
60-
- name: Measure disk usage differences from last commit (compressed)
61-
run: |
62-
BEFORE=$(git rev-parse HEAD^)
63-
AFTER=$(git rev-parse HEAD)
64-
ddev size diff $BEFORE $AFTER --compressed > diff-compressed.txt
65-
ddev size diff $BEFORE $AFTER --compressed --format png,csv,markdown
66-
cat diff-compressed.txt
67-
echo "# Size diff (compressed)" >> $GITHUB_STEP_SUMMARY
68-
if [ -f compressed_diff.md ]; then
69-
cat compressed_diff.md >> $GITHUB_STEP_SUMMARY
45+
# Send metrics to Datadog only on push to master
46+
if [ "${{ github.event.workflow_run.event }}" = "push" ] && [ "${{ github.ref_name }}" = "master" ]; then
47+
cmd="$cmd --to-dd-key ${{ secrets.DD_API_KEY }}"
7048
fi
49+
50+
$cmd
7151
72-
- name: Upload file sizes (uncompressed)
73-
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
74-
with:
75-
name: uncompressed_status.csv
76-
path: uncompressed_status.csv
77-
if-no-files-found: error
52+
$cmd --compressed
7853
79-
- name: Upload file sizes (compressed)
54+
- name: Upload JSON uncompressed sizes artifact
8055
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
8156
with:
82-
name: compressed_status.csv
83-
path: compressed_status.csv
57+
name: status_uncompressed.json
58+
path: status_uncompressed.json
8459
if-no-files-found: error
8560

86-
- name: Upload file sizes diff (uncompressed)
87-
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
88-
with:
89-
name: uncompressed_diff.csv
90-
path: uncompressed_diff.csv
91-
if-no-files-found: warn
92-
93-
- name: Upload file sizes diff (compressed)
94-
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
95-
with:
96-
name: compressed_diff.csv
97-
path: compressed_diff.csv
98-
if-no-files-found: warn
99-
100-
- name: Upload status PNGs
101-
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
102-
with:
103-
name: size-visuals
104-
path: size_status_visualizations/
105-
if-no-files-found: error
106-
107-
- name: Upload diff PNGs
61+
- name: Upload JSON compressed sizes artifact
10862
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
10963
with:
110-
name: diff-visuals
111-
path: size_diff_visualizations/
112-
if-no-files-found: warn
113-
64+
name: status_compressed.json
65+
path: status_compressed.json
66+
if-no-files-found: error

0 commit comments

Comments
 (0)