[pull] master from DataDog:master

.builders/build.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import json
 import os
 import shutil
 import subprocess
@@ -153,6 +154,10 @@ def build_macos():
         final_requirements = mount_dir / 'frozen.txt'
         shutil.move(final_requirements, output_dir)
 
+        # Move the dependency sizes to the output directory
+        dependency_sizes_dir = mount_dir / 'sizes.json'
+        shutil.move(dependency_sizes_dir, output_dir)
+
 
 def build_image():
     parser = argparse.ArgumentParser(prog='builder', allow_abbrev=False)
@@ -246,6 +251,9 @@ def build_image():
             # Move the final requirements file to the output directory
             shutil.move(final_requirements, output_dir)
 
+            # Move the dependency sizes to the output directory
+            shutil.move(mount_dir / 'sizes.json', output_dir)
+
 
 def main():
     if sys.platform == 'darwin':

.builders/scripts/build_wheels.py

@@ -1,11 +1,14 @@
 from __future__ import annotations
 
 import argparse
+import json
 import os
 import subprocess
 import sys
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from typing import TypedDict
+from zipfile import ZipFile
 
 from dotenv import dotenv_values
 from utils import extract_metadata, normalize_project_name
@@ -14,6 +17,10 @@
 CUSTOM_EXTERNAL_INDEX = f'{INDEX_BASE_URL}/external'
 CUSTOM_BUILT_INDEX = f'{INDEX_BASE_URL}/built'
 
+class WheelSizes(TypedDict):
+    compressed: int
+    uncompressed: int
+
 if sys.platform == 'win32':
     PY3_PATH = Path('C:\\py3\\Scripts\\python.exe')
     PY2_PATH = Path('C:\\py2\\Scripts\\python.exe')
@@ -55,6 +62,13 @@ def check_process(*args, **kwargs) -> subprocess.CompletedProcess:
     return process
 
 
+def calculate_wheel_sizes(wheel_path: Path) -> WheelSizes:
+    compressed_size = wheel_path.stat(follow_symlinks=True).st_size
+    with ZipFile(wheel_path) as zf:
+        uncompressed_size = sum(zinfo.file_size for zinfo in zf.infolist())
+    return {'compressed': compressed_size, 'uncompressed': uncompressed_size}
+
+
 def main():
     parser = argparse.ArgumentParser(prog='wheel-builder', allow_abbrev=False)
     parser.add_argument('--python', required=True)
@@ -109,9 +123,14 @@ def main():
 
         # Fetch or build wheels
         command_args = [
-            str(python_path), '-m', 'pip', 'wheel',
-            '-r', str(MOUNT_DIR / 'requirements.in'),
-            '--wheel-dir', str(staged_wheel_dir),
+            str(python_path),
+            '-m',
+            'pip',
+            'wheel',
+            '-r',
+            str(MOUNT_DIR / 'requirements.in'),
+            '--wheel-dir',
+            str(staged_wheel_dir),
             # Temporarily removing extra index urls. See below.
             # '--extra-index-url', CUSTOM_EXTERNAL_INDEX,
         ]
@@ -124,21 +143,37 @@ def main():
         check_process(command_args, env=env_vars)
 
         # Repair wheels
-        check_process([
-            sys.executable, '-u', str(MOUNT_DIR / 'scripts' / 'repair_wheels.py'),
-            '--source-dir', str(staged_wheel_dir),
-            '--built-dir', str(built_wheels_dir),
-            '--external-dir', str(external_wheels_dir),
-        ])
+        check_process(
+            [
+                sys.executable,
+                '-u',
+                str(MOUNT_DIR / 'scripts' / 'repair_wheels.py'),
+                '--source-dir',
+                str(staged_wheel_dir),
+                '--built-dir',
+                str(built_wheels_dir),
+                '--external-dir',
+                str(external_wheels_dir),
+            ]
+        )
 
     dependencies: dict[str, tuple[str, str]] = {}
+    sizes: dict[str, WheelSizes] = {}
+
     for wheel_dir in wheels_dir.iterdir():
-        for entry in wheel_dir.iterdir():
-            project_metadata = extract_metadata(entry)
+        for wheel in wheel_dir.iterdir():
+            project_metadata = extract_metadata(wheel)
             project_name = normalize_project_name(project_metadata['Name'])
             project_version = project_metadata['Version']
             dependencies[project_name] = project_version
 
+
+            sizes[project_name] = {'version': project_version, **calculate_wheel_sizes(wheel)}
+
+    output_path = MOUNT_DIR / 'sizes.json'
+    with output_path.open('w', encoding='utf-8') as fp:
+        json.dump(sizes, fp, indent=2, sort_keys=True)
+
     final_requirements = MOUNT_DIR / 'frozen.txt'
     with final_requirements.open('w', encoding='utf-8') as f:
         for project_name, project_version in sorted(dependencies.items()):

.github/workflows/measure-disk-usage.yml

@@ -1,113 +1,66 @@
 name: Measure Disk Usage
 
 on:
-  push:
-    branches:
-    - master 
+  workflow_run:
+    workflows: ['Resolve Dependencies and Build Wheels']
+    types:
+      - completed
 env:
   PYTHON_VERSION: "3.13"
 
 jobs:
+
   measure-disk-usage:
     runs-on: ubuntu-22.04
+    permissions:
+        contents: read
+        actions: read
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         fetch-depth: 0
+
     - name: Set up Python ${{ env.PYTHON_VERSION }}
       uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
       with:
         python-version: ${{ env.PYTHON_VERSION }}
+
     - name: Install ddev
       run: |
         pip install -e ./datadog_checks_dev[cli]
         pip install -e ./ddev
 
     - name: Configure ddev
       run: |
-        ddev config set repos.core .
-        ddev config set repo core
-    - name: Measure disk usage (uncompressed)
-      run: | 
-        ddev size status --to-dd-key ${{secrets.DD_API_KEY}} > size-uncompressed.txt 
-        ddev size status  --format png,csv,markdown
-        cat size-uncompressed.txt
-        echo "# Size (uncompressed)" >> $GITHUB_STEP_SUMMARY
-        cat uncompressed_status.md >> $GITHUB_STEP_SUMMARY
-
-    - name: Measure disk usage (compressed)
-      run: |
-        ddev size status --compressed --to-dd-key ${{secrets.DD_API_KEY}} > size-compressed.txt 
-        ddev size status --compressed  --format png,csv,markdown
-        cat size-compressed.txt
-        echo "# Size (compressed)" >> $GITHUB_STEP_SUMMARY
-        cat compressed_status.md >> $GITHUB_STEP_SUMMARY
-
+            ddev config override
 
+    - name: Measure disk usage
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        cmd="ddev size status \
+        --commit ${{ github.event.workflow_run.head_sha }} \
+        --format json"
 
-    - name: Measure disk usage differences from last commit (uncompressed)
-      run: | 
-        BEFORE=$(git rev-parse HEAD^)
-        AFTER=$(git rev-parse HEAD)
-        ddev size diff $BEFORE $AFTER > diff-uncompressed.txt
-        ddev size diff $BEFORE $AFTER  --format png,csv,markdown
-        cat diff-uncompressed.txt
-        echo "# Size diff (uncompressed)" >> $GITHUB_STEP_SUMMARY
-        if [ -f uncompressed_diff.md ]; then
-            cat uncompressed_diff.md >> $GITHUB_STEP_SUMMARY
-        fi
-
-    - name: Measure disk usage differences from last commit (compressed)
-      run: | 
-        BEFORE=$(git rev-parse HEAD^)
-        AFTER=$(git rev-parse HEAD)
-        ddev size diff $BEFORE $AFTER --compressed > diff-compressed.txt
-        ddev size diff $BEFORE $AFTER --compressed  --format png,csv,markdown
-        cat diff-compressed.txt
-        echo "# Size diff (compressed)" >> $GITHUB_STEP_SUMMARY
-        if [ -f compressed_diff.md ]; then
-            cat compressed_diff.md >> $GITHUB_STEP_SUMMARY
+        # Send metrics to Datadog only on push to master
+        if [ "${{ github.event.workflow_run.event }}" = "push" ] && [ "${{ github.ref_name }}" = "master" ]; then 
+          cmd="$cmd --to-dd-key ${{ secrets.DD_API_KEY }}"
         fi
+
+        $cmd
 
-    - name: Upload file sizes (uncompressed)
-      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-      with:
-        name: uncompressed_status.csv
-        path: uncompressed_status.csv
-        if-no-files-found: error
+        $cmd --compressed
 
-    - name: Upload file sizes (compressed)
+    - name: Upload JSON uncompressed sizes artifact
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
       with:
-        name: compressed_status.csv
-        path: compressed_status.csv
+        name: status_uncompressed.json
+        path: status_uncompressed.json
         if-no-files-found: error
 
-    - name: Upload file sizes diff (uncompressed)
-      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-      with:
-        name: uncompressed_diff.csv
-        path: uncompressed_diff.csv
-        if-no-files-found: warn
-
-    - name: Upload file sizes diff (compressed)
-      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-      with:
-        name: compressed_diff.csv
-        path: compressed_diff.csv
-        if-no-files-found: warn
-
-    - name: Upload status PNGs
-      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
-      with:
-        name: size-visuals
-        path: size_status_visualizations/
-        if-no-files-found: error
-
-    - name: Upload diff PNGs
+    - name: Upload JSON compressed sizes artifact
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
       with:
-        name: diff-visuals
-        path: size_diff_visualizations/
-        if-no-files-found: warn
-
+        name: status_compressed.json
+        path: status_compressed.json
+        if-no-files-found: error