Move artifact and size JSONs to temp directory (DataDog#21496)

* temp dir

* changelog

* add && to command

* echos

* test

* test

* Revert testing changes

* test

* Revert testing changes

* change changelog to fixed

* test

* Revert testing for suggested changes

* test

* test getting previous artifact

* Revert testing changes

* Add version to file_path

Author: lucia-sb

.github/workflows/measure-disk-usage.yml

@@ -34,22 +34,32 @@ jobs:
       run: |
             ddev config override
 
-    - name: Measure disk usage
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Define command
+      id: cmd
       run: |
         cmd="ddev size status \
         --commit ${{ github.event.workflow_run.head_sha }} \
         --format json"
 
-        # Send metrics to Datadog only on push to master
         if [ "${{ github.event.workflow_run.event }}" = "push" ] && [ "${{ github.ref_name }}" = "master" ]; then 
           cmd="$cmd --to-dd-key ${{ secrets.DD_API_KEY }}"
         fi
-        
-        $cmd
 
-        $cmd --compressed
+        echo "cmd=$cmd" >> $GITHUB_OUTPUT
+
+    - name: Measure disk usage (Uncompressed)
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        echo "Running for uncompressed sizes"
+        ${{ steps.cmd.outputs.cmd }}
+        
+    - name: Measure disk usage (Compressed)
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        echo "Running for compressed sizes"
+        ${{ steps.cmd.outputs.cmd }} --compressed
 
     - name: Upload JSON uncompressed sizes artifact
       uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2

ddev/changelog.d/21496.fixed

@@ -0,0 +1 @@
+The `ddev size status` now stores temporary files in a temporary directory that is removed when the commands finishes. This prevents littering the local disk with unnecessary files.

ddev/src/ddev/cli/size/status.py

@@ -78,7 +78,8 @@ def status(
             if commit:
                 from ddev.cli.size.utils.common_funcs import get_last_dependency_sizes_artifact
 
-                dependency_sizes = get_last_dependency_sizes_artifact(app, commit, plat)
+                dependency_sizes = get_last_dependency_sizes_artifact(app, commit, plat, ver, compressed)
+
             parameters: CLIParameters = {
                 "app": app,
                 "platform": plat,

ddev/src/ddev/cli/size/utils/common_funcs.py

@@ -954,34 +954,31 @@ def get_last_commit_data() -> tuple[str, list[str], list[str]]:
 
 
 @cache
-def get_last_dependency_sizes_artifact(app: Application, commit: str, platform: str) -> Path | None:
+def get_last_dependency_sizes_artifact(
+    app: Application, commit: str, platform: str, py_version: str, compressed: bool
+) -> Path | None:
     '''
     Lockfiles of dependencies are not updated in the same commit as the dependencies are updated.
     So in each commit, there is an artifact with the sizes of the wheels that were built to get the actual
     size of that commit.
     '''
-    dep_sizes_json = get_dep_sizes_json(commit, platform)
+    dep_sizes_json = get_dep_sizes_json(commit, platform, py_version)
     if not dep_sizes_json:
-        previous_sizes = get_previous_dep_sizes_json(app.repo.git.merge_base(commit, "origin/master"))
-        if previous_sizes:
-            compressed_json, uncompressed_json = previous_sizes["compressed"], previous_sizes["uncompressed"]
-            sizes = parse_sizes_json(compressed_json, uncompressed_json, platform)
-            with open(f"{platform}.json", "w") as f:
-                json.dump(sizes, f, indent=2)
-            dep_sizes_json = Path(f"{platform}.json")
-    return dep_sizes_json if dep_sizes_json else None
+        dep_sizes_json = get_previous_dep_sizes(
+            app.repo.git.merge_base(commit, "origin/master"), platform, py_version, compressed
+        )
+    return Path(dep_sizes_json) if dep_sizes_json else None
 
 
 @cache
-def get_dep_sizes_json(current_commit: str, platform: str) -> Path | None:
+def get_dep_sizes_json(current_commit: str, platform: str, py_version: str) -> Path | None:
     '''
     Gets the dependency sizes json for a given commit and platform when dependencies were resolved.
     '''
     print(f"Getting dependency sizes json for commit: {current_commit}, platform: {platform}")
     run_id = get_run_id(current_commit, RESOLVE_BUILD_DEPS_WORKFLOW)
     if run_id:
-        dep_sizes_json = get_current_sizes_json(run_id, platform)
-        print(f"Dependency sizes json path: {dep_sizes_json}")
+        dep_sizes_json = get_current_sizes_json(run_id, platform, py_version)
         return dep_sizes_json
     else:
         return None
@@ -1019,7 +1016,7 @@ def get_run_id(commit: str, workflow: str) -> str | None:
 
 
 @cache
-def get_current_sizes_json(run_id: str, platform: str) -> Path | None:
+def get_current_sizes_json(run_id: str, platform: str, py_version: str) -> Path | None:
     '''
     Downloads the dependency sizes json for a given run id and platform when dependencies were resolved.
     '''
@@ -1059,79 +1056,85 @@ def get_current_sizes_json(run_id: str, platform: str) -> Path | None:
             return None
 
         print(f"Found sizes artifact at {sizes_file}")
-        dest_path = sizes_file.rename(f"{platform}.json")
+        dest_path = sizes_file.rename(f"{platform}_{py_version}.json")
         return dest_path
 
 
 @cache
-def get_artifact(run_id: str, artifact_name: str) -> Path | None:
+def get_artifact(run_id: str, artifact_name: str, target_dir: str | None = None) -> Path | None:
     print(f"Downloading artifact: {artifact_name} from run_id={run_id}")
     try:
-        subprocess.run(
-            [
-                'gh',
-                'run',
-                'download',
-                run_id,
-                '--name',
-                artifact_name,
-            ],
-            check=True,
-            text=True,
-        )
+        cmd = [
+            'gh',
+            'run',
+            'download',
+            run_id,
+            '--name',
+            artifact_name,
+        ]
+        if target_dir:
+            cmd.extend(['--dir', target_dir])
+
+        subprocess.run(cmd, check=True, text=True)
     except subprocess.CalledProcessError as e:
         print(f"Failed to download artifact: {artifact_name} from run_id={run_id}: {e}")
         return None
 
-    print(f"Artifact downloaded to: {artifact_name}")
-    return Path(artifact_name)
+    artifact_path = Path(target_dir) / artifact_name if target_dir else Path(artifact_name)
+    print(f"Artifact downloaded to: {artifact_path}")
+    return artifact_path
 
 
 @cache
-def get_previous_dep_sizes_json(base_commit: str) -> dict[str, Path] | None:
+def get_previous_dep_sizes(base_commit: str, platform: str, py_version: str, compressed: bool) -> Path | None:
     '''
     Gets the dependency sizes for a given commit when dependencies were not resolved.
     '''
-    print(f"Getting previous dependency sizes json for {base_commit=}")
-    run_id = get_run_id(base_commit, MEASURE_DISK_USAGE_WORKFLOW)
-    print(f"Previous run_id: {run_id}")
-    compressed_json = None
-    uncompressed_json = None
-    if run_id:
-        compressed_json = get_artifact(run_id, 'status_compressed.json')
-    if run_id:
-        uncompressed_json = get_artifact(run_id, 'status_uncompressed.json')
-    print(f"Compressed json: {compressed_json}")
-    print(f"Uncompressed json: {uncompressed_json}")
-    if not compressed_json or not uncompressed_json:
-        return None
-    return {
-        "compressed": compressed_json,
-        "uncompressed": uncompressed_json,
-    }
+    with tempfile.TemporaryDirectory() as tmpdir:
+        print(f"Getting previous dependency sizes json for {base_commit=}")
+
+        if (run_id := get_run_id(base_commit, MEASURE_DISK_USAGE_WORKFLOW)) is None:
+            return None
+
+        print(f"Previous run_id: {run_id}")
+
+        artifact_name = 'status_compressed.json' if compressed else 'status_uncompressed.json'
+        sizes_json = get_artifact(run_id, artifact_name, tmpdir)
+
+        if not sizes_json:
+            return None
+
+        print(f"Sizes json: {sizes_json}")
+
+        sizes = parse_sizes_json(sizes_json, platform, py_version, compressed)
+
+        sizes_path = Path(tmpdir) / f"{platform}_{py_version}.json"
+        with open(sizes_path, "w") as f:
+            json.dump(sizes, f, indent=2)
+
+        target_path = f"{platform}_{py_version}.json"
+        shutil.copy(sizes_path, target_path)
+        return Path(target_path)
 
 
 @cache
 def parse_sizes_json(
-    compressed_json_path: Path, uncompressed_json_path: Path, platform: str
+    sizes_json_path: Path, platform: str, py_version: str, compressed: bool
 ) -> dict[str, dict[str, int]]:
-    compressed_list = list(json.loads(compressed_json_path.read_text()))
-    uncompressed_list = list(json.loads(uncompressed_json_path.read_text()))
+    sizes_list = list(json.loads(sizes_json_path.read_text()))
+    size_key = "compressed" if compressed else "uncompressed"
     sizes = {
         dep["Name"]: {
-            "compressed": int(dep["Size_Bytes"]),
+            size_key: int(dep["Size_Bytes"]),
             "version": dep.get("Version"),
+            "compression": compressed,
         }
-        for dep in compressed_list
-        if dep.get("Type") == "Dependency" and dep.get("Platform") == platform
+        for dep in sizes_list
+        if dep.get("Type") == "Dependency"
+        and dep.get("Platform") == platform
+        and dep.get("Python_Version") == py_version
     }
 
-    for dep in uncompressed_list:
-        if dep.get("Type") == "Dependency" and dep.get("Platform") == platform:
-            name = dep["Name"]
-            entry = sizes.setdefault(name, {"version": dep.get("Version")})
-            entry["uncompressed"] = int(dep["Size_Bytes"])
-
     return sizes
 
 

ddev/tests/size/test_common.py

@@ -384,61 +384,38 @@ def test_parse_sizes_json(tmp_path):
                 "Size": "2 B",
                 "Type": "Dependency",
                 "Platform": "linux-x86_64",
+                "Python_Version": "3.12",
             },
             {
                 "Name": "dep2",
                 "Size_Bytes": 123,
                 "Size": "2 B",
                 "Type": "Dependency",
                 "Platform": "macos-x86_64",
+                "Python_Version": "3.12",
             },
             {
                 "Name": "module1",
                 "Size_Bytes": 123,
                 "Size": "2 B",
                 "Type": "Integration",
                 "Platform": "linux-x86_64",
-            },
-        ]
-    )
-    uncompressed_data = json.dumps(
-        [
-            {
-                "Name": "dep1",
-                "Size_Bytes": 456,
-                "Size": "4 B",
-                "Type": "Dependency",
-                "Platform": "linux-x86_64",
-            },
-            {
-                "Name": "dep2",
-                "Size_Bytes": 456,
-                "Size": "4 B",
-                "Type": "Dependency",
-                "Platform": "macos-x86_64",
-            },
-            {
-                "Name": "module1",
-                "Size_Bytes": 456,
-                "Size": "4 B",
-                "Type": "Integration",
-                "Platform": "linux-x86_64",
+                "Python_Version": "3.12",
             },
         ]
     )
 
     expected_output = {
         "dep1": {
             "compressed": 123,
-            "uncompressed": 456,
+            "compression": True,
             "version": None,
         }
     }
     compressed_json_path = tmp_path / "compressed.json"
     compressed_json_path.write_text(compressed_data)
-    uncompressed_json_path = tmp_path / "uncompressed.json"
-    uncompressed_json_path.write_text(uncompressed_data)
-    result = parse_sizes_json(compressed_json_path, uncompressed_json_path, "linux-x86_64")
+
+    result = parse_sizes_json(compressed_json_path, "linux-x86_64", "3.12", True)
 
     assert result == expected_output