refactor: Improve weekly download flow

.github/workflows/weekly_download.yml

@@ -1,4 +1,4 @@
-name: Scheduled Pipeline
+name: Weekly download
 
 on:
   schedule:
@@ -7,7 +7,7 @@ on:
 
 
 jobs:
-  build:
+  download:
     runs-on: ubuntu-latest
     steps:
         - uses: actions/create-github-app-token@a8d616148505b5069dccd32f177bb87d7f39123b # v2.1.1
@@ -28,9 +28,9 @@ jobs:
 
         - name: Download packages from trusted sources
           run: |
-            uv run dependencies/scripts/download_packages.py run --ecosystem pypi 
-            uv run dependencies/scripts/download_packages.py run --ecosystem npm 
-
+            uv run --no-project dependencies/scripts/download_packages.py download pypi || echo 'Failed to download trusted pypi packages'
+            uv run --no-project dependencies/scripts/download_packages.py donwload npm || echo 'Failed to download trusted npm packages'
+            
         - name: Push changes to repo
           run: |
             git add .

dependencies/scripts/download_packages.py

@@ -1,11 +1,21 @@
 import json
+import logging
+from datetime import datetime
 from pathlib import Path
 from typing import Any
+from zoneinfo import ZoneInfo
 
 import click
 import httpx
 import stamina
 
+logger = logging.getLogger("weekly_download")
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
 
 def parse_npm(data: list[dict[str, Any]]) -> list[str]:
     return [x["name"] for x in data]
@@ -33,32 +43,36 @@ def entry_point() -> None:
 
 
 @entry_point.command()
-@click.option(
-    "--ecosystem",
+@click.argument(
+    "ecosystem",
     type=str,
     required=True,
-    help="Package ecosystem to download packages from.",
 )
-def run(ecosystem: str) -> None:
+def download(ecosystem: str) -> None:
     for attempt in stamina.retry_context(
         on=(
             httpx.TransportError,
             httpx.TimeoutException,
         ),
         attempts=3,
-        timeout=60,
+        wait_jitter=1,
+        wait_exp_base=2,
+        wait_max=8,
     ):
-        with attempt, httpx.Client() as client:
+        with attempt, httpx.Client(timeout=30) as client:
+            logger.info("Attempting to download %s packages. Attempt #%d.", ecosystem, attempt.num)
             response = client.get(ECOSYSTEMS[ecosystem]["url"])  # type: ignore[arg-type]
-        response.raise_for_status()
+            response.raise_for_status()
 
     fpath = Path("dependencies") / f"{ecosystem}.json"
 
-    data = ECOSYSTEMS[ecosystem]["parser"](response.json())  # type: ignore[operator]
-
+    packages = ECOSYSTEMS[ecosystem]["parser"](response.json())  # type: ignore[operator]
+    data = {"date": datetime.now(ZoneInfo("UTC")).isoformat(), "packages": packages}
     with open(str(fpath), "w") as fp:
         json.dump(data, fp)
 
+    logger.info("Saved `%s` file.", fpath)
+
 
 if __name__ == "__main__":
     entry_point()

justfile

@@ -55,4 +55,4 @@ install-dev: venv
     uv pip install -e .
 
 download ecosystem: venv
-    uv run dependencies/scripts/download_packages.py run --ecosystem {{ecosystem}}
+    uv run --no-project dependencies/scripts/download_packages.py download {{ecosystem}}