feat: save trusted packages in the repo (#314)

Author: sdn4z

.github/workflows/weekly_download.yml

@@ -0,0 +1,38 @@
+name: Scheduled Pipeline
+
+on:
+  schedule:
+    - cron: "0 0 * * 1"  # every Monday at 00:00 UTC
+  workflow_dispatch:
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+        - uses: actions/create-github-app-token@a8d616148505b5069dccd32f177bb87d7f39123b # v2.1.1
+          id: app-token
+          with:
+            app-id: ${{ vars.ELEMENTSINTERACTIVE_BOT_APP_ID }}
+            private-key: ${{ secrets.ELEMENTSINTERACTIVE_BOT_PRIVATE_KEY }}
+        - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+          with:
+            fetch-depth: 0
+            token: ${{ steps.app-token.outputs.token }}
+            ref: ${{ github.head_ref }}
+        - name: Install uv
+          uses: astral-sh/setup-uv@557e51de59eb14aaaba2ed9621916900a91d50c6 # v6.6.1
+
+        - name: Install the project
+          run: uv sync --locked --only-group download
+
+        - name: Download packages from trusted sources
+          run: |
+            uv run dependencies/scripts/download_packages.py run --ecosystem pypi 
+            uv run dependencies/scripts/download_packages.py run --ecosystem npm 
+        
+        - name: Push changes to repo
+          run: |
+            git add .
+            git commit -m "chore: Weekly update of trusted packages"
+            git push origin HEAD:main

dependencies/npm.json

@@ -0,0 +1,64 @@
+import json
+from pathlib import Path
+from typing import Any
+
+import click
+import httpx
+import stamina
+
+
+def parse_npm(data: list[dict[str, Any]]) -> list[str]:
+    return [x["name"] for x in data]
+
+
+def parse_pypi(data: dict[str, Any]) -> list[str]:
+    return [row["project"] for row in data["rows"]]
+
+
+ECOSYSTEMS = {
+    "npm": {
+        "url": "https://packages.ecosyste.ms/api/v1/registries/npmjs.org/packages?per_page=10000&page=1&sort=downloads",
+        "parser": parse_npm,
+    },
+    "pypi": {
+        "url": "https://hugovk.github.io/top-pypi-packages/top-pypi-packages.min.json",
+        "parser": parse_pypi,
+    },
+}
+
+
+@click.group()
+def entry_point() -> None:
+    pass
+
+
+@entry_point.command()
+@click.option(
+    "--ecosystem",
+    type=str,
+    required=True,
+    help="Package ecosystem to download packages from.",
+)
+def run(ecosystem: str) -> None:
+    for attempt in stamina.retry_context(
+        on=(
+            httpx.TransportError,
+            httpx.TimeoutException,
+        ),
+        attempts=3,
+        timeout=60,
+    ):
+        with attempt, httpx.Client() as client:
+            response = client.get(ECOSYSTEMS[ecosystem]["url"])  # type: ignore[arg-type]
+        response.raise_for_status()
+
+    fpath = Path("dependencies") / f"{ecosystem}.json"
+
+    data = ECOSYSTEMS[ecosystem]["parser"](response.json())  # type: ignore[operator]
+
+    with open(str(fpath), "w") as fp:
+        json.dump(data, fp)
+
+
+if __name__ == "__main__":
+    entry_point()

dependencies/pypi.json

@@ -5,7 +5,7 @@ run := "uv run"
 
 venv-exists := path_exists(venv)
 
-target_dirs := "src tests"
+target_dirs := "src tests dependencies"
 
 # ALIASES
 alias t := test
@@ -53,3 +53,6 @@ build: venv
 # Install package in development mode
 install-dev: venv
     uv pip install -e .
+
+download ecosystem: venv
+    uv run dependencies/scripts/download_packages.py run --ecosystem {{ecosystem}}

dependencies/scripts/download_packages.py

@@ -50,6 +50,11 @@ dev = [
     "freezegun>=1.5.5",
     "types-pyyaml>=6.0.12.20250822",
 ]
+download = [
+    "click>=8.1.8",
+    "httpx>=0.28.1",
+    "stamina>=25.1.0",
+]
 local = ["ipdb<1.0.0,>=0.13.9", "commitizen<5.0,>=2.38", "pdbpp<1.0.0,>=0.11.6"]