WIP: fetcher plugin for CI downloads

.buildkite/.gitignore

@@ -0,0 +1 @@
+/fetcher/.pdm.toml

.buildkite/fetcher/README.md

@@ -0,0 +1,3 @@
+# pdm-plugin-fetcher
+
+Generates a list of files to download for CI builds to help maintain an external file-based cache.

.buildkite/fetcher/pyproject.toml

@@ -0,0 +1,46 @@
+[project]
+name = "pdm-plugin-fetcher"
+version = "0.1.2"
+description = "A plugin to make builds faster on CI"
+authors = [
+    {name = "Embark Studios", email = "python@embark-studios.com"},
+]
+requires-python = ">=3.8"
+readme = "README.md"
+license = {text = "MIT"}
+dependencies = []
+
+[project.urls]
+repository = "https://github.com/EmbarkStudios/emote"
+
+[build-system]
+requires = ["pdm-backend>=1.0.0"]
+build-backend = "pdm.backend"
+
+[project.entry-points.pdm]
+pdm_plugin_fetcher = "pdm_plugin_fetcher.main:fetcher_plugin"
+
+[tool.pdm.build]
+package-dir = "src"
+includes = ["src/pdm_plugin_fetcher"]
+source-includes = ["tests", "CHANGELOG.md", "LICENSE", "README.md"]
+
+[tool.isort]
+py_version = 38
+profile = "black"
+combine_as_imports = true
+lines_between_types = 1
+lines_after_imports = 2
+src_paths = ["src"]
+
+[tool.black]
+target-version = ['py38']
+
+[tool.mypy]
+check_untyped_defs = true
+ignore_missing_imports = true
+show_error_codes = true
+warn_redundant_casts = true
+warn_unused_configs = true
+warn_unused_ignores = true
+files = "src"

.buildkite/fetcher/src/pdm_plugin_fetcher/__init__.py

@@ -0,0 +1,3 @@
+"""
+
+"""

.buildkite/fetcher/src/pdm_plugin_fetcher/main.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from pdm.cli.actions import resolve_candidates_from_lockfile
+from pdm.cli.commands.base import BaseCommand
+from pdm.cli.filters import GroupSelection
+from pdm.cli.options import (
+    clean_group,
+    dry_run_option,
+    groups_group,
+    install_group,
+    lockfile_option,
+    skip_option,
+    venv_option,
+)
+from pdm.core import Core
+from pdm.models.caches import SafeFileCache
+from pdm.models.candidates import _find_best_match_link
+
+
+class FetcherCommand(BaseCommand):
+    """Generate a lockfile for torch specifically."""
+
+    arguments = (
+        *BaseCommand.arguments,
+        groups_group,
+        dry_run_option,
+        lockfile_option,
+        skip_option,
+        clean_group,
+        install_group,
+        venv_option,
+    )
+
+    def handle(self, project, options):
+        selection = GroupSelection.from_options(project, options)
+        requirements = []
+        for group in selection:
+            requirements.extend(project.get_dependencies(group).values())
+        candidates = resolve_candidates_from_lockfile(project, requirements)
+
+        with project.core.ui.open_spinner("Preparing candidates..."):
+            prepared = [
+                candidate.prepare(project.environment)
+                for candidate in candidates.values()
+            ]
+
+        sfc = SafeFileCache(project.cache("http"))
+
+        links = []
+        with project.core.ui.open_spinner("Resolving links..."):
+            with project.environment.get_finder() as finder:
+                for c in prepared:
+                    link = _find_best_match_link(
+                        finder,
+                        c.req.as_pinned_version(c.candidate.version),
+                        c.candidate.hashes,
+                        ignore_compatibility=False,
+                    ).url
+
+                    links.append(
+                        {
+                            "link": link,
+                            "path": sfc._get_cache_path(link),
+                        }
+                    )
+
+        import json
+
+        print(json.dumps({"links": links}, indent=4))
+
+
+def fetcher_plugin(core: Core):
+    core.register_command(FetcherCommand, "fetch")

.buildkite/install-repo.sh

@@ -1,12 +1,16 @@
 set -eo pipefail
 
+
 echo --- Setting up google-cloud-sdk
 
 if [ -f '/gcloud/google-cloud-sdk/path.bash.inc' ]; then . '/gcloud/google-cloud-sdk/path.bash.inc'; fi
 gcloud config set account monorepo-ci@embark-builds.iam.gserviceaccount.com
 
 echo --- Installing dependencies
 
+${PDM_COMMAND:1:-1} plugin add .buildkite/fetcher
+${PDM_COMMAND:1:-1} fetch -G ci | .buildkite/pdm-fetcher downsync > manifest.json
 ${PDM_COMMAND:1:-1} install -d -G ci -k post_install
 ${PDM_COMMAND:1:-1} plugin add pdm-plugin-torch>=23.1.1
 ${PDM_COMMAND:1:-1} torch install cpu
+cat manifest.json | .buildkite/pdm-fetcher upsync

.buildkite/pipeline.yml

@@ -36,9 +36,9 @@ large: &large
       resources-limit-cpu: 14
       resources-limit-memory: 35Gi
 
-
 env:
   PDM_COMMAND: pdm25
+  RUST_BACKTRACE: 1
 
 steps:
   - group: ":passport_control: Validating PR"