support read only buildcaches; set defaults on mirrors.yaml inputs

Author: bcumming

docs/build-caches.md

@@ -46,11 +46,21 @@ buildcache:
 | Field | Required | Description |
 |-------|----------|-------------|
 | `url`            | yes | location of the cache (a `file://` path, or an `http(s)://`, `s3://` or `oci://` URL) |
-| `private_key`    | yes | PGP key used to sign and push packages (see [Keys](#keys)) |
+| `private_key`    | no  | PGP key used to sign and push packages (see [Keys](#keys)); omit for a read-only cache |
 | `public_key`     | no  | PGP key used to verify downloaded packages |
 | `name`           | no  | name Spack registers the mirror under (default `buildcache`) |
 | `mount_specific` | no  | store the cache in a per-mount-point sub-directory (default `false`) |
 
+### Read-only build cache
+
+Omit `private_key` to configure a read-only cache: Spack fetches packages from it but never signs or pushes anything back.
+This is useful for consuming a shared team cache that you are not permitted to write to.
+
+```yaml title="mirrors.yaml"
+buildcache:
+  url: file:///capstor/scratch/team/uenv-cache
+```
+
 ### `mount_specific`
 
 Spack binaries embed the install prefix (the image's mount point), so binaries built for `/user-environment` cannot be reused at a different mount point.

stackinator/mirror.py

@@ -36,9 +36,10 @@ class Mirrors:
 
     The kinds of mirror have separate types:
 
-      * buildcache     - at most one, signs and stores built packages (so it alone
-                         has a private key and the mount_specific flag). None if no
-                         build cache is configured.
+      * buildcache     - at most one, fetches and stores built packages (so it alone
+                         has the mount_specific flag). With a private key it signs
+                         and pushes packages too; without one it is read-only. None
+                         if no build cache is configured.
       * bootstrap      - at most one, used to bootstrap spack itself. None if absent.
       * source_mirrors - a name -> config mapping of any number of read-only source
                          mirrors (spack mirrors.yaml entries).
@@ -98,12 +99,9 @@ def __init__(
         except ValueError as err:
             raise MirrorError(f"Mirror config does not comply with schema.\n{err}")
 
-        # The build cache, if one is defined in mirrors.yaml.
-        buildcache = raw_mirrors.get("buildcache")
-        if buildcache:
-            if not buildcache.get("private_key"):
-                raise MirrorError("Mirror build cache config is missing a required 'private_key' path.")
-            self.buildcache = buildcache
+        # The build cache, if one is defined in mirrors.yaml. A build cache without
+        # a private_key is read-only: spack fetches from it but never pushes to it.
+        self.buildcache = raw_mirrors.get("buildcache")
 
         # A build cache passed via the deprecated cache.yaml file (the --cache CLI
         # option) takes precedence over a buildcache defined in mirrors.yaml.
@@ -122,16 +120,16 @@ def __init__(
             except ValueError as err:
                 raise MirrorError(f"Error validating contents of cache config at '{cmdline_cache}'.\n{err}")
 
+            # a cache.yaml without a key configures a read-only (fetch-only) cache
             self.buildcache = {
                 "name": "buildcache",
                 "url": raw_cache["root"],
                 "description": "Buildcache dest loaded from legacy cache.yaml",
+                "public_key": None,
+                "private_key": raw_cache.get("key"),
                 "mount_specific": True,
                 "cmdline": True,
             }
-            # a cache.yaml without a key configures a read-only (fetch-only) cache
-            if raw_cache.get("key") is not None:
-                self.buildcache["private_key"] = raw_cache["key"]
             self._logger.warning(
                 "Configuring the buildcache from the system cache.yaml file.\n"
                 "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n"
@@ -142,7 +140,7 @@ def __init__(
         # The bootstrap mirror, the read-only source mirrors, and the writable
         # source cache, if any are defined.
         self.bootstrap = raw_mirrors.get("bootstrap")
-        self.source_mirrors = dict(raw_mirrors.get("sourcemirror", {}))
+        self.source_mirrors = dict(raw_mirrors["sourcemirror"])
         self.source_cache = raw_mirrors.get("sourcecache")
 
         # Validate that every mirror url is well-formed (see _validate_url).
@@ -167,15 +165,15 @@ def __init__(
 
         # A build cache that pushes packages signs them with its private key. A build
         # cache without a key is read-only, and is fetched from but never pushed to.
-        if self.buildcache is not None and self.buildcache.get("private_key"):
+        if self.buildcache is not None and self.buildcache["private_key"] is not None:
             name = self.buildcache["name"]
             self._key_files.append(
                 (key_store / f"{name}.priv.gpg", self._read_key(self.buildcache["private_key"], name))
             )
 
         # Any mirror may provide a public key, used to verify downloaded packages.
         for name, mirror in self._iter_mirrors():
-            public_key = mirror.get("public_key")
+            public_key = mirror["public_key"]
             if public_key is not None:
                 self._key_files.append((key_store / f"{name}.pub.gpg", self._read_key(public_key, name)))
 
@@ -197,7 +195,7 @@ def push_to_build_cache(self) -> Optional[str]:
         is read-only - fetched from but never pushed to.
         """
 
-        if self.buildcache is not None and self.buildcache.get("private_key"):
+        if self.buildcache is not None and self.buildcache["private_key"] is not None:
             return self.buildcache["name"]
         return None
 
@@ -307,7 +305,7 @@ def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]:
                 url = url.rstrip("/") + "/" + self._mount_path.as_posix().lstrip("/")
             entry = {"fetch": {"url": url}}
             # only a build cache with a signing key is pushed to
-            if self.buildcache.get("private_key"):
+            if self.buildcache["private_key"] is not None:
                 entry["push"] = {"url": url}
             spack_mirrors["mirrors"][self.buildcache["name"]] = entry
 

stackinator/schema/mirror.json

@@ -5,9 +5,9 @@
         "bootstrap": {
             "type": "object",
             "properties": {
-                "description": {"type": "string"},
+                "description": {"type": "string", "default": ""},
                 "url": {"type": "string"},
-                "public_key": {"type": "string"}
+                "public_key": {"type": ["string", "null"], "default": null}
             },
             "additionalProperties": false,
             "required": ["url"]
@@ -19,10 +19,10 @@
                     "type": "string",
                     "default": "buildcache"
                 },
-                "description": {"type": "string"},
+                "description": {"type": "string", "default": ""},
                 "url": {"type": "string"},
-                "public_key": {"type": "string"},
-                "private_key": {"type": "string"},
+                "public_key": {"type": ["string", "null"], "default": null},
+                "private_key": {"type": ["string", "null"], "default": null},
                 "mount_specific": {
                     "type": "boolean",
                     "default": false
@@ -33,16 +33,17 @@
                 }
             },
             "additionalProperties": false,
-            "required": ["url", "private_key"]
+            "required": ["url"]
         },
         "sourcemirror": {
             "type": "object",
+            "default": {},
             "additionalProperties": {
                 "type": "object",
                 "properties": {
-                    "description": {"type": "string"},
+                    "description": {"type": "string", "default": ""},
                     "url": {"type": "string"},
-                    "public_key": {"type": "string"}
+                    "public_key": {"type": ["string", "null"], "default": null}
                 },
                 "additionalProperties": false,
                 "required": ["url"]
@@ -51,7 +52,7 @@
         "sourcecache": {
             "type": "object",
             "properties": {
-                "description": {"type": "string"},
+                "description": {"type": "string", "default": ""},
                 "path": {"type": "string"}
             },
             "additionalProperties": false,

unittests/data/systems/mirror-readonly-cache/mirrors.yaml

@@ -0,0 +1,2 @@
+buildcache:
+  url: https://mirror.spack.io

unittests/test_mirrors.py

@@ -27,25 +27,31 @@ def test_mirror_init(systems_path, mount_path):
     with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file:
         pub_key_b64 = base64.b64encode(pub_key_file.read()).decode()
 
-    # the build cache, with its schema-defaulted name and flags
+    # the build cache, with its schema-defaulted name, flags and (empty) fields
     assert mirrors_obj.buildcache == {
         "name": "buildcache",
+        "description": "",
         "url": "https://mirror.spack.io",
         "private_key": "../../test-gpg-priv.asc",
         "public_key": pub_key_b64,
         "mount_specific": False,
         "cmdline": False,
     }
 
-    assert mirrors_obj.bootstrap == {"url": "https://mirror.spack.io"}
+    assert mirrors_obj.bootstrap == {
+        "url": "https://mirror.spack.io",
+        "description": "",
+        "public_key": None,
+    }
 
+    # non-required fields are always present, defaulted to "" / None by the schema
     assert mirrors_obj.source_mirrors == {
-        "mirror1": {"url": "https://github.com", "public_key": "../../test-gpg-pub.asc"},
-        "mirror2": {"url": "https://github.com/spack"},
+        "mirror1": {"url": "https://github.com", "public_key": "../../test-gpg-pub.asc", "description": ""},
+        "mirror2": {"url": "https://github.com/spack", "public_key": None, "description": ""},
     }
 
     # the writable, populate-as-you-go source cache
-    assert mirrors_obj.source_cache == {"path": "/scratch/spack-sources"}
+    assert mirrors_obj.source_cache == {"path": "/scratch/spack-sources", "description": ""}
 
     # the build cache mirror name is derived from the build cache's 'name' field
     assert mirrors_obj.build_cache_mirror == "buildcache"
@@ -92,7 +98,7 @@ def test_keyless_command_line_cache(tmp_path, systems_path, mount_path):
 
     # the cache exists (so it is fetched from), but has no signing key ...
     assert mirrors.build_cache_mirror == "buildcache"
-    assert "private_key" not in mirrors.buildcache
+    assert mirrors.buildcache["private_key"] is None
 
     # ... so it is never pushed to
     assert mirrors.push_to_build_cache is None
@@ -107,6 +113,28 @@ def test_keyless_command_line_cache(tmp_path, systems_path, mount_path):
     assert data["mirrors"]["buildcache"] == {"fetch": {"url": "/tmp/foo/user-environment"}}
 
 
+def test_readonly_buildcache(tmp_path, systems_path, mount_path):
+    """A buildcache in mirrors.yaml without a private_key is read-only (fetch-only)."""
+
+    mirrors = mirror.Mirrors(systems_path / "mirror-readonly-cache", mount_path)
+
+    # the cache exists (so it is fetched from), but has no signing key ...
+    assert mirrors.build_cache_mirror == "buildcache"
+    assert mirrors.buildcache["private_key"] is None
+
+    # ... so it is never pushed to
+    assert mirrors.push_to_build_cache is None
+
+    files = mirrors.config_files(tmp_path)
+
+    # no private key is written
+    assert tmp_path / "key_store" / "buildcache.priv.gpg" not in files
+
+    # the mirror is emitted with a fetch url but no push url
+    data = yaml.safe_load(files[tmp_path / "mirrors.yaml"])
+    assert data["mirrors"]["buildcache"] == {"fetch": {"url": "https://mirror.spack.io"}}
+
+
 def test_config_files(tmp_path, systems_path, mount_path):
     """Check that config_files presents the complete set of mirror config artifacts."""