more s3 stuff

Author: grodzki-lanl

pyproject.toml

@@ -15,6 +15,7 @@ dependencies = [
   "Jinja2",
   "jsonschema",
   "PyYAML",
+  "boto3"
 ]
 classifiers = [
   "Development Status :: 5 - Production/Stable",

stackinator/mirror.py

@@ -276,11 +276,17 @@ def build_cache_mirror(self) -> Optional[str]:
     def push_to_build_cache(self) -> Optional[str]:
         """The build cache mirror name to push built packages to, or None.
 
-        Pushing requires a private signing key; a build cache configured without one
-        is read-only - fetched from but never pushed to.
+        Pushing requires a private signing key or an access pair for an S3 bucket; 
+        a build cache configured without one is read-only - fetched from but never pushed to.
         """
 
-        if self.buildcache is not None and self.buildcache["private_key"] is not None:
+        if (
+            self.buildcache is not None
+            and (
+                self.buildcache.get("private_key") is not None
+                or self.buildcache.get("access_pair") is not None
+            )
+        ):
             return self.buildcache["name"]
         return None
 
@@ -403,20 +409,51 @@ def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]:
             # point needs its own cache to avoid relocation issues.
             if self.buildcache["mount_specific"]:
                 url = url.rstrip("/") + "/" + self._mount_path.as_posix().lstrip("/")
-            entry = {"fetch": {"url": url}}
-            # only a build cache with a signing key is pushed to
-            if self.buildcache["private_key"] is not None:
-                entry["push"] = {"url": url}
-            spack_mirrors["mirrors"][self.buildcache["name"]] = entry
-
-        # source mirrors are read-only and provide sources only: fetch url, no push.
-        for name, mirror in self.source_mirrors.items():
-            spack_mirrors["mirrors"][name] = {
-                "source": True,
-                "binary": False,
+            if self.buildcache["access_pair"] and self.buildcache["endpoint_url"]:
+                spack_mirrors["mirrors"][self.buildcache["name"]] = {
+                    "fetch": {
+                        "url": mirror["url"],
+                        "access_pair": mirror["access_pair"],
+                        "endpoint_url": mirror["url"]
+                    },
+                    "push": {
+                        "url": mirror["url"],
+                        "access_pair": mirror["access_pair"],
+                        "endpoint_url": mirror["url"]
+                    },
+                }
+            else:
+                spack_mirrors["mirrors"][self.buildcache["name"]] = {
                 "fetch": {"url": mirror["url"]},
+                "push": {"url": mirror["url"]},
             }
 
+
+        # source mirrors are read-only and provide sources only. Push url is required but never used
+        for name, mirror in self.source_mirrors.items():
+            if mirror["access_pair"] and mirror["endpoint_url"]:
+                spack_mirrors["mirrors"][name] = {
+                    "source": True,
+                    "binary": False,
+                    "fetch": {
+                        "url": mirror["url"],
+                        "access_pair": mirror["access_pair"],
+                        "endpoint_url": mirror["url"]
+                    },
+                    "push": {
+                        "url": mirror["url"],
+                        "access_pair": mirror["access_pair"],
+                        "endpoint_url": mirror["url"]
+                    },
+                }
+            else:
+                spack_mirrors["mirrors"][name] = {
+                    "source": True,
+                    "binary": False,
+                    "fetch": {"url": mirror["url"]},
+                    "push": {"url": mirror["url"]},
+                }
+
         files[config_root / self.MIRRORS_YAML] = yaml.dump(
             spack_mirrors, default_flow_style=False, sort_keys=False
         ).encode()

stackinator/schema/mirror.json

@@ -6,14 +6,7 @@
             "type": "object",
             "properties": {
                 "description": {"type": "string", "default": ""},
-                "url": {"type": "string"},
-                "access_pair": {
-                    "type": "array",
-                    "items": { "type": "string" },
-                    "minItems": 2,
-                    "maxItems": 2
-                },
-                "endpoint_url": {"type": "string"}
+                "url": {"type": "string"}
             },
             "additionalProperties": false,
             "required": ["url"]