Skip to content

Commit 71e4812

Browse files
bcummingbcumming
committed
use spack schema for mirrors
1 parent e1ed765 commit 71e4812

6 files changed

Lines changed: 240 additions & 94 deletions

File tree

‎stackinator/mirror.py‎

Lines changed: 64 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -168,10 +168,11 @@ def __init__(
168168
"name": "buildcache",
169169
"url": raw_cache["root"],
170170
"description": "Buildcache dest loaded from legacy cache.yaml",
171+
"source": False,
172+
"binary": True,
171173
"public_key": None,
172174
"private_key": raw_cache.get("key"),
173175
"mount_specific": True,
174-
"cmdline": True,
175176
}
176177
self._logger.warning(
177178
"Configuring the buildcache from the system cache.yaml file.\n"
@@ -183,14 +184,10 @@ def __init__(
183184
# The bootstrap mirror, the read-only source mirrors, the writable source
184185
# cache, and the writable concretizer cache, if any are defined.
185186
self.bootstrap = raw_mirrors.get("bootstrap")
186-
self.source_mirrors = dict(raw_mirrors["sourcemirror"])
187+
self.source_mirrors = raw_mirrors.get("sourcemirror") or {}
187188
self.source_cache = raw_mirrors.get("sourcecache")
188189
self.concretizer_cache = raw_mirrors.get("concretizer")
189190

190-
# Validate that every mirror url is well-formed (see _validate_url).
191-
for name, mirror in self._iter_mirrors():
192-
self._validate_url(mirror["url"], name)
193-
194191
# The source and concretizer caches are single writable local directories
195192
# (spack config:source_cache / concretizer:concretization_cache), not mirrors:
196193
# validate that each is an absolute path. Expand env vars now, because the
@@ -224,7 +221,6 @@ def __init__(
224221
self._bootstrap_metadata_dirs: List[str] = []
225222
if self.bootstrap is not None:
226223
url = self.bootstrap["url"]
227-
self._validate_url(url, "bootstrap")
228224
if self._is_remote_url(url):
229225
self._bootstrap_remote = True
230226
else:
@@ -284,19 +280,6 @@ def push_to_build_cache(self) -> Optional[str]:
284280
return self.buildcache["name"]
285281
return None
286282

287-
def _iter_mirrors(self):
288-
"""Yield (spack mirror name, config dict) for every real spack mirror.
289-
290-
These are the entries written to the spack mirrors.yaml: the build cache
291-
(whose name is the configurable 'name' field) and the source mirrors (named
292-
by their key in mirrors.yaml). The bootstrap mirror is not a mirrors.yaml
293-
entry, so it is handled separately.
294-
"""
295-
296-
if self.buildcache is not None:
297-
yield self.buildcache["name"], self.buildcache
298-
yield from self.source_mirrors.items()
299-
300283
@staticmethod
301284
def _is_remote_url(url: str) -> bool:
302285
"""True if url is a remote url (has a non-file scheme and a host)."""
@@ -312,31 +295,50 @@ def _local_path(url: str) -> pathlib.Path:
312295
url = url[len("file://") :]
313296
return pathlib.Path(os.path.expandvars(url))
314297

315-
def _validate_url(self, url: str, name: str):
316-
"""Validate that a mirror url is well-formed.
298+
# the spack mirror connection fields (everything that may appear inside a
299+
# fetch/push block, or at the top level of a mirror entry as a shorthand).
300+
_CONNECTION_KEYS = ("url", "view", "profile", "endpoint_url", "access_token_variable", "access_pair")
301+
302+
@classmethod
303+
def _connection(cls, entry: Dict, key: str, mount_path: Optional[pathlib.Path] = None) -> Optional[Dict]:
304+
"""Resolve a mirror entry's fetch/push connection to a spack connection dict.
317305
318-
Only the format of the url is checked: no attempt is made to connect to
319-
remote mirrors, because a valid-but-unreachable url would otherwise block
320-
until the network request times out.
306+
The connection comes from the entry's explicit `key` (either a url string or a
307+
connection object), falling back to the entry's top-level connection fields
308+
(url + auth) when no explicit fetch/push block is given - matching how spack
309+
applies a top-level url/auth to a mirror with no fetch/push. Returns None if
310+
no url can be resolved. When mount_path is set the url gains the mount point as
311+
a sub-directory (mount-specific build caches).
321312
"""
322313

323-
if url.startswith("file://"):
324-
# local mirror: verify that the root path is an existing directory
325-
path = pathlib.Path(os.path.expandvars(url[len("file://") :]))
326-
if not path.is_absolute():
327-
raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not absolute")
328-
if not path.is_dir():
329-
raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not a directory")
330-
return
314+
conn = entry.get(key)
315+
if conn is None:
316+
# shorthand: assemble the connection from the entry's top-level fields
317+
conn = {k: entry[k] for k in cls._CONNECTION_KEYS if entry.get(k) is not None}
318+
elif isinstance(conn, str):
319+
conn = {"url": conn}
320+
else:
321+
# copy so the relocation below never mutates the stored config
322+
conn = dict(conn)
331323

332-
parsed = urllib.parse.urlparse(url)
333-
if not parsed.scheme:
334-
# a bare path is accepted if absolute (e.g. the legacy command line cache)
335-
if not pathlib.Path(url).is_absolute():
336-
raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url or absolute path")
337-
elif not parsed.netloc:
338-
# a remote mirror requires a well-formed url with both a scheme and a host
339-
raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url")
324+
if "url" not in conn:
325+
return None
326+
327+
if mount_path is not None:
328+
conn["url"] = conn["url"].rstrip("/") + "/" + mount_path.as_posix().lstrip("/")
329+
return conn
330+
331+
@staticmethod
332+
def _add_optional_flags(entry: Dict, mirror: Dict):
333+
"""Pass the optional spack mirror flags (signed, autopush) through if set.
334+
335+
These have no default in the schema, so they are present only when the user set
336+
them; they are copied verbatim into the emitted spack mirror entry.
337+
"""
338+
339+
for flag in ("signed", "autopush"):
340+
if flag in mirror:
341+
entry[flag] = mirror[flag]
340342

341343
def _read_key(self, key: str, name: str) -> bytes:
342344
"""Resolve a key (a file path or base64 blob) to validated gpg key bytes.
@@ -397,25 +399,34 @@ def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]:
397399
spack_mirrors: Dict[str, Dict] = {"mirrors": {}}
398400

399401
if self.buildcache is not None:
400-
url = self.buildcache["url"]
401402
# a mount-specific build cache lives in a sub-directory named after the
402403
# mount point: spack binaries embed the install prefix, so each mount
403404
# point needs its own cache to avoid relocation issues.
404-
if self.buildcache["mount_specific"]:
405-
url = url.rstrip("/") + "/" + self._mount_path.as_posix().lstrip("/")
406-
entry = {"fetch": {"url": url}}
407-
# only a build cache with a signing key is pushed to
408-
if self.buildcache["private_key"] is not None:
409-
entry["push"] = {"url": url}
405+
mount = self._mount_path if self.buildcache["mount_specific"] else None
406+
entry = {
407+
"source": self.buildcache["source"],
408+
"binary": self.buildcache["binary"],
409+
"fetch": self._connection(self.buildcache, "fetch", mount),
410+
}
411+
# a build cache is only pushed to when it has a signing key; a keyless
412+
# build cache is read-only (fetched from, never pushed to).
413+
if self.push_to_build_cache is not None:
414+
entry["push"] = self._connection(self.buildcache, "push", mount)
415+
self._add_optional_flags(entry, self.buildcache)
410416
spack_mirrors["mirrors"][self.buildcache["name"]] = entry
411417

412-
# source mirrors are read-only and provide sources only: fetch url, no push.
418+
# source mirrors are read-only by default (fetch only); a push connection is
419+
# emitted only when one was explicitly configured.
413420
for name, mirror in self.source_mirrors.items():
414-
spack_mirrors["mirrors"][name] = {
415-
"source": True,
416-
"binary": False,
417-
"fetch": {"url": mirror["url"]},
421+
entry = {
422+
"source": mirror["source"],
423+
"binary": mirror["binary"],
424+
"fetch": self._connection(mirror, "fetch"),
418425
}
426+
if mirror.get("push") is not None:
427+
entry["push"] = self._connection(mirror, "push")
428+
self._add_optional_flags(entry, mirror)
429+
spack_mirrors["mirrors"][name] = entry
419430

420431
files[config_root / self.MIRRORS_YAML] = yaml.dump(
421432
spack_mirrors, default_flow_style=False, sort_keys=False

‎stackinator/schema/mirror.json‎

Lines changed: 95 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,50 @@
11
{
2+
"$defs": {
3+
"connection": {
4+
"type": "object",
5+
"additionalProperties": false,
6+
"properties": {
7+
"url": {
8+
"type": "string",
9+
"description": "URL pointing to the mirror directory: local filesystem (file://) or remote (http://, https://, s3://, oci://)"
10+
},
11+
"view": { "type": "string" },
12+
"profile": {
13+
"type": ["string", "null"],
14+
"description": "AWS profile name to use for S3 mirror authentication"
15+
},
16+
"endpoint_url": {
17+
"type": ["string", "null"],
18+
"description": "Custom endpoint URL for S3-compatible storage services"
19+
},
20+
"access_token_variable": {
21+
"type": ["string", "null"],
22+
"description": "Environment variable containing an access token for OCI registry authentication"
23+
},
24+
"access_pair": {
25+
"type": "object",
26+
"description": "Authentication credentials for accessing private mirrors with ID and secret pairs",
27+
"required": ["secret_variable"],
28+
"oneOf": [
29+
{"required": ["id"]},
30+
{"required": ["id_variable"]}
31+
],
32+
"additionalProperties": false,
33+
"properties": {
34+
"id": { "type": "string" },
35+
"id_variable": { "type": "string" },
36+
"secret_variable": { "type": "string" }
37+
}
38+
}
39+
}
40+
},
41+
"fetch_and_push": {
42+
"anyOf": [
43+
{ "type": "string" },
44+
{ "$ref": "#/$defs/connection" }
45+
]
46+
}
47+
},
248
"type" : "object",
349
"additionalProperties": false,
450
"properties": {
@@ -13,38 +59,64 @@
1359
},
1460
"buildcache": {
1561
"type": "object",
62+
"additionalProperties": false,
63+
"anyOf": [
64+
{"required": ["url"]},
65+
{"required": ["fetch"]},
66+
{"required": ["push"]}
67+
],
1668
"properties": {
17-
"name": {
18-
"type": "string",
19-
"default": "buildcache"
69+
"name": { "type": "string", "default": "buildcache" },
70+
"description": { "type": "string", "default": "" },
71+
"mount_specific": { "type": "boolean", "default": false },
72+
"source": { "type": "boolean", "default": false },
73+
"binary": { "type": "boolean", "default": true },
74+
"signed": { "type": "boolean" },
75+
"autopush": { "type": "boolean" },
76+
"private_key": {
77+
"oneOf": [ {"type" : "string"}, {"type" : "null"} ],
78+
"default": null
2079
},
21-
"description": {"type": "string", "default": ""},
22-
"url": {"type": "string"},
23-
"public_key": {"type": ["string", "null"], "default": null},
24-
"private_key": {"type": ["string", "null"], "default": null},
25-
"mount_specific": {
26-
"type": "boolean",
27-
"default": false
80+
"public_key": {
81+
"oneOf": [ {"type" : "string"}, {"type" : "null"} ],
82+
"default": null
2883
},
29-
"cmdline": {
30-
"type": "boolean",
31-
"default": false
32-
}
33-
},
34-
"additionalProperties": false,
35-
"required": ["url"]
84+
"url": { "type": "string" },
85+
"view": { "type": "string" },
86+
"profile": { "type": ["string", "null"] },
87+
"endpoint_url": { "type": ["string", "null"] },
88+
"access_token_variable": { "type": ["string", "null"] },
89+
"access_pair": { "$ref": "#/$defs/connection/properties/access_pair" },
90+
"fetch": { "$ref": "#/$defs/fetch_and_push" },
91+
"push": { "$ref": "#/$defs/fetch_and_push" }
92+
}
3693
},
3794
"sourcemirror": {
3895
"type": "object",
3996
"default": {},
4097
"additionalProperties": {
4198
"type": "object",
42-
"properties": {
43-
"description": {"type": "string", "default": ""},
44-
"url": {"type": "string"}
45-
},
4699
"additionalProperties": false,
47-
"required": ["url"]
100+
"anyOf": [
101+
{"required": ["url"]},
102+
{"required": ["fetch"]},
103+
{"required": ["push"]}
104+
],
105+
"properties": {
106+
"description": { "type": "string", "default": "" },
107+
"source": { "type": "boolean", "default": true },
108+
"binary": { "type": "boolean", "default": false },
109+
"signed": { "type": "boolean" },
110+
"autopush": { "type": "boolean" },
111+
"url": { "type": "string" },
112+
"view": { "type": "string" },
113+
"profile": { "type": ["string", "null"] },
114+
"endpoint_url": { "type": ["string", "null"] },
115+
"access_token_variable": { "type": ["string", "null"] },
116+
"access_pair": { "$ref": "#/$defs/connection/properties/access_pair" },
117+
"fetch": { "$ref": "#/$defs/fetch_and_push" },
118+
"push": { "$ref": "#/$defs/fetch_and_push" }
119+
}
48120
}
49121
},
50122
"sourcecache": {
@@ -66,4 +138,4 @@
66138
"required": ["path"]
67139
}
68140
}
69-
}
141+
}

‎unittests/data/systems/mirror-bad-url/mirrors.yaml‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

‎unittests/data/systems/mirror-ok/mirrors.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ buildcache:
4545
gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3"
4646
private_key: ../../test-gpg-priv.asc
4747
mount_specific: false
48-
cmdline: false
4948
sourcemirror:
5049
mirror1:
5150
url: https://github.com

‎unittests/data/systems/mirror-s3/mirrors.yaml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# a buildcache and a source mirror that use the spack-style fetch/push
2+
# connection objects, with distinct read/write endpoints and S3/OCI auth.
3+
buildcache:
4+
private_key: ../../test-gpg-priv.asc
5+
fetch:
6+
url: s3://my-bucket/buildcache
7+
endpoint_url: https://s3.example.com
8+
access_pair:
9+
id_variable: AWS_ACCESS_KEY_ID
10+
secret_variable: AWS_SECRET_ACCESS_KEY
11+
push:
12+
url: s3://my-bucket/buildcache
13+
endpoint_url: https://s3.example.com
14+
access_pair:
15+
id_variable: AWS_ACCESS_KEY_ID
16+
secret_variable: AWS_SECRET_ACCESS_KEY
17+
sourcemirror:
18+
s3-sources:
19+
fetch:
20+
url: s3://my-bucket/sources
21+
profile: my-profile
22+
endpoint_url: https://s3.example.com
23+
push:
24+
url: s3://my-bucket/sources
25+
profile: my-profile
26+
endpoint_url: https://s3.example.com

0 commit comments

Comments
 (0)