Skip to content

Commit 225761d

Browse files
mwestphallmwestphall
authored
Merge pull request #147 from matyasselmeci/pr/SOFTWARE-5597.authfile-update-better-path
Rewrite authfile-update in Python; add better paths for .local files (SOFTWARE-5597)
2 parents cc76174 + 6442f30 commit 225761d

1 file changed

Lines changed: 241 additions & 138 deletions

File tree

‎src/authfile-update‎

Lines changed: 241 additions & 138 deletions
Original file line numberDiff line numberDiff line change
@@ -1,153 +1,256 @@
1-
#!/bin/bash
2-
3-
fetch () {
4-
local src=$1 dst=$2
5-
local tmp=$2.tmp
6-
local ret
7-
8-
if [[ ! -d $(dirname "$dst") ]]; then
9-
echo >&2 "Destination directory does not exist"
10-
return 1
11-
fi
12-
rm -f "$tmp"
13-
wget "$src" -q --content-on-error -O "$tmp"; ret=$?
14-
if [[ $ret != 0 ]]; then
15-
echo >&2 "Error fetching $src"
16-
if [[ -s $tmp ]]; then
17-
echo >&2 "File contents:"
18-
cat >&2 "$tmp"
19-
fi
20-
return 1
21-
else
22-
mv -f "$tmp" "$dst"
23-
fi
24-
}
1+
#!/usr/bin/env python3
2+
# -*- coding: utf-8 -*-
3+
import os
4+
import shutil
5+
import socket
6+
import subprocess
7+
import sys
8+
from typing import Optional, Tuple
9+
import urllib
10+
import urllib.error
11+
import urllib.request
2512

26-
die_with_usage () {
27-
echo >&2 "Usage: $(basename "$0") --cache|--origin"
28-
echo >&2 " Or: $(basename "$0") stash-cache|stash-cache-auth|stash-origin|stash-origin-auth"
29-
echo >&2 "Environment variables used:"
30-
echo >&2 "CACHE_FQDN FQDN used for cache authfile query (default `hostname -f`)"
31-
echo >&2 "ORIGIN_FQDN FQDN used for origin authfile query (default `hostname -f`)"
32-
echo >&2 "TOPOLOGY Topology server to get the data from (default https://topology.opensciencegrid.org)"
33-
exit 2
34-
}
3513

36-
if [[ $# -ne 1 ]]; then
37-
die_with_usage
38-
fi
39-
40-
TOPOLOGY=${TOPOLOGY:-https://topology.opensciencegrid.org}
41-
CACHE_FQDN=${CACHE_FQDN:-$(hostname -f)}
42-
ORIGIN_FQDN=${ORIGIN_FQDN:-$(hostname -f)}
43-
44-
DESTDIR=${DESTDIR:-/run}
45-
46-
47-
prepend_local_additions () {
48-
local base="$1"
49-
local tmp="${base}.$$"
50-
local ret=0
51-
if [[ -s $base && -f ${base}.local ]]; then
52-
echo -e "\n# The following lines are from $(basename "${base}.local"):" > "$tmp" && \
53-
cat "${base}.local" >> "$tmp" && \
54-
echo -e "\n# The following lines are from OSG Topology:" >> "$tmp" && \
55-
cat "${base}" >> "$tmp" && \
56-
mv -f "$tmp" "$base"
57-
ret=$?
58-
fi
59-
rm -f "$tmp"
60-
return $ret
61-
}
14+
KNOWN_INSTANCES = [
15+
"stash-origin",
16+
"stash-origin-auth",
17+
"stash-cache",
18+
"stash-cache-auth",
19+
]
6220

6321

64-
append_local_additions () {
65-
local base="$1"
66-
if [[ -s $base && -f ${base}.local ]]; then
67-
echo -e "\n# The following lines are from $(basename "${base}.local"):" >> "$base"
68-
cat "${base}.local" >> "$base"
69-
fi
22+
# fmt: off
23+
ENDPOINTS = {
24+
"Authfile": {
25+
"stash-origin" : "/origin/Authfile-public?fqdn={fqdn}",
26+
"stash-origin-auth" : "/origin/Authfile?fqdn={fqdn}",
27+
"stash-cache" : "/cache/Authfile-public?fqdn={fqdn}",
28+
"stash-cache-auth" : "/cache/Authfile?fqdn={fqdn}",
29+
},
30+
"scitokens.conf": {
31+
"stash-origin" : None,
32+
"stash-origin-auth" : "/origin/scitokens.conf?fqdn={fqdn}",
33+
"stash-cache" : None,
34+
"stash-cache-auth" : "/cache/scitokens.conf?fqdn={fqdn}",
35+
},
36+
"grid-mapfile": {
37+
"stash-origin" : None,
38+
"stash-origin-auth" : "/origin/grid-mapfile?fqdn={fqdn}",
39+
"stash-cache" : None,
40+
"stash-cache-auth" : "/cache/grid-mapfile?fqdn={fqdn}",
41+
},
7042
}
43+
# fmt: on
7144

7245

73-
fetch_cache_data () {
74-
mkdir -p "$DESTDIR/stash-cache"
75-
fetch "${TOPOLOGY}/cache/Authfile-public?fqdn=${CACHE_FQDN}" "$DESTDIR/stash-cache/Authfile" && \
76-
append_local_additions "$DESTDIR/stash-cache/Authfile"
77-
}
46+
CONFIG_FILES = list(ENDPOINTS.keys())
7847

7948

80-
fetch_cache_auth_data () {
81-
mkdir -p "$DESTDIR/stash-cache-auth"
82-
local ret=0
83-
fetch "${TOPOLOGY}/cache/Authfile?fqdn=${CACHE_FQDN}" "$DESTDIR/stash-cache-auth/Authfile" && \
84-
append_local_additions "$DESTDIR/stash-cache-auth/Authfile"
85-
ret=$(( $ret | $? ))
86-
fetch "${TOPOLOGY}/cache/scitokens.conf?fqdn=${CACHE_FQDN}" "$DESTDIR/stash-cache-auth/scitokens.conf" && \
87-
append_local_additions "$DESTDIR/stash-cache-auth/scitokens.conf"
88-
ret=$(( $ret | $? ))
89-
fetch "${TOPOLOGY}/cache/grid-mapfile?fqdn=${CACHE_FQDN}" "$DESTDIR/stash-cache-auth/grid-mapfile" && \
90-
prepend_local_additions "$DESTDIR/stash-cache-auth/grid-mapfile"
91-
ret=$(( $ret | $? ))
92-
return $ret
93-
}
49+
def complain(*values, **kwargs):
50+
# print to sys.stderr
51+
kwargs["file"] = sys.stderr
52+
return print(*values, **kwargs)
9453

9554

96-
fetch_origin_data () {
97-
mkdir -p "$DESTDIR/stash-origin"
98-
fetch "${TOPOLOGY}/origin/Authfile-public?fqdn=${ORIGIN_FQDN}" "$DESTDIR/stash-origin/Authfile" && \
99-
append_local_additions "$DESTDIR/stash-origin/Authfile"
100-
}
55+
def die_with_usage(prog):
56+
print(
57+
f"""
58+
Usage: {prog} <instance>
59+
or {prog} --cache
60+
or {prog} --origin
10161
62+
where <instance> is one of:
63+
stash-cache
64+
stash-cache-auth
65+
stash-origin
66+
stash-origin-auth
10267
103-
fetch_origin_auth_data () {
104-
mkdir -p "$DESTDIR/stash-origin-auth"
105-
local ret=0
106-
fetch "${TOPOLOGY}/origin/Authfile?fqdn=${ORIGIN_FQDN}" "$DESTDIR/stash-origin-auth/Authfile" && \
107-
append_local_additions "$DESTDIR/stash-origin-auth/Authfile"
108-
ret=$(( $ret | $? ))
109-
fetch "${TOPOLOGY}/origin/scitokens.conf?fqdn=${ORIGIN_FQDN}" "$DESTDIR/stash-origin-auth/scitokens.conf" && \
110-
append_local_additions "$DESTDIR/stash-origin-auth/scitokens.conf"
111-
ret=$(( $ret | $? ))
112-
fetch "${TOPOLOGY}/origin/grid-mapfile?fqdn=${ORIGIN_FQDN}" "$DESTDIR/stash-origin-auth/grid-mapfile" && \
113-
prepend_local_additions "$DESTDIR/stash-origin-auth/grid-mapfile"
114-
ret=$(( $ret | $? ))
115-
return $ret
116-
}
68+
--cache is equivalent to running it for stash-cache and stash-cache-auth
69+
--origin is equivalent to running it for stash-origin and stash-origin-auth
70+
71+
Environment variables used:
72+
CACHE_FQDN FQDN used for cache authfile query (default: the full hostname)
73+
ORIGIN_FQDN FQDN used for origin authfile query (default: the full hostname)
74+
TOPOLOGY Topology server to get the data from (default: https://topology.opensciencegrid.org)
75+
DESTDIR The base directory to write results to (default: /run)
76+
""",
77+
file=sys.stderr,
78+
)
79+
sys.exit(2)
80+
81+
82+
class Download:
83+
def __init__(self, topology, destdir, instance, config_file, fqdn):
84+
self.topology = topology
85+
self.destdir = destdir
86+
self.instance = instance
87+
self.config_file = config_file
88+
self.fqdn = fqdn
89+
90+
self.full_destdir = f"{self.destdir}/{self.instance}"
91+
self.dest_file = f"{self.full_destdir}/{self.config_file}"
92+
self.local_files = [
93+
f"{self.destdir}/{self.instance}/{self.config_file}.local",
94+
f"/etc/xrootd/{self.instance}-{self.config_file}.local",
95+
]
96+
self.prepend_local = config_file == "grid-mapfile"
97+
# ^^ local additions to the grid-mapfile are prepended, not appended
98+
# to what's downloaded from topology
99+
100+
def fetch(self) -> Tuple[Optional[str], bool]:
101+
"""Download the data for this config file from Topology and return
102+
the content of the download (`text`) and a boolean indicating
103+
success/failure (based on HTTP return code) (`ok`).
104+
105+
Returns (None, True) if there is no endpoint for this config file e.g.
106+
scitokens.conf for an unauthenticated cache.
107+
108+
"""
109+
endpoint = ENDPOINTS[self.config_file][self.instance]
110+
if not endpoint:
111+
return None, True
112+
113+
url = self.topology + endpoint.format(fqdn=self.fqdn)
114+
try:
115+
response = urllib.request.urlopen(url)
116+
text = response.read()
117+
if text:
118+
ok = True
119+
else:
120+
ok = False
121+
except urllib.error.HTTPError as err:
122+
# An HTTP error might indicate an error with the Topology registration
123+
# or the query; the contents are useful.
124+
text = err.read()
125+
ok = False
126+
if not isinstance(text, str):
127+
text = text.decode("utf-8", errors="replace")
128+
129+
return text, ok
130+
131+
def combine_with_local_files(self, text: str) -> str:
132+
"""Return the given text with the additions from the local files
133+
for this config file, if there are any. Missing files are silently
134+
skipped; other read errors are reported but are not failures.
135+
136+
"""
137+
new_text = ""
138+
for local_file in self.local_files:
139+
try:
140+
with open(local_file, "rt", encoding="utf-8", errors="replace") as fh:
141+
new_text += (
142+
f"## The following lines are from {local_file}:\n"
143+
+ fh.read().rstrip("\n")
144+
+ "\n\n"
145+
)
146+
except FileNotFoundError:
147+
pass
148+
except OSError as err:
149+
complain(f"Couldn't read local file {local_file}: {err} (continuing)")
150+
151+
if new_text:
152+
if self.prepend_local:
153+
new_text += "## The following lines are from OSG Topology:\n" + text
154+
else:
155+
new_text = text + "\n\n" + new_text
156+
return new_text.rstrip("\n") + "\n" # have exactly one final newline
157+
else:
158+
return text
159+
160+
def report_download_error(self, text):
161+
"""Print errors downloading the config file for the instance."""
162+
complain(f"Error fetching {self.config_file} for {self.instance}")
163+
if not text:
164+
complain("No data received")
165+
return
166+
complain("Response follows:")
167+
complain(text)
168+
169+
def write_dest_file(self, text):
170+
"""Writes the destination file atomically."""
171+
with open(self.dest_file + ".new", "wt", encoding="utf-8") as new_fh:
172+
new_fh.write(text)
173+
shutil.move(self.dest_file + ".new", self.dest_file)
174+
lines = text.count("\n")
175+
print(f"{lines} lines written successfully to {self.dest_file}.")
176+
177+
178+
def handle_instance(instance, topology, destdir):
179+
if "cache" in instance:
180+
fqdn = os.environ.get("CACHE_FQDN", socket.getfqdn())
181+
elif "origin" in instance:
182+
fqdn = os.environ.get("ORIGIN_FQDN", socket.getfqdn())
183+
else:
184+
assert False, f"bad instance {instance} should have been caught"
185+
186+
ret = 0
187+
188+
for config_file in CONFIG_FILES:
189+
dl = Download(
190+
topology=topology,
191+
destdir=destdir,
192+
instance=instance,
193+
config_file=config_file,
194+
fqdn=fqdn,
195+
)
196+
197+
if not os.path.isdir(dl.full_destdir):
198+
complain(f"Destination directory {dl.full_destdir} doesn't exist")
199+
return 1 # none of the other downloads will work either
200+
201+
text, ok = dl.fetch()
202+
203+
if not ok:
204+
# some failure happened; inform user of the error but then continue with
205+
# the next file
206+
ret = 1
207+
dl.report_download_error(text)
208+
continue
209+
210+
if not text:
211+
# we didn't download test but that may be ok for this instance
212+
continue
213+
214+
# download is successful; now combine the file with any local files
215+
text = dl.combine_with_local_files(text)
216+
217+
try:
218+
dl.write_dest_file(text)
219+
except OSError as err:
220+
complain(f"Couldn't write {dl.dest_file}: {err}")
221+
ret = 1
222+
continue
223+
224+
return ret
225+
226+
227+
def main(argv=None):
228+
if argv is None:
229+
argv = sys.argv
230+
231+
topology = os.environ.get("TOPOLOGY", "https://topology.opensciencegrid.org")
232+
destdir = os.environ.get("DESTDIR", "/run")
233+
ret = 0
234+
235+
if len(argv) != 2:
236+
die_with_usage(argv[0])
237+
238+
if argv[1] == "--cache":
239+
instances = ["stash-cache", "stash-cache-auth"]
240+
elif argv[1] == "--origin":
241+
instances = ["stash-origin", "stash-origin-auth"]
242+
else:
243+
if argv[1] not in KNOWN_INSTANCES:
244+
complain(f"Unknown instance {argv[1]}")
245+
die_with_usage(argv[0])
246+
else:
247+
instances = [argv[1]]
248+
249+
for instance in instances:
250+
ret |= handle_instance(instance, topology=topology, destdir=destdir)
251+
252+
return ret
117253

118254

119-
case $1 in
120-
--cache)
121-
ret=0
122-
fetch_cache_data
123-
ret=$(( $ret | $? ))
124-
fetch_cache_auth_data
125-
ret=$(( $ret | $? ))
126-
;;
127-
--origin)
128-
ret=0
129-
fetch_origin_data
130-
ret=$(( $ret | $? ))
131-
fetch_origin_auth_data
132-
ret=$(( $ret | $? ))
133-
;;
134-
stash-cache)
135-
fetch_cache_data
136-
ret=$?
137-
;;
138-
stash-cache-auth)
139-
fetch_cache_auth_data
140-
ret=$?
141-
;;
142-
stash-origin)
143-
fetch_origin_data
144-
ret=$?
145-
;;
146-
stash-origin-auth)
147-
fetch_origin_auth_data
148-
ret=$?
149-
;;
150-
*)
151-
die_with_usage
152-
esac
153-
exit $ret
255+
if __name__ == "__main__":
256+
sys.exit(main())

0 commit comments

Comments
 (0)