Skip to content

Commit 3c67694

Browse files
teoincontattoteoincontatto
committed
chore: script to create an offilne extensions repository
1 parent 83508cb commit 3c67694

3 files changed

Lines changed: 168 additions & 71 deletions

File tree

‎stackgres-k8s/install/helm/Dockerfile.offline-extensions‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
FROM quay.io/ongres/kubectl AS offline-extensions
2+
COPY stackgres-operator/extensions-cache-conciliator.sh /extensions-cache-conciliator.sh
3+
ENV OFFLINE=true
4+
USER root
5+
RUN mkdir -p /opt/app-root/src
6+
RUN chmod a+xw /opt/app-root/src
7+
USER stackgres
8+
ARG EXTENSIONS_CACHE_PRELOADED_EXTENSIONS
9+
ARG EXTENSIONS_CACHE_LOG_LEVEL=INFO
10+
RUN sh /extensions-cache-conciliator.sh run offline offline
11+
12+
FROM registry.access.redhat.com/ubi8/nginx-120
13+
COPY --from=offline-extensions /home/stackgres /opt/app-root/src
14+
USER root
15+
RUN rm /etc/nginx/nginx.conf
16+
RUN cat <<'EOF' > /etc/nginx/nginx.conf
17+
daemon off;
18+
pid /var/run/nginx.pid;
19+
error_log stderr info;
20+
events{
21+
}
22+
http {
23+
include /etc/nginx/mime.types;
24+
default_type application/octet-stream;
25+
26+
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
27+
'$status $body_bytes_sent "$http_referer" '
28+
'"$http_user_agent" "$http_x_forwarded_for"';
29+
30+
access_log /dev/stdout main;
31+
32+
sendfile on;
33+
34+
keepalive_timeout 65;
35+
36+
include /etc/nginx/conf.d/*.conf;
37+
}
38+
EOF
39+
RUN cat <<'EOF' > /etc/nginx/conf.d/stackgres-operator-extensions-cache.conf
40+
server {
41+
server_name localhost;
42+
43+
listen 8080;
44+
root /opt/app-root/src;
45+
46+
location / {
47+
try_files /$host$uri =404;
48+
}
49+
}
50+
EOF
51+
USER default
52+
EXPOSE 8080/tcp
53+
CMD [ "nginx" ]

‎stackgres-k8s/install/helm/build-offline-extensions.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/sh
2+
3+
cd "$(dirname "$0")"
4+
5+
if [ "x$1" = x ]
6+
then
7+
>&2 echo "You must specify one or more expressions to preload extensions (like 'x86_64/linux/timescaledb-1\.7\.4-pg12')"
8+
exit 1
9+
fi
10+
11+
if [ -f "$1" ]
12+
then
13+
EXTENSIONS_CACHE_PRELOADED_EXTENSIONS="$(printf true | jq --slurpfile arg "$1" -c '$arg')"
14+
else
15+
EXTENSIONS_CACHE_PRELOADED_EXTENSIONS="$(printf "$*" | tr ' ' '\n' | jq -R . | jq -c -s)"
16+
fi
17+
18+
docker build \
19+
--build-arg EXTENSIONS_CACHE_PRELOADED_EXTENSIONS="$EXTENSIONS_CACHE_PRELOADED_EXTENSIONS" \
20+
$(echo "$-" | grep -q x && printf %s '--build-arg EXTENSIONS_CACHE_LOG_LEVEL=TRACE' || true) \
21+
-t "${IMAGE:-stackgres-offline-extensions}" \
22+
-f Dockerfile.offline-extensions .

‎stackgres-k8s/install/helm/stackgres-operator/extensions-cache-conciliator.sh‎

Lines changed: 93 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ set -e
77

88
{ [ "$EXTENSIONS_CACHE_LOG_LEVEL" != DEBUG ] && [ "$EXTENSIONS_CACHE_LOG_LEVEL" != TRACE ]; } || set -x
99

10+
EXTENSIONS_REPOSITORY_URLS="${EXTENSIONS_REPOSITORY_URLS:-https://extensions.stackgres.io/postgres/repository}"
11+
1012
run () {
1113
# shellcheck disable=SC2153
1214
[ -n "$EXTENSIONS_REPOSITORY_URLS" ]
@@ -57,21 +59,27 @@ run () {
5759

5860
echo "Updating extensions..."
5961
(
60-
[ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
61-
STATEFULSET="$(kubectl get statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" -o json)"
62-
printf '%s' "$STATEFULSET" > "$STATEFULSET_JSON_FILE"
62+
if [ "$OFFLINE" != true ]
63+
then
64+
[ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
65+
STATEFULSET="$(kubectl get statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" -o json)"
66+
printf '%s' "$STATEFULSET" > "$STATEFULSET_JSON_FILE"
67+
fi
6368
)
6469
(
6570
[ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
6671
get_to_install_extensions > full_to_install_extensions
6772
sort full_to_install_extensions \
6873
| uniq | { grep -v '^$' || true; } > to_install_extensions
6974
TO_INSTALL_EXTENSIONS_JSON_STRING="$(jq -sR . to_install_extensions)"
70-
PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING="$(
71-
jq '.metadata.annotations
72-
| if . != null then .pulled_extensions else "" end
73-
| if . != null then . else "" end' "$STATEFULSET_JSON_FILE")"
74-
printf '%s' "$PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" | jq -r . > already_pulled_to_install_extensions
75+
if [ "$OFFLINE" != true ]
76+
then
77+
PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING="$(
78+
jq '.metadata.annotations
79+
| if . != null then .pulled_extensions else "" end
80+
| if . != null then . else "" end' "$STATEFULSET_JSON_FILE")"
81+
printf '%s' "$PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" | jq -r . > already_pulled_to_install_extensions
82+
fi
7583
if [ "$TO_INSTALL_EXTENSIONS_JSON_STRING" != "$PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" ]
7684
then
7785
while read -r LINE
@@ -101,47 +109,54 @@ run () {
101109
fi
102110
done
103111
(
104-
[ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
105-
ALREADY_PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING="$(
106-
jq '.metadata.annotations.pulled_extensions' "$STATEFULSET_JSON_FILE")"
107-
PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING="$(
108-
cat pulled_to_install_extensions already_pulled_to_install_extensions \
109-
| sort | uniq | jq -sR .)"
110-
if [ "$ALREADY_PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" != "$PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" ]
112+
if [ "$OFFLINE" != true ]
111113
then
112-
RUNNING_IMAGES="$(kubectl get statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" \
113-
--template '{{ range .spec.template.spec.containers }}{{ printf "%s\n" .image }}{{ end }}')"
114-
RUNNING_IMAGES="$(printf '%s' "$RUNNING_IMAGES" | sort)"
115-
REQUIRED_IMAGES="$(jq '.spec.template.spec.containers[].image' "$STATEFULSET_JSON_FILE")"
116-
REQUIRED_IMAGES="$(printf '%s' "$REQUIRED_IMAGES" | sort)"
117-
if [ "$RUNNING_IMAGES" != "$REQUIRED_IMAGES" ]
114+
[ "$EXTENSIONS_CACHE_LOG_LEVEL" = TRACE ] || set +x
115+
ALREADY_PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING="$(
116+
jq '.metadata.annotations.pulled_extensions' "$STATEFULSET_JSON_FILE")"
117+
PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING="$(
118+
cat pulled_to_install_extensions already_pulled_to_install_extensions \
119+
| sort | uniq | jq -sR .)"
120+
if [ "$ALREADY_PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" != "$PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" ]
121+
then
122+
RUNNING_IMAGES="$(kubectl get statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" \
123+
--template '{{ range .spec.template.spec.containers }}{{ printf "%s\n" .image }}{{ end }}')"
124+
RUNNING_IMAGES="$(printf '%s' "$RUNNING_IMAGES" | sort)"
125+
REQUIRED_IMAGES="$(jq '.spec.template.spec.containers[].image' "$STATEFULSET_JSON_FILE")"
126+
REQUIRED_IMAGES="$(printf '%s' "$REQUIRED_IMAGES" | sort)"
127+
if [ "$RUNNING_IMAGES" != "$REQUIRED_IMAGES" ]
128+
then
129+
touch /tmp/need-restart
130+
fi
131+
STATEFULSET="$(cat "$STATEFULSET_JSON_FILE")"
132+
printf '%s' "$STATEFULSET" \
133+
| jq ".metadata.annotations.pulled_extensions = $PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" \
134+
> "$STATEFULSET_JSON_FILE"
135+
cp "$STATEFULSET_JSON_FILE" "$STATEFULSET_JSON_FILE.new"
136+
until kubectl replace statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" \
137+
--cascade orphan -f "$STATEFULSET_JSON_FILE.new"
138+
do
139+
sleep 5
140+
STATEFULSET="$(kubectl get statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" -o json)"
141+
printf '%s' "$STATEFULSET" > "$STATEFULSET_JSON_FILE.lastest"
142+
jq -s '.[0].spec.template = .[1].spec.template | .[0]' \
143+
"$STATEFULSET_JSON_FILE.lastest" "$STATEFULSET_JSON_FILE" > "$STATEFULSET_JSON_FILE.new"
144+
done
145+
fi
146+
if ! any_image_repository_url && test -f /tmp/need-restart
118147
then
119-
touch /tmp/need-restart
148+
kubectl delete pod -n "$NAMESPACE" "$STATEFULSET_NAME-0"
120149
fi
121-
STATEFULSET="$(cat "$STATEFULSET_JSON_FILE")"
122-
printf '%s' "$STATEFULSET" \
123-
| jq ".metadata.annotations.pulled_extensions = $PULLED_TO_INSTALL_EXTENSIONS_JSON_STRING" \
124-
> "$STATEFULSET_JSON_FILE"
125-
cp "$STATEFULSET_JSON_FILE" "$STATEFULSET_JSON_FILE.new"
126-
until kubectl replace statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" \
127-
--cascade orphan -f "$STATEFULSET_JSON_FILE.new"
128-
do
129-
sleep 5
130-
STATEFULSET="$(kubectl get statefulset -n "$NAMESPACE" "$STATEFULSET_NAME" -o json)"
131-
printf '%s' "$STATEFULSET" > "$STATEFULSET_JSON_FILE.lastest"
132-
jq -s '.[0].spec.template = .[1].spec.template | .[0]' \
133-
"$STATEFULSET_JSON_FILE.lastest" "$STATEFULSET_JSON_FILE" > "$STATEFULSET_JSON_FILE.new"
134-
done
135-
fi
136-
if ! any_image_repository_url && test -f /tmp/need-restart
137-
then
138-
kubectl delete pod -n "$NAMESPACE" "$STATEFULSET_NAME-0"
139150
fi
140151
)
141152
echo "done"
142153
echo
143154
)
144155
EXIT_CODE="$?"
156+
if [ "$OFFLINE" = true ]
157+
then
158+
exit "$EXIT_CODE"
159+
fi
145160
if [ "$EXIT_CODE" = 0 ]
146161
then
147162
if ! test -f /tmp/need-restart
@@ -162,6 +177,10 @@ run () {
162177
}
163178

164179
any_image_repository_url() {
180+
if [ "$OFFLINE" = true ]
181+
then
182+
return 1
183+
fi
165184
local EXTENSIONS_REPOSITORY_URL
166185
for EXTENSIONS_REPOSITORY_URL in $(echo "$EXTENSIONS_REPOSITORY_URLS" | tr ',' '\n')
167186
do
@@ -377,37 +396,40 @@ update_repositories_credentials() {
377396
}
378397

379398
get_to_install_extensions() {
380-
(
381-
CLUSTER_EXTENSIONS="$([ -n "$ALLOWED_NAMESPACES" ] \
382-
&& printf %s "$ALLOWED_NAMESPACES" \
383-
| tr ' ' '\n' \
384-
| xargs -I @NAMESPACE sh $([ "$EXTENSIONS_CACHE_LOG_LEVEL" != TRACE ] || printf %s -x) -c \
385-
"kubectl get sgcluster -n @NAMESPACE -o json | jq '.items[]'" \
386-
|| kubectl get sgcluster -A -o json | jq '.items[]')"
387-
DISTRIBUTEDLOGS_EXTENSIONS="$([ -n "$ALLOWED_NAMESPACES" ] \
388-
&& printf %s "$ALLOWED_NAMESPACES" \
389-
| tr ' ' '\n' \
390-
| xargs -I @NAMESPACE sh $([ "$EXTENSIONS_CACHE_LOG_LEVEL" != TRACE ] || printf %s -x) -c \
391-
"kubectl get sgdistributedlogs -n @NAMESPACE -o json | jq '.items[]'" \
392-
|| kubectl get sgdistributedlogs -A -o json | jq '.items[]')"
393-
printf '%s' "$CLUSTER_EXTENSIONS"
394-
printf '%s' "$DISTRIBUTEDLOGS_EXTENSIONS"
395-
) | jq -r -s '.[]
396-
| select(has("status") and (.status | has("arch") and has("os")))
397-
| .status as $status
398-
| .spec.toInstallPostgresExtensions
399-
| if . != null then . else [] end
400-
| .[]
401-
| .repository + " "
402-
+ .publisher + " "
403-
+ .name + " "
404-
+ .version + " "
405-
+ .postgresVersion
406-
+ " " + (.flavor | if . != null then . else "'"$DEFAULT_FLAVOR"'" end)
407-
+ " " + $status.arch
408-
+ " " + $status.os
409-
+ " " + (.build | if . != null then . else "" end)
410-
'
399+
if [ "$OFFLINE" != true ]
400+
then
401+
(
402+
CLUSTER_EXTENSIONS="$([ -n "$ALLOWED_NAMESPACES" ] \
403+
&& printf %s "$ALLOWED_NAMESPACES" \
404+
| tr ' ' '\n' \
405+
| xargs -I @NAMESPACE sh $([ "$EXTENSIONS_CACHE_LOG_LEVEL" != TRACE ] || printf %s -x) -c \
406+
"kubectl get sgcluster -n @NAMESPACE -o json | jq '.items[]'" \
407+
|| kubectl get sgcluster -A -o json | jq '.items[]')"
408+
DISTRIBUTEDLOGS_EXTENSIONS="$([ -n "$ALLOWED_NAMESPACES" ] \
409+
&& printf %s "$ALLOWED_NAMESPACES" \
410+
| tr ' ' '\n' \
411+
| xargs -I @NAMESPACE sh $([ "$EXTENSIONS_CACHE_LOG_LEVEL" != TRACE ] || printf %s -x) -c \
412+
"kubectl get sgdistributedlogs -n @NAMESPACE -o json | jq '.items[]'" \
413+
|| kubectl get sgdistributedlogs -A -o json | jq '.items[]')"
414+
printf '%s' "$CLUSTER_EXTENSIONS"
415+
printf '%s' "$DISTRIBUTEDLOGS_EXTENSIONS"
416+
) | jq -r -s '.[]
417+
| select(has("status") and (.status | has("arch") and has("os")))
418+
| .status as $status
419+
| .spec.toInstallPostgresExtensions
420+
| if . != null then . else [] end
421+
| .[]
422+
| .repository + " "
423+
+ .publisher + " "
424+
+ .name + " "
425+
+ .version + " "
426+
+ .postgresVersion
427+
+ " " + (.flavor | if . != null then . else "'"$DEFAULT_FLAVOR"'" end)
428+
+ " " + $status.arch
429+
+ " " + $status.os
430+
+ " " + (.build | if . != null then . else "" end)
431+
'
432+
fi
411433
jq -r '
412434
.extensions[] | . as $extension
413435
| .versions[] | . as $version

0 commit comments

Comments
 (0)