Skip to content

Commit 09cb5a8

Browse files
Rub21Rub21
committed
Merge branch 'staging' into web-upstream
2 parents 0886cd7 + d71c818 commit 09cb5a8

11 files changed

Lines changed: 569 additions & 20 deletions

‎.github/workflows/frontend-overpass.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ jobs:
4545
uses: actions/checkout@v6
4646
with:
4747
repository: OpenHistoricalMap/overpass-turbo
48-
ref: 527c3accf412f73766b70193f4577f2d2021d534
48+
ref: d10732ea33ef79b2f79a8132f73d0034272155f4
4949
path: overpass-turbo
5050

5151
- name: Enable Corepack

‎.github/workflows/preview-web-down.yaml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: Preview website down (k3s)
2+
# Remove a website preview: helm uninstall + delete its namespace.
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
ref:
7+
description: web-* branch whose preview to remove
8+
required: true
9+
delete:
10+
pull_request:
11+
types: [closed]
12+
13+
jobs:
14+
teardown:
15+
runs-on: ubuntu-22.04
16+
timeout-minutes: 20
17+
steps:
18+
- name: Resolve branch
19+
id: r
20+
run: |
21+
# branch name from whichever event fired
22+
case "${{ github.event_name }}" in
23+
workflow_dispatch) BRANCH="${{ github.event.inputs.ref }}" ;;
24+
delete) BRANCH="${{ github.event.ref }}" ;;
25+
pull_request) BRANCH="${{ github.event.pull_request.head.ref }}" ;;
26+
esac
27+
case "$BRANCH" in
28+
web-*) ;;
29+
*) echo "branch '$BRANCH' is not a web-* preview; nothing to do"; echo "skip=true" >> $GITHUB_OUTPUT; exit 0 ;;
30+
esac
31+
SLUG="$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' \
32+
| sed 's/[^a-z0-9]/-/g; s/-\+/-/g; s/^-//; s/-$//' | cut -c1-40 | sed 's/-$//')"
33+
echo "skip=false" >> $GITHUB_OUTPUT
34+
echo "branch=$BRANCH" >> $GITHUB_OUTPUT
35+
echo "release=$SLUG" >> $GITHUB_OUTPUT
36+
# shared namespace for all previews; only the release is per-branch
37+
echo "namespace=preview" >> $GITHUB_OUTPUT
38+
echo "host=$SLUG.ohmstaging.org" >> $GITHUB_OUTPUT
39+
40+
- uses: actions/checkout@v4
41+
if: steps.r.outputs.skip != 'true'
42+
43+
- name: Install cloudflared
44+
if: steps.r.outputs.skip != 'true'
45+
run: |
46+
sudo curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 \
47+
-o /usr/local/bin/cloudflared
48+
sudo chmod +x /usr/local/bin/cloudflared
49+
50+
- name: Install helm
51+
if: steps.r.outputs.skip != 'true'
52+
uses: azure/setup-helm@v4
53+
with:
54+
version: v3.15.1
55+
56+
- name: Setup kubeconfig
57+
if: steps.r.outputs.skip != 'true'
58+
run: |
59+
mkdir -p $HOME/.kube
60+
echo "${{ secrets.STAGING_K3S_KUBECONFIG }}" | base64 -d > $HOME/.kube/config
61+
chmod 600 $HOME/.kube/config
62+
63+
- name: Open Cloudflare Access tunnel to k3s API
64+
if: steps.r.outputs.skip != 'true'
65+
env:
66+
TUNNEL_SERVICE_TOKEN_ID: ${{ secrets.STAGING_CF_ACCESS_CLIENT_ID }}
67+
TUNNEL_SERVICE_TOKEN_SECRET: ${{ secrets.STAGING_CF_ACCESS_CLIENT_SECRET }}
68+
run: |
69+
cloudflared access tcp --hostname k3s.ohmstaging.org --url 127.0.0.1:16443 &
70+
for i in {1..30}; do
71+
curl -sk -o /dev/null --max-time 5 https://127.0.0.1:16443/livez && exit 0
72+
sleep 2
73+
done
74+
echo "Tunnel failed to reach k3s" >&2; exit 1
75+
76+
# Uninstall only this branch's release. The shared `preview` namespace and
77+
# its middleware stay (other previews live there).
78+
- name: Helm uninstall
79+
if: steps.r.outputs.skip != 'true'
80+
run: |
81+
helm -n ${{ steps.r.outputs.namespace }} uninstall ${{ steps.r.outputs.release }} || echo "release already gone"
82+
kubectl -n ${{ steps.r.outputs.namespace }} delete job ${{ steps.r.outputs.release }}-restore --ignore-not-found

‎.github/workflows/preview-web-up.yaml‎

Lines changed: 264 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,264 @@
1+
name: Preview website (k3s)
2+
3+
# Website preview for a web-* branch on k3s staging, reachable at
4+
# web-<branch>.ohmstaging.org.
5+
on:
6+
workflow_dispatch:
7+
inputs:
8+
ref:
9+
description: web-* branch to preview
10+
required: true
11+
backup_url:
12+
description: apidb backup URL (.sql or .sql.gz); blank = PREVIEW_BACKUP_URL secret
13+
required: false
14+
push:
15+
branches:
16+
- 'web-*'
17+
18+
concurrency:
19+
# one run per branch; a new push cancels the in-flight preview build
20+
group: preview-web-${{ github.event.inputs.ref || github.ref_name }}
21+
cancel-in-progress: true
22+
23+
jobs:
24+
preview:
25+
runs-on: ubuntu-22.04
26+
timeout-minutes: 60
27+
steps:
28+
- name: Resolve names
29+
id: n
30+
run: |
31+
BRANCH="${{ github.event.inputs.ref || github.ref_name }}"
32+
case "$BRANCH" in
33+
web-*) ;;
34+
*) echo "::error::branch '$BRANCH' must start with web-"; exit 1 ;;
35+
esac
36+
# DNS-safe slug: lowercase, non-alnum -> '-', trim, max 40 chars.
37+
SLUG="$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' \
38+
| sed 's/[^a-z0-9]/-/g; s/-\+/-/g; s/^-//; s/-$//' | cut -c1-40 | sed 's/-$//')"
39+
echo "branch=$BRANCH" >> $GITHUB_OUTPUT
40+
echo "slug=$SLUG" >> $GITHUB_OUTPUT
41+
echo "release=$SLUG" >> $GITHUB_OUTPUT
42+
# one shared namespace for all previews; each branch is its own release
43+
echo "namespace=preview" >> $GITHUB_OUTPUT
44+
echo "host=$SLUG.ohmstaging.org" >> $GITHUB_OUTPUT
45+
46+
- uses: actions/checkout@v4
47+
with:
48+
ref: ${{ steps.n.outputs.branch }}
49+
fetch-depth: 0
50+
51+
- name: Login to GitHub Container Registry
52+
uses: docker/login-action@v3
53+
with:
54+
registry: ghcr.io
55+
username: ${{ github.repository_owner }}
56+
password: ${{ secrets.GHCR_GITHUB_TOKEN }}
57+
58+
- name: Setup Python
59+
uses: actions/setup-python@v5
60+
with:
61+
python-version: '3.11'
62+
63+
- name: Setup git
64+
run: |
65+
git config --global user.email "noreply@developmentseed.org"
66+
git config --global user.name "Github Action"
67+
68+
- name: Install chartpress
69+
run: pip install chartpress==2.3.0 ruamel.yaml
70+
71+
- name: Run chartpress (build + push)
72+
env:
73+
GITHUB_TOKEN: ${{ secrets.GHCR_GITHUB_TOKEN }}
74+
run: chartpress --push
75+
76+
- name: Install cloudflared
77+
run: |
78+
sudo curl -fsSL https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64 \
79+
-o /usr/local/bin/cloudflared
80+
sudo chmod +x /usr/local/bin/cloudflared
81+
82+
- name: Install helm
83+
uses: azure/setup-helm@v4
84+
with:
85+
version: v3.15.1
86+
87+
- name: Setup kubeconfig
88+
run: |
89+
mkdir -p $HOME/.kube
90+
KCFG="${{ secrets.STAGING_K3S_KUBECONFIG }}"
91+
if [ -z "$KCFG" ]; then
92+
echo "ERROR: STAGING_K3S_KUBECONFIG is empty" >&2; exit 1
93+
fi
94+
echo "$KCFG" | base64 -d > $HOME/.kube/config
95+
chmod 600 $HOME/.kube/config
96+
97+
- name: Open Cloudflare Access tunnel to k3s API
98+
env:
99+
TUNNEL_SERVICE_TOKEN_ID: ${{ secrets.STAGING_CF_ACCESS_CLIENT_ID }}
100+
TUNNEL_SERVICE_TOKEN_SECRET: ${{ secrets.STAGING_CF_ACCESS_CLIENT_SECRET }}
101+
run: |
102+
cloudflared access tcp --hostname k3s.ohmstaging.org --url 127.0.0.1:16443 &
103+
CF_PID=$!
104+
for i in {1..30}; do
105+
if curl -sk -o /dev/null --max-time 5 https://127.0.0.1:16443/livez; then
106+
echo "tunnel up (k3s reachable)"; exit 0
107+
fi
108+
sleep 2
109+
done
110+
echo "Tunnel failed to reach k3s" >&2; kill $CF_PID 2>/dev/null || true; exit 1
111+
112+
- name: Verify access
113+
run: kubectl get nodes
114+
115+
- name: Substitute secrets into preview values
116+
uses: bluwy/substitute-string-action@v3
117+
with:
118+
_input-file: 'values.k3s.preview.template.yaml'
119+
_format-key: '{{key}}'
120+
_output-file: 'values.k3s.preview.yaml'
121+
PREVIEW_HOST: ${{ steps.n.outputs.host }}
122+
PREVIEW_NS: ${{ steps.n.outputs.namespace }}
123+
PREVIEW_DB_PASSWORD: ${{ secrets.PREVIEW_DB_PASSWORD }}
124+
MAILER_ADDRESS: ${{ secrets.MAILER_ADDRESS }}
125+
MAILER_USERNAME: ${{ secrets.MAILER_USERNAME }}
126+
MAILER_PASSWORD: ${{ secrets.MAILER_PASSWORD }}
127+
STAGING_OPENSTREETMAP_AUTH_ID: ${{ secrets.STAGING_OPENSTREETMAP_AUTH_ID }}
128+
STAGING_OPENSTREETMAP_AUTH_SECRET: ${{ secrets.STAGING_OPENSTREETMAP_AUTH_SECRET }}
129+
STAGING_WIKIPEDIA_AUTH_ID: ${{ secrets.STAGING_WIKIPEDIA_AUTH_ID }}
130+
STAGING_WIKIPEDIA_AUTH_SECRET: ${{ secrets.STAGING_WIKIPEDIA_AUTH_SECRET }}
131+
STAGING_RAILS_CREDENTIALS_YML_ENC: ${{ secrets.STAGING_RAILS_CREDENTIALS_YML_ENC }}
132+
STAGING_RAILS_MASTER_KEY: ${{ secrets.STAGING_RAILS_MASTER_KEY }}
133+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
134+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
135+
136+
- name: Helm dep up
137+
run: cd ohm && helm dep up
138+
139+
# Step 1: bring up db + memcached + cgimap, web OFF, so the restore can
140+
# load the apidb before web boots and runs migrations.
141+
- name: Deploy data tier (web off)
142+
run: |
143+
helm upgrade --install ${{ steps.n.outputs.release }} ./ohm \
144+
-n ${{ steps.n.outputs.namespace }} --create-namespace \
145+
-f ./ohm/values.yaml \
146+
-f ./values.k3s.preview.yaml \
147+
--set osm-seed.web.enabled=false \
148+
--wait --timeout=10m
149+
150+
- name: Restore apidb from backup
151+
env:
152+
NS: ${{ steps.n.outputs.namespace }}
153+
RELEASE: ${{ steps.n.outputs.release }}
154+
BACKUP_URL: ${{ github.event.inputs.backup_url || secrets.PREVIEW_BACKUP_URL }}
155+
run: |
156+
if [ -z "$BACKUP_URL" ]; then
157+
echo "::error::set the PREVIEW_BACKUP_URL secret (or pass backup_url on dispatch)"; exit 1
158+
fi
159+
echo "backup: $BACKUP_URL"
160+
WEB_IMAGE="$(helm -n "$NS" get values "$RELEASE" -a -o json \
161+
| python3 -c 'import sys,json;w=json.load(sys.stdin)["osm-seed"]["web"]["image"];print(w["name"]+":"+w["tag"])')"
162+
echo "restore image: $WEB_IMAGE"
163+
kubectl -n "$NS" delete job "$RELEASE-restore" --ignore-not-found
164+
cat <<EOF | kubectl -n "$NS" apply -f -
165+
apiVersion: batch/v1
166+
kind: Job
167+
metadata:
168+
name: $RELEASE-restore
169+
spec:
170+
backoffLimit: 1
171+
ttlSecondsAfterFinished: 600
172+
template:
173+
spec:
174+
restartPolicy: Never
175+
containers:
176+
- name: restore
177+
image: $WEB_IMAGE
178+
env:
179+
- name: PGPASSWORD
180+
value: "${{ secrets.PREVIEW_DB_PASSWORD }}"
181+
- name: BACKUP_URL
182+
value: "$BACKUP_URL"
183+
command: ["bash","-c"]
184+
args:
185+
- |
186+
set -euo pipefail
187+
echo "waiting for db..."
188+
until pg_isready -h $RELEASE-db -p 5432; do sleep 2; done
189+
# the db is ephemeral but survives a helm upgrade that does not
190+
# restart it; skip the restore if it already has data.
191+
if [ "\$(psql -h $RELEASE-db -U postgres -d openhistoricalmap -tAc "SELECT to_regclass('public.users') IS NOT NULL")" = "t" ]; then
192+
echo "db already populated; skipping restore"
193+
exit 0
194+
fi
195+
echo "downloading + restoring backup..."
196+
# handle both plain .sql and gzipped .sql.gz
197+
case "\$BACKUP_URL" in
198+
*.gz) curl -fsSL "\$BACKUP_URL" | gunzip -c | psql -h $RELEASE-db -U postgres -d openhistoricalmap ;;
199+
*) curl -fsSL "\$BACKUP_URL" | psql -h $RELEASE-db -U postgres -d openhistoricalmap ;;
200+
esac
201+
echo "restore done"
202+
EOF
203+
kubectl -n "$NS" wait --for=condition=complete --timeout=30m job/$RELEASE-restore \
204+
|| { kubectl -n "$NS" logs job/$RELEASE-restore --tail=80; exit 1; }
205+
206+
# The chart renders the web Ingress (osm-seed.web.ingress.enabled) with a
207+
# router.middlewares annotation pointing at this Middleware, so create it
208+
# first. It forces X-Forwarded-Proto=https: Cloudflare already terminated
209+
# TLS, but Traefik would otherwise send http and the app's force_ssl would
210+
# redirect to https forever.
211+
- name: Create Traefik middleware (force https proto)
212+
run: |
213+
cat <<EOF | kubectl -n ${{ steps.n.outputs.namespace }} apply -f -
214+
apiVersion: traefik.io/v1alpha1
215+
kind: Middleware
216+
metadata:
217+
name: https-proto
218+
spec:
219+
headers:
220+
customRequestHeaders:
221+
X-Forwarded-Proto: https
222+
EOF
223+
224+
# Step 2: enable web. It waits for db, runs rails db:migrate on the restored
225+
# data, then serves. The chart renders the Traefik Ingress for the host.
226+
- name: Deploy web
227+
run: |
228+
helm upgrade --install ${{ steps.n.outputs.release }} ./ohm \
229+
-n ${{ steps.n.outputs.namespace }} \
230+
-f ./ohm/values.yaml \
231+
-f ./values.k3s.preview.yaml \
232+
--wait --timeout=20m
233+
234+
- name: Purge Cloudflare cache for the preview host
235+
continue-on-error: true
236+
env:
237+
CF_TOKEN: ${{ secrets.STAGING_CF_API_TOKEN }}
238+
CF_ZONE: ${{ secrets.STAGING_CF_ZONE_ID }}
239+
HOST: ${{ steps.n.outputs.host }}
240+
run: |
241+
if [ -z "$CF_TOKEN" ] || [ -z "$CF_ZONE" ]; then
242+
echo "no STAGING_CF_API_TOKEN / STAGING_CF_ZONE_ID; skipping purge"; exit 0
243+
fi
244+
resp=$(curl -sS -X POST \
245+
"https://api.cloudflare.com/client/v4/zones/$CF_ZONE/purge_cache" \
246+
-H "Authorization: Bearer $CF_TOKEN" \
247+
-H "Content-Type: application/json" \
248+
--data "{\"hosts\":[\"$HOST\"]}")
249+
echo "$resp"
250+
echo "$resp" | grep -q '"success":true' \
251+
|| echo "::warning::cache purge failed (purge-by-host needs Enterprise; or set a bypass Cache Rule for web-*.ohmstaging.org)"
252+
253+
- name: Summary
254+
if: always()
255+
run: |
256+
{
257+
echo "### Preview"
258+
echo ""
259+
echo "- branch: \`${{ steps.n.outputs.branch }}\`"
260+
echo "- url: https://${{ steps.n.outputs.host }}"
261+
echo "- release: \`${{ steps.n.outputs.release }}\` (ns \`${{ steps.n.outputs.namespace }}\`)"
262+
echo ""
263+
echo "Tear it down with the **Preview website teardown** workflow."
264+
} >> $GITHUB_STEP_SUMMARY

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,6 @@ images/tiler-server-martin/config/nginx.conf
4545
values.k3s.staging.direct.yaml
4646
ohm/charts/
4747
k3s.sh
48-
*.zip
48+
*.zip
49+
test-preview.sh
50+
values.k3s.preview.yaml

‎images/web/start.sh‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -130,11 +130,14 @@ setup_production() {
130130

131131
# Update map styles. This line should be removed later, as the configuration should come from the module.
132132
SERVER_URL_="${SERVER_URL/www./}"
133+
# Tiler host. Defaults to vtiles.<server domain>; override VTILES_DOMAIN to use a
134+
# shared tiler (e.g. a preview that has no tiler of its own).
135+
VTILES_DOMAIN="${VTILES_DOMAIN:-vtiles.${SERVER_URL_}}"
133136
find /var/www/node_modules/@openhistoricalmap/map-styles/dist/ -type f -name "*.json" -exec sed -i.bak "s|openhistoricalmap.github.io|${SERVER_URL}|g" {} +
134137
find /var/www/node_modules/@openhistoricalmap/map-styles/dist/ -type f -name "*.json" -exec sed -i.bak "s|http://localhost:8888|https://${SERVER_URL}/map-styles|g" {} +
135138
find /var/www/node_modules/@openhistoricalmap/map-styles/dist/ -type f -name "*.json" -exec sed -i.bak "s|www.openhistoricalmap.org|${SERVER_URL}|g" {} +
136-
find /var/www/node_modules/@openhistoricalmap/map-styles/dist/ -type f -name "*.json" -exec sed -i.bak "s|vtiles.openhistoricalmap.org|vtiles.${SERVER_URL_}|g" {} +
137-
find /var/www/node_modules/@openhistoricalmap/map-styles/dist/ -type f -name "*.json" -exec sed -i.bak "s|vtiles.staging.openhistoricalmap.org|vtiles.${SERVER_URL_}|g" {} +
139+
find /var/www/node_modules/@openhistoricalmap/map-styles/dist/ -type f -name "*.json" -exec sed -i.bak "s|vtiles.openhistoricalmap.org|${VTILES_DOMAIN}|g" {} +
140+
find /var/www/node_modules/@openhistoricalmap/map-styles/dist/ -type f -name "*.json" -exec sed -i.bak "s|vtiles.staging.openhistoricalmap.org|${VTILES_DOMAIN}|g" {} +
138141

139142
# Replace URLs in the public directory
140143
find "/var/www/public" -type f \( \
@@ -152,8 +155,8 @@ setup_production() {
152155
-e "s|openhistoricalmap.github.io|${SERVER_URL}|g" \
153156
-e "s|http://localhost:8888|https://${SERVER_URL}/map-styles|g" \
154157
-e "s|www.openhistoricalmap.org|${SERVER_URL}|g" \
155-
-e "s|vtiles.openhistoricalmap.org|vtiles.${SERVER_URL_}|g" \
156-
-e "s|vtiles.staging.openhistoricalmap.org|vtiles.${SERVER_URL_}|g" \
158+
-e "s|vtiles.openhistoricalmap.org|${VTILES_DOMAIN}|g" \
159+
-e "s|vtiles.staging.openhistoricalmap.org|${VTILES_DOMAIN}|g" \
157160
"$file"
158161
done
159162

0 commit comments

Comments
 (0)