Skip to content

Commit 35687f8

Browse files
author
Kevin Wang
committed
refactor(dstack-ingress): make one process own the certificate lifecycle
1 parent 8a180eb commit 35687f8

6 files changed

Lines changed: 366 additions & 336 deletions

File tree

custom-domain/dstack-ingress/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@ environment:
182182
| `TLSALPN_PORT` | `9443` | Loopback port lego's ACME responder binds to |
183183
| `TLS_TERMINATE_PORT` | `9444` | Loopback port the TLS frontend moves to in tls-alpn-01 mode |
184184
| `RENEW_DAYS_BEFORE` | lego default | Days of remaining lifetime that trigger renewal |
185+
| `RENEW_INTERVAL` | `43200` | Seconds between successful certificate passes |
185186
| `DNS_SETTLE_SECONDS` | `30` | Wait after DNS verifies, so the gateway's own TXT cache expires |
186187
| `MAXCONN` | `4096` | HAProxy max connections |
187188
| `TIMEOUT_CONNECT` | `10s` | Backend connect timeout |
Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#!/bin/bash
2+
# cert-manager.sh - the single owner of the certificate lifecycle.
3+
#
4+
# There used to be two drivers: a `bootstrap` that ran once at startup and a
5+
# renewal daemon that woke every 12 hours. Both ended up calling
6+
# renew-certificate.sh, so they were the same operation with different
7+
# preludes. That was harmless only because bootstrap finished before the daemon
8+
# started -- an ordering that stopped holding once tls-alpn-01 forced bootstrap
9+
# into the background (haproxy has to be listening before the first certificate
10+
# can be issued). The two then raced: both ACME clients tried to bind the same
11+
# challenge port, and the loser still spent one of Let's Encrypt's five failed
12+
# validations per hostname per hour.
13+
#
14+
# So there is now one pass, used for both jobs. process_domain is idempotent --
15+
# the DNS setters short-circuit when a record already holds the right value, the
16+
# ACME account registration returns early when it exists, and the ACME client
17+
# reports "no renewal needed" when the certificate is still fresh -- so the
18+
# first pass and the ten-thousandth are the same code.
19+
#
20+
# Usage:
21+
# cert-manager.sh --once run a single pass, exit with its status
22+
# cert-manager.sh run passes forever
23+
24+
set -uo pipefail
25+
26+
source /scripts/functions.sh
27+
source /scripts/domains.sh
28+
29+
# Self-sufficient rather than trusting the parent's exports, for the same
30+
# reason domains.sh carries its own defaults.
31+
if [ -z "${DSTACK_INSTANCE_ID:-}" ] || [ -z "${DSTACK_APP_ID:-}" ]; then
32+
load_dstack_identity
33+
fi
34+
35+
RENEW_INTERVAL=${RENEW_INTERVAL:-43200} # 12h between successful passes
36+
RETRY_INTERVAL_MIN=${RETRY_INTERVAL_MIN:-60} # backoff floor after a failed pass
37+
RETRY_INTERVAL_MAX=${RETRY_INTERVAL_MAX:-1800}
38+
39+
reload_haproxy() {
40+
if [ ! -f /var/run/haproxy/haproxy.pid ]; then
41+
# Normal during the dns-01 startup pass: haproxy has not been exec'd
42+
# yet, and it will read the fresh certificates when it starts.
43+
echo "HAProxy is not running yet; certificates will be picked up at start"
44+
return 0
45+
fi
46+
if kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)"; then
47+
echo "HAProxy reloaded with the new certificates"
48+
else
49+
echo "HAProxy reload failed: SIGUSR2 to PID $(cat /var/run/haproxy/haproxy.pid) failed" >&2
50+
return 1
51+
fi
52+
}
53+
54+
# One pass over every configured domain. Returns non-zero if any domain failed.
55+
run_pass() {
56+
local all_domains domain failed=0 changed=0
57+
58+
all_domains=$(get-all-domains.sh)
59+
if [ -z "$all_domains" ]; then
60+
echo "Error: No domains found. Set either DOMAIN or DOMAINS" >&2
61+
return 1
62+
fi
63+
64+
local status
65+
while IFS= read -r domain; do
66+
[[ -n "$domain" ]] || continue
67+
# Subshell: the process_domain helpers `exit` on fatal misconfiguration,
68+
# which should fail this domain rather than kill the manager.
69+
( process_domain "$domain" )
70+
status=$?
71+
# renew-certificate.sh reports 0 = certificate changed, 2 = nothing to
72+
# do, 1 = failure, and process_domain passes that through. Reloading
73+
# only on 0 is what keeps a quiet pass from churning haproxy.
74+
case $status in
75+
0) changed=1 ;;
76+
2) ;;
77+
*)
78+
echo "Certificate management failed for $domain" >&2
79+
failed=1
80+
;;
81+
esac
82+
done <<<"$all_domains"
83+
84+
if [ "$changed" -eq 1 ]; then
85+
generate-evidences.sh || echo "Evidence generation failed" >&2
86+
build-combined-pems.sh || echo "Combined PEM build failed" >&2
87+
reload_haproxy || true
88+
fi
89+
90+
if [ "$failed" -eq 0 ]; then
91+
touch /.bootstrapped
92+
return 0
93+
fi
94+
return 1
95+
}
96+
97+
if [ "${1:-}" = "--once" ]; then
98+
run_pass
99+
exit $?
100+
fi
101+
102+
attempt=0
103+
while true; do
104+
if run_pass; then
105+
attempt=0
106+
delay=$RENEW_INTERVAL
107+
else
108+
# Falling back to the 12-hour interval would be a terrible retry rate
109+
# for a flow whose normal state is "waiting for an operator to create a
110+
# DNS record". Back off, but stay responsive.
111+
attempt=$((attempt + 1))
112+
delay=$((RETRY_INTERVAL_MIN * attempt))
113+
[ "$delay" -gt "$RETRY_INTERVAL_MAX" ] && delay=$RETRY_INTERVAL_MAX
114+
echo "[$(date)] Pass failed (attempt ${attempt}); retrying in ${delay}s"
115+
fi
116+
echo "[$(date)] Next certificate check in ${delay}s"
117+
sleep "$delay"
118+
done
Lines changed: 228 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,228 @@
1+
#!/bin/bash
2+
# domains.sh - what DNS records each domain needs, and how to make them exist.
3+
#
4+
# Sourced by both entrypoint.sh and cert-manager.sh so there is exactly one
5+
# definition of "process this domain", whether it is the first pass at startup
6+
# or the periodic renewal pass.
7+
8+
# This file is sourced by two independent processes (entrypoint.sh and
9+
# cert-manager.sh), so it owns its own defaults rather than trusting the caller
10+
# to have computed and exported them. Getting that wrong is quiet and nasty: an
11+
# unset TXT_PREFIX builds a TXT name that can never exist, and the DNS wait then
12+
# blocks forever on a record nobody could create.
13+
#
14+
# GATEWAY_DOMAIN, CERTBOT_EMAIL and SET_CAA are genuinely external (compose
15+
# supplies them) and are deliberately not defaulted here.
16+
PORT=${PORT:-443}
17+
TXT_PREFIX=${TXT_PREFIX:-_dstack-app-address}
18+
CHALLENGE_TYPE=${CHALLENGE_TYPE:-dns-01}
19+
DNS_SETUP_MODE=${DNS_SETUP_MODE:-wait}
20+
DNS_SETTLE_SECONDS=${DNS_SETTLE_SECONDS:-30}
21+
SET_CAA=${SET_CAA:-false}
22+
APP_ID=${APP_ID:-}
23+
24+
set_alias_record() {
25+
local domain="$1"
26+
echo "Setting alias record for $domain"
27+
dnsman.py set_alias \
28+
--domain "$domain" \
29+
--content "$GATEWAY_DOMAIN"
30+
31+
if [ $? -ne 0 ]; then
32+
echo "Error: Failed to set alias record for $domain"
33+
exit 1
34+
fi
35+
echo "Alias record set for $domain"
36+
}
37+
38+
# Query the guest agent once for this app's identity.
39+
load_dstack_identity() {
40+
local info
41+
if [[ -S /var/run/dstack.sock ]]; then
42+
info=$(curl -s --unix-socket /var/run/dstack.sock http://localhost/Info)
43+
else
44+
info=$(curl -s --unix-socket /var/run/tappd.sock http://localhost/prpc/Tappd.Info)
45+
fi
46+
47+
DSTACK_APP_ID=$(echo "$info" | jq -j .app_id)
48+
DSTACK_INSTANCE_ID=$(echo "$info" | jq -j .instance_id)
49+
export DSTACK_APP_ID DSTACK_INSTANCE_ID
50+
51+
if [ -z "$DSTACK_APP_ID" ] || [ "$DSTACK_APP_ID" = "null" ]; then
52+
echo "Error: could not read app_id from the dstack guest agent" >&2
53+
exit 1
54+
fi
55+
if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ] &&
56+
{ [ -z "$DSTACK_INSTANCE_ID" ] || [ "$DSTACK_INSTANCE_ID" = "null" ]; }; then
57+
echo "Error: could not read instance_id from the dstack guest agent, which" >&2
58+
echo "tls-alpn-01 needs to pin gateway routing to this instance" >&2
59+
exit 1
60+
fi
61+
}
62+
63+
txt_record_name() {
64+
local domain="$1"
65+
if [[ "$domain" == \*.* ]]; then
66+
# Wildcard domain: *.myapp.com → _dstack-app-address-wildcard.myapp.com
67+
echo "${TXT_PREFIX}-wildcard.${domain#\*.}"
68+
else
69+
echo "${TXT_PREFIX}.${domain}"
70+
fi
71+
}
72+
73+
# What the gateway should route this hostname to.
74+
#
75+
# dns-01 uses the app id, so the gateway load-balances across every instance of
76+
# the app. tls-alpn-01 cannot: the challenge is answered by whichever instance
77+
# certbot runs on, and routing by app id makes the gateway race connections
78+
# across the app's instances (select_top_n_hosts -> connect_multiple_hosts)
79+
# while the CA validates from several vantage points at once. The challenge
80+
# would land on the wrong replica. Pinning to the instance id makes validation
81+
# deterministic -- at the cost of sending all traffic to this one instance, so
82+
# tls-alpn-01 mode is effectively single-instance.
83+
txt_record_value() {
84+
if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then
85+
echo "${DSTACK_INSTANCE_ID}:${PORT}"
86+
else
87+
echo "${APP_ID:-$DSTACK_APP_ID}:${PORT}"
88+
fi
89+
}
90+
91+
caa_tag_for() {
92+
if [[ "$1" == \*.* ]]; then echo "issuewild"; else echo "issue"; fi
93+
}
94+
95+
set_txt_record() {
96+
local domain="$1"
97+
local txt_domain
98+
txt_domain=$(txt_record_name "$domain")
99+
100+
dnsman.py set_txt \
101+
--domain "$txt_domain" \
102+
--content "$(txt_record_value)"
103+
104+
if [ $? -ne 0 ]; then
105+
echo "Error: Failed to set TXT record for $domain"
106+
exit 1
107+
fi
108+
}
109+
110+
set_caa_record() {
111+
local domain="$1"
112+
if [ "$SET_CAA" != "true" ]; then
113+
echo "Skipping CAA record setup"
114+
return
115+
fi
116+
117+
local ACCOUNT_URI
118+
local account_file
119+
120+
if ! account_file=$(get_letsencrypt_account_file); then
121+
echo "Warning: Cannot set CAA record - account file not found"
122+
echo "This is not critical - certificates can still be issued without CAA records"
123+
return
124+
fi
125+
126+
local caa_domain caa_tag
127+
caa_domain="${domain#\*.}"
128+
caa_tag=$(caa_tag_for "$domain")
129+
130+
ACCOUNT_URI=$(jq -j '.uri' "$account_file")
131+
echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$ACCOUNT_URI"
132+
dnsman.py set_caa \
133+
--domain "$caa_domain" \
134+
--caa-tag "$caa_tag" \
135+
--caa-value "letsencrypt.org;validationmethods=${CHALLENGE_TYPE};accounturi=$ACCOUNT_URI"
136+
137+
if [ $? -ne 0 ]; then
138+
echo "Warning: Failed to set CAA record for $domain"
139+
echo "This is not critical - certificates can still be issued without CAA records"
140+
echo "Consider disabling CAA records by setting SET_CAA=false if this continues to fail"
141+
fi
142+
}
143+
144+
process_domain_dns01() {
145+
local domain="$1"
146+
147+
set_alias_record "$domain"
148+
set_txt_record "$domain"
149+
renew-certificate.sh "$domain" || echo "First certificate renewal failed for $domain, will retry after set CAA record"
150+
set_caa_record "$domain"
151+
renew-certificate.sh "$domain"
152+
}
153+
154+
process_domain_tlsalpn() {
155+
local domain="$1"
156+
157+
if [[ "$domain" == \*.* ]]; then
158+
echo "Error: cannot issue a wildcard certificate for $domain with tls-alpn-01." >&2
159+
echo "RFC 8737 forbids it; use CHALLENGE_TYPE=dns-01 for wildcards." >&2
160+
exit 1
161+
fi
162+
163+
# Register the ACME account first so the CAA record we print can already
164+
# pin accounturi. Without this the operator would have to add CAA in a
165+
# second pass, after the account exists.
166+
if ! legoman.py register --email "$CERTBOT_EMAIL"; then
167+
echo "Error: failed to register the ACME account" >&2
168+
exit 1
169+
fi
170+
171+
local account_uri caa_value
172+
account_uri=$(legoman.py account-uri --email "$CERTBOT_EMAIL" 2>/dev/null || true)
173+
if [ -n "$account_uri" ]; then
174+
caa_value="letsencrypt.org;validationmethods=tls-alpn-01;accounturi=$account_uri"
175+
else
176+
echo "Warning: could not read the ACME account URI; the CAA record will not pin it" >&2
177+
caa_value="letsencrypt.org;validationmethods=tls-alpn-01"
178+
fi
179+
180+
# Wait for the records routing depends on. An existing CAA record is also
181+
# checked for compatibility -- not because CAA is required (an absent CAA
182+
# record set permits every CA), but because an incompatible one makes the CA
183+
# refuse and burns Let's Encrypt's failed-validation budget: 5 per account
184+
# per hostname per hour.
185+
if ! dnsguide.py \
186+
--domain "$domain" \
187+
--alias-target "$GATEWAY_DOMAIN" \
188+
--txt-name "$(txt_record_name "$domain")" \
189+
--txt-value "$(txt_record_value)" \
190+
--caa-name "${domain#\*.}" \
191+
--caa-tag "$(caa_tag_for "$domain")" \
192+
--caa-value "$caa_value" \
193+
--account-uri "$account_uri" \
194+
--challenge tls-alpn-01 \
195+
--mode "$DNS_SETUP_MODE"; then
196+
echo "Error: required DNS records for $domain are not in place" >&2
197+
exit 1
198+
fi
199+
200+
# Public DNS being correct is not the same as the gateway acting on it. The
201+
# gateway resolves the app-address TXT itself and caches the answer for the
202+
# record's TTL, so right after the value changes -- which is every time the
203+
# CVM instance is replaced, since the record names the instance -- it can
204+
# still route the challenge to the previous target. Observed exactly that:
205+
# dnsguide passed, then the CA got "Error getting validation data" because
206+
# the gateway was still sending it to the old instance.
207+
# Only on the first pass: the settle wait exists because the TXT value has
208+
# just changed, which is true at bootstrap (and after an instance
209+
# replacement, which is a fresh container and therefore also a first pass).
210+
# Renewal passes reuse a record the gateway resolved long ago.
211+
if [ ! -f /.bootstrapped ] && [ "${DNS_SETTLE_SECONDS}" -gt 0 ]; then
212+
echo "Waiting ${DNS_SETTLE_SECONDS}s for the gateway's DNS cache to expire"
213+
sleep "${DNS_SETTLE_SECONDS}"
214+
fi
215+
216+
renew-certificate.sh "$domain"
217+
}
218+
219+
process_domain() {
220+
local domain="$1"
221+
echo "Processing domain: $domain (challenge: $CHALLENGE_TYPE)"
222+
223+
if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then
224+
process_domain_tlsalpn "$domain"
225+
else
226+
process_domain_dns01 "$domain"
227+
fi
228+
}

0 commit comments

Comments
 (0)