fix(dstack-ingress): serialize bootstrap with the renewal daemon and let the gateway's DNS cache expire

Author: Kevin Wang

custom-domain/dstack-ingress/README.md

@@ -182,6 +182,7 @@ environment:
 | `TLSALPN_PORT` | `9443` | Loopback port lego's ACME responder binds to |
 | `TLS_TERMINATE_PORT` | `9444` | Loopback port the TLS frontend moves to in tls-alpn-01 mode |
 | `RENEW_DAYS_BEFORE` | lego default | Days of remaining lifetime that trigger renewal |
+| `DNS_SETTLE_SECONDS` | `30` | Wait after DNS verifies, so the gateway's own TXT cache expires |
 | `MAXCONN` | `4096` | HAProxy max connections |
 | `TIMEOUT_CONNECT` | `10s` | Backend connect timeout |
 | `TIMEOUT_CLIENT` | `86400s` | Client-side timeout (24h for long-lived connections) |

custom-domain/dstack-ingress/scripts/entrypoint.sh

@@ -528,6 +528,18 @@ process_domain_tlsalpn() {
         exit 1
     fi
 
+    # Public DNS being correct is not the same as the gateway acting on it. The
+    # gateway resolves the app-address TXT itself and caches the answer for the
+    # record's TTL, so right after the value changes -- which is every time the
+    # CVM instance is replaced, since the record names the instance -- it can
+    # still route the challenge to the previous target. Observed exactly that:
+    # dnsguide passed, then the CA got "Error getting validation data" because
+    # the gateway was still sending it to the old instance.
+    if [ "${DNS_SETTLE_SECONDS:-30}" -gt 0 ]; then
+        echo "Waiting ${DNS_SETTLE_SECONDS:-30}s for the gateway's DNS cache to expire"
+        sleep "${DNS_SETTLE_SECONDS:-30}"
+    fi
+
     renew-certificate.sh "$domain"
 }
 
@@ -591,18 +603,30 @@ ensure_placeholder_certs() {
 
 # Obtain certificates once the proxy is already serving, then swap them in.
 bootstrap_and_reload() {
-    # Nested subshell so an `exit 1` from a process_domain helper is reported as
-    # a failed condition here rather than silently killing this background job.
-    if ( bootstrap ); then
-        build-combined-pems.sh || echo "Combined PEM build failed" >&2
-        if [ -f /var/run/haproxy/haproxy.pid ]; then
-            kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)" &&
-                echo "Certificates installed and HAProxy reloaded"
+    local attempt=0 delay
+    while true; do
+        attempt=$((attempt + 1))
+        # Nested subshell so an `exit 1` from a process_domain helper is
+        # reported as a failed condition here rather than silently killing
+        # this background job.
+        if ( bootstrap ); then
+            build-combined-pems.sh || echo "Combined PEM build failed" >&2
+            if [ -f /var/run/haproxy/haproxy.pid ]; then
+                kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)" &&
+                    echo "Certificates installed and HAProxy reloaded"
+            fi
+            return 0
         fi
-    else
-        echo "Bootstrap failed; the proxy keeps serving the placeholder certificate." >&2
-        echo "Fix the DNS records above; the renewal daemon retries every 12 hours." >&2
-    fi
+
+        # Falling through to the 12-hour renewal daemon would be a terrible
+        # retry interval for a flow whose normal state is "waiting for the
+        # operator to create a DNS record". Back off, but keep trying.
+        delay=$((60 * attempt))
+        [ "$delay" -gt 1800 ] && delay=1800
+        echo "Bootstrap attempt ${attempt} failed; the proxy keeps serving the" >&2
+        echo "placeholder certificate. Retrying in ${delay}s." >&2
+        sleep "$delay"
+    done
 }
 
 # Credentials are now handled by certman.py setup

custom-domain/dstack-ingress/scripts/renewal-daemon.sh

@@ -1,6 +1,18 @@
 #!/bin/bash
 
 while true; do
+    # Never run concurrently with the initial bootstrap. Under tls-alpn-01 the
+    # proxy has to be listening before the first certificate can be issued, so
+    # bootstrap runs in the background while this daemon is already up. Two
+    # ACME clients at once fight over the challenge port ("bind: Address
+    # already in use") and, worse, the loser still spends one of Let's
+    # Encrypt's five failed validations per hostname per hour.
+    if [ ! -f /.bootstrapped ]; then
+        echo "[$(date)] Waiting for bootstrap to finish before checking renewals"
+        sleep 30
+        continue
+    fi
+
     echo "[$(date)] Checking for certificate renewal"
 
     all_domains=$(get-all-domains.sh)