fix(dstack-ingress): reload haproxy only when a certificate was actually renewed

certman.py distinguishes 'renewed' (0) from 'nothing to renew' (2), but
renew-certificate.sh collapsed both to exit 0, so the renewal daemon
rebuilt the PEMs and reloaded haproxy on every 12-hour check cycle even
though certificates renew roughly every 60 days. Propagate exit 2 and
make the daemon treat only a real renewal as reload-worthy.

entrypoint.sh runs under set -e and calls renew-certificate.sh
unguarded during bootstrap, where the certificate has just been issued
and the second call now exits 2 — tolerate that explicitly so
bootstrap does not abort.

Author: pacoyang

custom-domain/dstack-ingress/scripts/entrypoint.sh

@@ -356,9 +356,11 @@ process_domain() {
 
     set_alias_record "$domain"
     set_txt_record "$domain"
-    renew-certificate.sh "$domain" || echo "First certificate renewal failed for $domain, will retry after set CAA record"
+    # renew-certificate.sh exits 0 on renewal, 2 when nothing needed renewal
+    # (not a failure — this script runs under `set -e`), 1 on real failure.
+    renew-certificate.sh "$domain" || [ $? -eq 2 ] || echo "First certificate renewal failed for $domain, will retry after set CAA record"
     set_caa_record "$domain"
-    renew-certificate.sh "$domain"
+    renew-certificate.sh "$domain" || [ $? -eq 2 ]
 }
 
 bootstrap() {
@@ -400,8 +402,12 @@ else
     generate-evidences.sh
 fi
 
-# Build combined PEM files for haproxy
+# Build combined PEM files for haproxy. Stamp the build so the renewal
+# daemon's certs_pending_apply check starts from "everything applied" —
+# haproxy loads these fresh PEMs itself at startup, no reload pending.
 build-combined-pems.sh
+mkdir -p /var/run/haproxy
+touch /var/run/haproxy/renewal-applied.stamp
 
 # Generate haproxy config
 if [ -n "$ROUTING_MAP" ]; then

custom-domain/dstack-ingress/scripts/renew-certificate.sh

@@ -8,11 +8,17 @@ SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
 python3 "$SCRIPT_DIR/certman.py" auto --domain "$DOMAIN" --email "$CERTBOT_EMAIL"
 CERT_STATUS=$?
 
-if [ $CERT_STATUS -eq 1 ]; then
-    echo "Certificate management failed" >&2
-    exit 1
-elif [ $CERT_STATUS -eq 2 ]; then
+if [ $CERT_STATUS -eq 2 ]; then
     echo "No certificates need renewal, skipping evidence generation"
+    # Propagate "nothing renewed" so the renewal daemon does not rebuild
+    # PEMs and reload haproxy on every check cycle.
+    exit 2
+elif [ $CERT_STATUS -ne 0 ]; then
+    # 1 is certman's failure exit; anything else (127 missing interpreter,
+    # 130 signal, ...) is just as much a failure — never let it fall through
+    # to "renewed", which would trigger a spurious PEM rebuild + reload.
+    echo "Certificate management failed (status $CERT_STATUS)" >&2
+    exit 1
 fi
 
 exit 0

custom-domain/dstack-ingress/scripts/renewal-daemon.sh

@@ -1,5 +1,20 @@
 #!/bin/bash
 
+# Written after renewed certificates have been fully applied (evidences +
+# combined PEMs rebuilt + haproxy reloaded). Certificate material newer than
+# this stamp means a renewal has not reached haproxy yet, so the apply step
+# retries on the next cycle instead of waiting for the next renewal (~60
+# days) — without it, a single failed reload would leave haproxy serving the
+# old certificate until it expires. The entrypoint touches the stamp after
+# the initial PEM build, so a fresh container start does not trigger a
+# spurious reload.
+STAMP=/var/run/haproxy/renewal-applied.stamp
+
+certs_pending_apply() {
+    [ -f "$STAMP" ] || return 0
+    [ -n "$(find /etc/letsencrypt/live -name fullchain.pem -newer "$STAMP" -print -quit 2>/dev/null)" ]
+}
+
 while true; do
     echo "[$(date)] Checking for certificate renewal"
 
@@ -10,28 +25,43 @@ while true; do
         while IFS= read -r domain; do
             [[ -n "$domain" ]] || continue
             echo "[$(date)] Checking renewal for domain: $domain"
-            if renew-certificate.sh "$domain"; then
+            renew-certificate.sh "$domain"
+            status=$?
+            # 0 = a certificate was actually renewed; 2 = nothing to renew.
+            # Only a real renewal warrants a PEM rebuild + haproxy reload —
+            # reloading on every check cycle replaced the worker (and its
+            # accumulated state) twice a day for no reason.
+            if [ $status -eq 0 ]; then
                 renewal_occurred=true
-            else
-                echo "Certificate renewal check failed for $domain with status $?"
+            elif [ $status -ne 2 ]; then
+                echo "Certificate renewal check failed for $domain with status $status"
             fi
         done <<<"$all_domains"
 
-        if [ "$renewal_occurred" = true ]; then
-            echo "[$(date)] Generating evidence files after renewals..."
-            generate-evidences.sh || echo "Evidence generation failed"
+        if [ "$renewal_occurred" = true ] || certs_pending_apply; then
+            apply_ok=true
+            echo "[$(date)] Applying renewed certificates..."
+            generate-evidences.sh || { echo "Evidence generation failed"; apply_ok=false; }
 
             # Rebuild combined PEM files for haproxy
-            build-combined-pems.sh || echo "Combined PEM build failed"
+            build-combined-pems.sh || { echo "Combined PEM build failed"; apply_ok=false; }
 
             # Graceful reload: send SIGUSR2 to haproxy master process
             if [ ! -f /var/run/haproxy/haproxy.pid ]; then
                 echo "HAProxy reload failed: PID file /var/run/haproxy/haproxy.pid not found" >&2
+                apply_ok=false
             elif ! kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)"; then
                 echo "HAProxy reload failed: SIGUSR2 to PID $(cat /var/run/haproxy/haproxy.pid) failed" >&2
+                apply_ok=false
             else
                 echo "Certificate renewed and HAProxy reloaded successfully"
             fi
+
+            if [ "$apply_ok" = true ]; then
+                touch "$STAMP"
+            else
+                echo "[$(date)] Certificate apply incomplete; will retry on the next check cycle"
+            fi
         fi
     else
         echo "[$(date)] No domains configured for renewal"