fix(dstack-ingress): retry applying a renewed certificate until it sticks

Renewing and applying were treated as one step:

    if [ "$changed" -eq 1 ]; then
        collect_evidence     || echo "Evidence generation failed" >&2
        build_combined_pems  || echo "Combined PEM build failed" >&2
        haproxy_reload       || true
    fi

Every failure there is swallowed, and none of them set `failed`, so the
pass still reports success. Twelve hours later the ACME client sees a
fresh certificate on disk, reports "nothing to renew", `changed` is 0,
and the apply is never attempted again. HAProxy goes on serving the
pre-renewal certificate until it expires -- which is soon, because being
close to expiry is why it renewed in the first place. The symptom arrives
weeks after the cause, as an expired certificate, with nothing in the
logs at that moment to connect the two.

Separate the two: `APPLY_PENDING` is set when something renewed and only
cleared once evidence, PEMs and the reload have all succeeded, so a
failed apply is retried on the next pass instead of waiting for the next
renewal.

Quiet passes stay quiet. With nothing renewed and nothing pending the
block does not run at all, so this does not bring back the reload-on-
every-check behaviour that made HAProxy replace its worker twice a day.

Kept in both mode scripts rather than lifted into a library: the apply
steps are mode-specific (certbot and lego lay certificates out
differently), so a shared helper would have to call back into functions
its callers define, which is harder to follow than the dozen lines it
would save.

Author: pacoyang

custom-domain/dstack-ingress/scripts/dns01.sh

@@ -288,6 +288,10 @@ collect_evidence() {
     evidence_finalize
 }
 
+# Set when certificates have been renewed but not yet applied to haproxy.
+# Survives across passes, so a failed apply is retried on the next one.
+APPLY_PENDING=false
+
 # One pass over every domain: make the DNS records right, then issue or renew.
 # Idempotent, so the first pass and every later one are the same code.
 run_pass() {
@@ -305,10 +309,27 @@ run_pass() {
         esac
     done <<<"$(get-all-domains.sh)"
 
+    # `if`, not `[ ... ] && ...`: the AND-list returns non-zero when nothing
+    # changed, and the startup pass runs bare under `set -e`.
     if [ "$changed" -eq 1 ]; then
-        collect_evidence || echo "Evidence generation failed" >&2
-        build_combined_pems || echo "Combined PEM build failed" >&2
-        haproxy_reload || true
+        APPLY_PENDING=true
+    fi
+
+    # Applying is separate from renewing, and retried until it sticks. A
+    # renewal that reaches disk but not haproxy is invisible: the next pass
+    # sees a fresh certificate, reports "nothing to renew", and never reloads
+    # again -- so haproxy would serve the pre-renewal certificate until it
+    # expires, which is soon, because being near expiry is why it renewed.
+    if [ "$APPLY_PENDING" = true ]; then
+        local applied=true
+        collect_evidence || { echo "Evidence generation failed" >&2; applied=false; }
+        build_combined_pems || { echo "Combined PEM build failed" >&2; applied=false; }
+        haproxy_reload || applied=false
+        if [ "$applied" = true ]; then
+            APPLY_PENDING=false
+        else
+            echo "[$(date)] Certificate apply incomplete; retrying on the next pass" >&2
+        fi
     fi
     [ "$failed" -eq 0 ]
 }

custom-domain/dstack-ingress/scripts/tlsalpn.sh

@@ -262,6 +262,10 @@ process_domain() {
     legoman.py auto --domain "$domain" ${ACME_EMAIL:+--email "$ACME_EMAIL"}
 }
 
+# Set when certificates have been renewed but not yet applied to haproxy.
+# Survives across passes, so a failed apply is retried on the next one.
+APPLY_PENDING=false
+
 # One pass over every domain: make sure the records exist, then issue or renew.
 # Idempotent, so the first pass and every later one are the same code. There is
 # deliberately only one of these -- an earlier split between a one-shot
@@ -284,10 +288,27 @@ run_pass() {
         esac
     done <<<"$(get-all-domains.sh)"
 
+    # `if`, not `[ ... ] && ...`: the AND-list returns non-zero when nothing
+    # changed, and the startup pass runs bare under `set -e`.
     if [ "$changed" -eq 1 ]; then
-        collect_evidence || echo "Evidence generation failed" >&2
-        build_combined_pems || echo "Combined PEM build failed" >&2
-        haproxy_reload || true
+        APPLY_PENDING=true
+    fi
+
+    # Applying is separate from renewing, and retried until it sticks. A
+    # renewal that reaches disk but not haproxy is invisible: the next pass
+    # sees a fresh certificate, reports "nothing to renew", and never reloads
+    # again -- so haproxy would serve the pre-renewal certificate until it
+    # expires, which is soon, because being near expiry is why it renewed.
+    if [ "$APPLY_PENDING" = true ]; then
+        local applied=true
+        collect_evidence || { echo "Evidence generation failed" >&2; applied=false; }
+        build_combined_pems || { echo "Combined PEM build failed" >&2; applied=false; }
+        haproxy_reload || applied=false
+        if [ "$applied" = true ]; then
+            APPLY_PENDING=false
+        else
+            echo "[$(date)] Certificate apply incomplete; retrying on the next pass" >&2
+        fi
     fi
     FIRST_PASS=false
     [ "$failed" -eq 0 ]