|
| 1 | +#!/bin/bash |
| 2 | +# domains.sh - what DNS records each domain needs, and how to make them exist. |
| 3 | +# |
| 4 | +# Sourced by both entrypoint.sh and cert-manager.sh so there is exactly one |
| 5 | +# definition of "process this domain", whether it is the first pass at startup |
| 6 | +# or the periodic renewal pass. |
| 7 | + |
| 8 | +# This file is sourced by two independent processes (entrypoint.sh and |
| 9 | +# cert-manager.sh), so it owns its own defaults rather than trusting the caller |
| 10 | +# to have computed and exported them. Getting that wrong is quiet and nasty: an |
| 11 | +# unset TXT_PREFIX builds a TXT name that can never exist, and the DNS wait then |
| 12 | +# blocks forever on a record nobody could create. |
| 13 | +# |
| 14 | +# GATEWAY_DOMAIN, CERTBOT_EMAIL and SET_CAA are genuinely external (compose |
| 15 | +# supplies them) and are deliberately not defaulted here. |
| 16 | +PORT=${PORT:-443} |
| 17 | +TXT_PREFIX=${TXT_PREFIX:-_dstack-app-address} |
| 18 | +CHALLENGE_TYPE=${CHALLENGE_TYPE:-dns-01} |
| 19 | +DNS_SETUP_MODE=${DNS_SETUP_MODE:-wait} |
| 20 | +DNS_SETTLE_SECONDS=${DNS_SETTLE_SECONDS:-30} |
| 21 | +SET_CAA=${SET_CAA:-false} |
| 22 | +APP_ID=${APP_ID:-} |
| 23 | + |
| 24 | +set_alias_record() { |
| 25 | + local domain="$1" |
| 26 | + echo "Setting alias record for $domain" |
| 27 | + dnsman.py set_alias \ |
| 28 | + --domain "$domain" \ |
| 29 | + --content "$GATEWAY_DOMAIN" |
| 30 | + |
| 31 | + if [ $? -ne 0 ]; then |
| 32 | + echo "Error: Failed to set alias record for $domain" |
| 33 | + exit 1 |
| 34 | + fi |
| 35 | + echo "Alias record set for $domain" |
| 36 | +} |
| 37 | + |
| 38 | +# Query the guest agent once for this app's identity. |
| 39 | +load_dstack_identity() { |
| 40 | + local info |
| 41 | + if [[ -S /var/run/dstack.sock ]]; then |
| 42 | + info=$(curl -s --unix-socket /var/run/dstack.sock http://localhost/Info) |
| 43 | + else |
| 44 | + info=$(curl -s --unix-socket /var/run/tappd.sock http://localhost/prpc/Tappd.Info) |
| 45 | + fi |
| 46 | + |
| 47 | + DSTACK_APP_ID=$(echo "$info" | jq -j .app_id) |
| 48 | + DSTACK_INSTANCE_ID=$(echo "$info" | jq -j .instance_id) |
| 49 | + export DSTACK_APP_ID DSTACK_INSTANCE_ID |
| 50 | + |
| 51 | + if [ -z "$DSTACK_APP_ID" ] || [ "$DSTACK_APP_ID" = "null" ]; then |
| 52 | + echo "Error: could not read app_id from the dstack guest agent" >&2 |
| 53 | + exit 1 |
| 54 | + fi |
| 55 | + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ] && |
| 56 | + { [ -z "$DSTACK_INSTANCE_ID" ] || [ "$DSTACK_INSTANCE_ID" = "null" ]; }; then |
| 57 | + echo "Error: could not read instance_id from the dstack guest agent, which" >&2 |
| 58 | + echo "tls-alpn-01 needs to pin gateway routing to this instance" >&2 |
| 59 | + exit 1 |
| 60 | + fi |
| 61 | +} |
| 62 | + |
| 63 | +txt_record_name() { |
| 64 | + local domain="$1" |
| 65 | + if [[ "$domain" == \*.* ]]; then |
| 66 | + # Wildcard domain: *.myapp.com → _dstack-app-address-wildcard.myapp.com |
| 67 | + echo "${TXT_PREFIX}-wildcard.${domain#\*.}" |
| 68 | + else |
| 69 | + echo "${TXT_PREFIX}.${domain}" |
| 70 | + fi |
| 71 | +} |
| 72 | + |
| 73 | +# What the gateway should route this hostname to. |
| 74 | +# |
| 75 | +# dns-01 uses the app id, so the gateway load-balances across every instance of |
| 76 | +# the app. tls-alpn-01 cannot: the challenge is answered by whichever instance |
| 77 | +# certbot runs on, and routing by app id makes the gateway race connections |
| 78 | +# across the app's instances (select_top_n_hosts -> connect_multiple_hosts) |
| 79 | +# while the CA validates from several vantage points at once. The challenge |
| 80 | +# would land on the wrong replica. Pinning to the instance id makes validation |
| 81 | +# deterministic -- at the cost of sending all traffic to this one instance, so |
| 82 | +# tls-alpn-01 mode is effectively single-instance. |
| 83 | +txt_record_value() { |
| 84 | + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then |
| 85 | + echo "${DSTACK_INSTANCE_ID}:${PORT}" |
| 86 | + else |
| 87 | + echo "${APP_ID:-$DSTACK_APP_ID}:${PORT}" |
| 88 | + fi |
| 89 | +} |
| 90 | + |
| 91 | +caa_tag_for() { |
| 92 | + if [[ "$1" == \*.* ]]; then echo "issuewild"; else echo "issue"; fi |
| 93 | +} |
| 94 | + |
| 95 | +set_txt_record() { |
| 96 | + local domain="$1" |
| 97 | + local txt_domain |
| 98 | + txt_domain=$(txt_record_name "$domain") |
| 99 | + |
| 100 | + dnsman.py set_txt \ |
| 101 | + --domain "$txt_domain" \ |
| 102 | + --content "$(txt_record_value)" |
| 103 | + |
| 104 | + if [ $? -ne 0 ]; then |
| 105 | + echo "Error: Failed to set TXT record for $domain" |
| 106 | + exit 1 |
| 107 | + fi |
| 108 | +} |
| 109 | + |
| 110 | +set_caa_record() { |
| 111 | + local domain="$1" |
| 112 | + if [ "$SET_CAA" != "true" ]; then |
| 113 | + echo "Skipping CAA record setup" |
| 114 | + return |
| 115 | + fi |
| 116 | + |
| 117 | + local ACCOUNT_URI |
| 118 | + local account_file |
| 119 | + |
| 120 | + if ! account_file=$(get_letsencrypt_account_file); then |
| 121 | + echo "Warning: Cannot set CAA record - account file not found" |
| 122 | + echo "This is not critical - certificates can still be issued without CAA records" |
| 123 | + return |
| 124 | + fi |
| 125 | + |
| 126 | + local caa_domain caa_tag |
| 127 | + caa_domain="${domain#\*.}" |
| 128 | + caa_tag=$(caa_tag_for "$domain") |
| 129 | + |
| 130 | + ACCOUNT_URI=$(jq -j '.uri' "$account_file") |
| 131 | + echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$ACCOUNT_URI" |
| 132 | + dnsman.py set_caa \ |
| 133 | + --domain "$caa_domain" \ |
| 134 | + --caa-tag "$caa_tag" \ |
| 135 | + --caa-value "letsencrypt.org;validationmethods=${CHALLENGE_TYPE};accounturi=$ACCOUNT_URI" |
| 136 | + |
| 137 | + if [ $? -ne 0 ]; then |
| 138 | + echo "Warning: Failed to set CAA record for $domain" |
| 139 | + echo "This is not critical - certificates can still be issued without CAA records" |
| 140 | + echo "Consider disabling CAA records by setting SET_CAA=false if this continues to fail" |
| 141 | + fi |
| 142 | +} |
| 143 | + |
| 144 | +process_domain_dns01() { |
| 145 | + local domain="$1" |
| 146 | + |
| 147 | + set_alias_record "$domain" |
| 148 | + set_txt_record "$domain" |
| 149 | + renew-certificate.sh "$domain" || echo "First certificate renewal failed for $domain, will retry after set CAA record" |
| 150 | + set_caa_record "$domain" |
| 151 | + renew-certificate.sh "$domain" |
| 152 | +} |
| 153 | + |
| 154 | +process_domain_tlsalpn() { |
| 155 | + local domain="$1" |
| 156 | + |
| 157 | + if [[ "$domain" == \*.* ]]; then |
| 158 | + echo "Error: cannot issue a wildcard certificate for $domain with tls-alpn-01." >&2 |
| 159 | + echo "RFC 8737 forbids it; use CHALLENGE_TYPE=dns-01 for wildcards." >&2 |
| 160 | + exit 1 |
| 161 | + fi |
| 162 | + |
| 163 | + # Register the ACME account first so the CAA record we print can already |
| 164 | + # pin accounturi. Without this the operator would have to add CAA in a |
| 165 | + # second pass, after the account exists. |
| 166 | + if ! legoman.py register --email "$CERTBOT_EMAIL"; then |
| 167 | + echo "Error: failed to register the ACME account" >&2 |
| 168 | + exit 1 |
| 169 | + fi |
| 170 | + |
| 171 | + local account_uri caa_value |
| 172 | + account_uri=$(legoman.py account-uri --email "$CERTBOT_EMAIL" 2>/dev/null || true) |
| 173 | + if [ -n "$account_uri" ]; then |
| 174 | + caa_value="letsencrypt.org;validationmethods=tls-alpn-01;accounturi=$account_uri" |
| 175 | + else |
| 176 | + echo "Warning: could not read the ACME account URI; the CAA record will not pin it" >&2 |
| 177 | + caa_value="letsencrypt.org;validationmethods=tls-alpn-01" |
| 178 | + fi |
| 179 | + |
| 180 | + # Wait for the records routing depends on. An existing CAA record is also |
| 181 | + # checked for compatibility -- not because CAA is required (an absent CAA |
| 182 | + # record set permits every CA), but because an incompatible one makes the CA |
| 183 | + # refuse and burns Let's Encrypt's failed-validation budget: 5 per account |
| 184 | + # per hostname per hour. |
| 185 | + if ! dnsguide.py \ |
| 186 | + --domain "$domain" \ |
| 187 | + --alias-target "$GATEWAY_DOMAIN" \ |
| 188 | + --txt-name "$(txt_record_name "$domain")" \ |
| 189 | + --txt-value "$(txt_record_value)" \ |
| 190 | + --caa-name "${domain#\*.}" \ |
| 191 | + --caa-tag "$(caa_tag_for "$domain")" \ |
| 192 | + --caa-value "$caa_value" \ |
| 193 | + --account-uri "$account_uri" \ |
| 194 | + --challenge tls-alpn-01 \ |
| 195 | + --mode "$DNS_SETUP_MODE"; then |
| 196 | + echo "Error: required DNS records for $domain are not in place" >&2 |
| 197 | + exit 1 |
| 198 | + fi |
| 199 | + |
| 200 | + # Public DNS being correct is not the same as the gateway acting on it. The |
| 201 | + # gateway resolves the app-address TXT itself and caches the answer for the |
| 202 | + # record's TTL, so right after the value changes -- which is every time the |
| 203 | + # CVM instance is replaced, since the record names the instance -- it can |
| 204 | + # still route the challenge to the previous target. Observed exactly that: |
| 205 | + # dnsguide passed, then the CA got "Error getting validation data" because |
| 206 | + # the gateway was still sending it to the old instance. |
| 207 | + # Only on the first pass: the settle wait exists because the TXT value has |
| 208 | + # just changed, which is true at bootstrap (and after an instance |
| 209 | + # replacement, which is a fresh container and therefore also a first pass). |
| 210 | + # Renewal passes reuse a record the gateway resolved long ago. |
| 211 | + if [ ! -f /.bootstrapped ] && [ "${DNS_SETTLE_SECONDS}" -gt 0 ]; then |
| 212 | + echo "Waiting ${DNS_SETTLE_SECONDS}s for the gateway's DNS cache to expire" |
| 213 | + sleep "${DNS_SETTLE_SECONDS}" |
| 214 | + fi |
| 215 | + |
| 216 | + renew-certificate.sh "$domain" |
| 217 | +} |
| 218 | + |
| 219 | +process_domain() { |
| 220 | + local domain="$1" |
| 221 | + echo "Processing domain: $domain (challenge: $CHALLENGE_TYPE)" |
| 222 | + |
| 223 | + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then |
| 224 | + process_domain_tlsalpn "$domain" |
| 225 | + else |
| 226 | + process_domain_dns01 "$domain" |
| 227 | + fi |
| 228 | +} |
0 commit comments