From f61e591f9e98df59ea27e7aabbd144b0a3009345 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 18:01:37 -0700 Subject: [PATCH 01/16] feat(dstack-ingress): add tls-alpn-01 mode that needs no DNS credentials --- custom-domain/dstack-ingress/Dockerfile | 16 + custom-domain/dstack-ingress/README.md | 129 ++++- .../scripts/build-combined-pems.sh | 7 +- .../dstack-ingress/scripts/dnsguide.py | 457 ++++++++++++++++++ .../dstack-ingress/scripts/dnshook.py | 183 +++++++ .../dstack-ingress/scripts/entrypoint.sh | 299 ++++++++++-- .../dstack-ingress/scripts/functions.sh | 62 +++ .../scripts/generate-evidences.sh | 14 +- .../dstack-ingress/scripts/legoman.py | 240 +++++++++ .../scripts/renew-certificate.sh | 10 +- .../scripts/tests/test_dnsguide.py | 130 +++++ 11 files changed, 1509 insertions(+), 38 deletions(-) create mode 100644 custom-domain/dstack-ingress/scripts/dnsguide.py create mode 100644 custom-domain/dstack-ingress/scripts/dnshook.py create mode 100644 custom-domain/dstack-ingress/scripts/legoman.py create mode 100644 custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py diff --git a/custom-domain/dstack-ingress/Dockerfile b/custom-domain/dstack-ingress/Dockerfile index a018ac95..a6452448 100644 --- a/custom-domain/dstack-ingress/Dockerfile +++ b/custom-domain/dstack-ingress/Dockerfile @@ -34,6 +34,22 @@ RUN --mount=type=bind,source=pinned-packages.txt,target=/tmp/pinned-packages.txt mini-httpd && \ rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache +# lego, for the tls-alpn-01 challenge. certbot cannot do tls-alpn-01: its +# standalone plugin is HTTP-01 only and the acme library removed the challenge +# in 4.2.0. Pinned by checksum so the build stays reproducible; bump both the +# version and the digest together. +ARG LEGO_VERSION=5.3.1 +ARG LEGO_SHA256=b3c71b122ee1947eacfe0b809b955647f6377239fe4bfc49f73b1a091ae1252a +RUN set -eu; \ + url="https://github.com/go-acme/lego/releases/download/v${LEGO_VERSION}/lego_v${LEGO_VERSION}_linux_amd64.tar.gz"; \ + curl -fsSL -o /tmp/lego.tar.gz "$url"; \ + echo "${LEGO_SHA256} /tmp/lego.tar.gz" | sha256sum -c -; \ + tar -xzf /tmp/lego.tar.gz -C /usr/local/bin lego; \ + rm -f /tmp/lego.tar.gz; \ + chmod 755 /usr/local/bin/lego; \ + touch -d @0 /usr/local/bin/lego; \ + lego --version + RUN mkdir -p \ /etc/letsencrypt \ /var/www/certbot \ diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index ef16d2ea..e840336f 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -169,9 +169,19 @@ environment: | `PORT` | `443` | HAProxy listen port | | `DOMAINS` | | Multiple domains, one per line | | `ROUTING_MAP` | | Multi-domain routing: `domain=host:port` per line | -| `SET_CAA` | `false` | Enable CAA DNS record | +| `SET_CAA` | `false` | Enable CAA DNS record (dns-01 only; tls-alpn-01 cannot write DNS) | | `TXT_PREFIX` | `_dstack-app-address` | DNS TXT record prefix | | `CERTBOT_STAGING` | `false` | Use Let's Encrypt staging server | +| `CHALLENGE_TYPE` | `dns-01` | `dns-01` (certbot + DNS credentials) or `tls-alpn-01` (lego, no DNS credentials) | +| `DNS_SETUP_MODE` | `wait` | tls-alpn-01 only: `wait`, `print` or `webhook` — see below | +| `DNS_SETUP_TIMEOUT` | `1800` | tls-alpn-01 only: seconds to wait for the records to appear | +| `DNS_SETUP_INTERVAL` | `15` | tls-alpn-01 only: seconds between DNS checks | +| `DNS_WEBHOOK_URL` | | tls-alpn-01 + `DNS_SETUP_MODE=webhook`: endpoint to notify | +| `DNS_WEBHOOK_TOKEN` | | Shared secret; the payload is HMAC-SHA256 signed with it | +| `DOH_RESOLVERS` | Google + Cloudflare | Comma-separated DoH endpoints used to verify records | +| `TLSALPN_PORT` | `9443` | Loopback port lego's ACME responder binds to | +| `TLS_TERMINATE_PORT` | `9444` | Loopback port the TLS frontend moves to in tls-alpn-01 mode | +| `RENEW_DAYS_BEFORE` | lego default | Days of remaining lifetime that trigger renewal | | `MAXCONN` | `4096` | HAProxy max connections | | `TIMEOUT_CONNECT` | `10s` | Backend connect timeout | | `TIMEOUT_CLIENT` | `86400s` | Client-side timeout (24h for long-lived connections) | @@ -266,3 +276,120 @@ Permission is hereby granted, free of charge, to any person obtaining a copy of The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +## Certificates without DNS credentials (tls-alpn-01) + +The default `dns-01` flow needs a DNS API token inside the CVM: the container +creates the CNAME, TXT and CAA records itself. `CHALLENGE_TYPE=tls-alpn-01` +removes that requirement — nothing in the container can touch your DNS zone — +at the cost of you creating three records by hand (or via a webhook). + +```yaml +services: + dstack-ingress: + image: dstacktee/dstack-ingress: + environment: + - CHALLENGE_TYPE=tls-alpn-01 + - DOMAIN=app.example.com + - TARGET_ENDPOINT=http://app:80 + - GATEWAY_DOMAIN=_.dstack-prod5.phala.network + - CERTBOT_EMAIL=you@example.com + # - DNS_SETUP_MODE=wait # default; blocks until the records exist + ports: + - "443:443" + volumes: + - /var/run/dstack.sock:/var/run/dstack.sock + - cert-data:/etc/letsencrypt + - evidences:/evidences +``` + +On first start the container prints the exact records to create, then polls +public DNS until they are visible: + +``` +========================================================================== + DNS records required for app.example.com +========================================================================== + CNAME app.example.com + -> _.dstack-prod5.phala.network + TXT _dstack-app-address.app.example.com + -> b1ea785543bbbb19ce9de33744321360992bf63b:443 + CAA app.example.com + -> 0 issue "letsencrypt.org;validationmethods=tls-alpn-01;accounturi=https://..." +========================================================================== +``` + +`DNS_SETUP_MODE` picks what happens after printing: `wait` (default) blocks +until the records resolve or `DNS_SETUP_TIMEOUT` elapses; `print` continues +immediately; `webhook` POSTs the records to `DNS_WEBHOOK_URL` first and then +waits, so a service of yours can create them automatically. + +### The TXT record names an *instance*, not an app + +Under `dns-01` the TXT record carries the **app ID** and the gateway +load-balances across every instance of the app. tls-alpn-01 cannot work that +way. The challenge is answered by whichever instance holds the ACME order, and +Let's Encrypt validates from several vantage points at once (5 distinct source +IPs within ~2 seconds, measured), while the gateway races connections across the +app's instances. The challenge would land on the wrong replica almost every +time. So this mode publishes the **instance ID** instead, pinning the hostname +to one instance. + +Two consequences: + +- **tls-alpn-01 mode is effectively single-instance.** All traffic for the + hostname goes to the pinned instance; you lose the gateway's failover. +- **The TXT record changes when the CVM instance is replaced.** Redeploying + means updating DNS. `DNS_SETUP_MODE=webhook` exists so this can be automated; + doing it by hand means downtime on every redeploy. + +### Limitations + +- **No wildcards.** RFC 8737 forbids tls-alpn-01 for wildcard identifiers, and + the CA will not offer the challenge. Use `dns-01` for `*.example.com`. +- **The gateway must be reachable on port 443.** The CA connects to port 443 of + whatever the CNAME resolves to; the port is fixed by the protocol. +- **CAA must permit `tls-alpn-01`.** A record left over from a `dns-01` + deployment says `validationmethods=dns-01` and will make the CA refuse. The + container checks this before asking for a certificate, so you get a clear + message instead of a failed validation. +- **Losing the `cert-data` volume changes the account URI.** With `dns-01` the + container just rewrites the CAA record; here it cannot, so a pinned + `accounturi` would start rejecting issuance until you update it by hand. + +### Webhook payload + +`DNS_SETUP_MODE=webhook` POSTs this envelope to `DNS_WEBHOOK_URL`: + +```json +{ + "payload": "{\"version\":1,\"domain\":\"app.example.com\",\"records\":[...]}", + "hmac_sha256": "…", + "attestation": { "quote": "…", "report_data": "…" } +} +``` + +`payload` is a *string* so you sign and hash exactly the bytes you received. +Verify `hmac_sha256` with `DNS_WEBHOOK_TOKEN`, and — since this request asks you +to point a hostname at the instance it names — verify `attestation` too: the +quote's `report_data` is `sha256(payload)`, so it proves which enclave produced +those records. Check the app ID and measurements in it before changing DNS. + +### Why lego and not certbot + +certbot cannot do tls-alpn-01. Its standalone plugin is HTTP-01 only, the +maintainers declined to implement the challenge +([certbot#6724](https://github.com/certbot/certbot/issues/6724)), and the `acme` +library removed it outright in 4.2.0 — the last release carrying +`acme.standalone.TLSALPN01Server` is 4.1.1, where it is already marked +deprecated. This path therefore runs [lego](https://github.com/go-acme/lego), +pinned by checksum in the Dockerfile. The `dns-01` path still uses certbot and +is untouched. + +Because haproxy owns the public port, the proxy peeks at each ClientHello and +forwards only connections advertising the `acme-tls/1` ALPN protocol to lego's +responder on loopback; everything else goes to the normal TLS frontend. Issuance +and renewal therefore never interrupt serving traffic. In this mode haproxy +starts on a self-signed placeholder certificate — it has to be listening before +the first certificate can be issued — and reloads onto the real one as soon as +it arrives. diff --git a/custom-domain/dstack-ingress/scripts/build-combined-pems.sh b/custom-domain/dstack-ingress/scripts/build-combined-pems.sh index 6e10fcb4..8b0a31c6 100644 --- a/custom-domain/dstack-ingress/scripts/build-combined-pems.sh +++ b/custom-domain/dstack-ingress/scripts/build-combined-pems.sh @@ -13,10 +13,11 @@ all_domains=$(get-all-domains.sh) while IFS= read -r domain; do [[ -n "$domain" ]] || continue - le_dir="/etc/letsencrypt/live/$(cert_dir_name "$domain")" + fullchain=$(cert_fullchain_path "$domain") + privkey=$(cert_privkey_path "$domain") combined="${CERT_DIR}/${domain}.pem" - if [ -f "${le_dir}/fullchain.pem" ] && [ -f "${le_dir}/privkey.pem" ]; then - cat "${le_dir}/fullchain.pem" "${le_dir}/privkey.pem" > "$combined" + if [ -f "$fullchain" ] && [ -f "$privkey" ]; then + cat "$fullchain" "$privkey" > "$combined" chmod 600 "$combined" echo "Combined PEM created: ${combined}" else diff --git a/custom-domain/dstack-ingress/scripts/dnsguide.py b/custom-domain/dstack-ingress/scripts/dnsguide.py new file mode 100644 index 00000000..5c5a1747 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/dnsguide.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python3 +"""Guide the operator through the DNS records dstack-ingress needs. + +In tls-alpn-01 mode the container holds no DNS credentials, so it cannot create +the CNAME / TXT / CAA records itself. Instead it computes exactly which records +are required, prints them, optionally pushes them to a webhook, and then waits +until they are observable in public DNS before letting certificate issuance +proceed. + +Waiting matters more than it looks. If the TXT record is missing the gateway has +no route for the hostname, so the CA's connection never reaches this container +and the failure surfaces as an opaque timeout -- while still burning Let's +Encrypt's "failed validations" budget (5 per account per hostname per hour). + +Resolution goes through DoH rather than the container resolver: it sidesteps +local caching and needs no extra package (python3-requests is already present). +This is a convenience check for operator workflow, not a security control -- the +authoritative CAA evaluation is the CA's own, at issuance time. +""" + +import argparse +import json +import os +import sys +import time +from dataclasses import dataclass, asdict +from typing import Dict, List, Optional, Sequence, Tuple + +import requests + +# Two independent resolvers, because one is not enough to trust either answer. +# Observed while testing this: a resolver that was asked for a CAA record before +# the record existed kept serving the negative answer well past the record's +# TTL, while the other resolver already had it. A single resolver would have +# reported "no CAA, issuance unrestricted" for a name that in fact restricted +# issuance -- exactly the check we cannot afford to get wrong. +# +# So the two record classes are read with opposite quorums: +# propagation checks (CNAME, TXT) require *every* resolver to agree, which is +# what "wait until it is really out there" means; +# the CAA permission check takes the *union*, so any resolver seeing a +# restriction is enough to stop us. +DEFAULT_DOH = "https://dns.google/resolve,https://cloudflare-dns.com/dns-query" +DEFAULT_TIMEOUT = 1800 +DEFAULT_INTERVAL = 15 +QUERY_TIMEOUT = 10 + +RR_A = 1 +RR_CNAME = 5 +RR_TXT = 16 +RR_AAAA = 28 +RR_CAA = 257 + +LE_ISSUER = "letsencrypt.org" + + +@dataclass +class Record: + """A DNS record the operator has to create.""" + + type: str + name: str + value: str + note: str = "" + + +class ResolveError(RuntimeError): + """A DoH query failed in a way that is not simply "no such record".""" + + +def doh_query(name: str, rr_type: int, resolver: str) -> List[str]: + """Return the rdata strings for name/type, or [] when there are none.""" + try: + resp = requests.get( + resolver, + params={"name": name, "type": str(rr_type)}, + headers={"accept": "application/dns-json"}, + timeout=QUERY_TIMEOUT, + ) + except requests.RequestException as exc: + raise ResolveError(f"DoH query for {name} failed: {exc}") from exc + + if resp.status_code != 200: + raise ResolveError(f"DoH query for {name} returned HTTP {resp.status_code}") + + try: + payload = resp.json() + except ValueError as exc: + raise ResolveError(f"DoH response for {name} is not JSON: {exc}") from exc + + # Status 3 is NXDOMAIN: a definite "no such name", not an error for us. + status = payload.get("Status") + if status not in (0, 3): + raise ResolveError(f"DoH query for {name} returned DNS status {status}") + + answers = payload.get("Answer") or [] + return [a["data"] for a in answers if a.get("type") == rr_type and "data" in a] + + +def split_resolvers(spec: str) -> List[str]: + return [r.strip() for r in spec.split(",") if r.strip()] + + +def query_per_resolver(name: str, rr_type: int, resolvers: str) -> List[Tuple[str, List[str]]]: + """Query every resolver; raise only if none of them answered.""" + results = [] + errors = [] + for resolver in split_resolvers(resolvers): + try: + results.append((resolver, doh_query(name, rr_type, resolver))) + except ResolveError as exc: + errors.append(str(exc)) + if not results: + raise ResolveError("; ".join(errors) or f"no resolvers configured for {name}") + return results + + +def query_union(name: str, rr_type: int, resolvers: str) -> List[str]: + seen: List[str] = [] + for _resolver, values in query_per_resolver(name, rr_type, resolvers): + for value in values: + if value not in seen: + seen.append(value) + return seen + + +def _unquote_txt(value: str) -> str: + """Normalise TXT rdata, which resolvers hand back quoted and possibly split.""" + value = value.strip() + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + # A long TXT value arrives as several quoted chunks that concatenate. + return value.replace('" "', "") + + +def _fqdn(name: str) -> str: + return name.rstrip(".").lower() + + +def check_txt(record: Record, resolvers: str) -> Tuple[bool, str]: + lagging = [] + observed: List[str] = [] + for resolver, raw in query_per_resolver(record.name, RR_TXT, resolvers): + values = [_unquote_txt(v) for v in raw] + observed.extend(v for v in values if v not in observed) + if record.value not in values: + lagging.append(resolver) + if not lagging: + return True, "ok" + if not observed: + return False, f"no TXT record found (checked {len(lagging)} resolver(s))" + return False, ( + f"TXT present but wrong value: {observed!r} (want {record.value!r})" + if len(lagging) == len(split_resolvers(resolvers)) + else f"not yet propagated to {', '.join(lagging)} (saw {observed!r})" + ) + + +def check_alias(record: Record, resolvers: str) -> Tuple[bool, str]: + """Verify the hostname ends up at the gateway. + + Accepts either a CNAME pointing at the gateway or an address record whose + value matches one of the gateway's, so a flattened / ALIAS record at a zone + apex is not rejected. + """ + target = _fqdn(record.value) + + cnames = [_fqdn(v) for v in query_union(record.name, RR_CNAME, resolvers)] + if target in cnames: + return True, "ok (CNAME)" + + own_addrs = set(query_union(record.name, RR_A, resolvers)) | set( + query_union(record.name, RR_AAAA, resolvers) + ) + if not own_addrs: + return False, "hostname does not resolve yet" + + gw_addrs = set(query_union(target, RR_A, resolvers)) | set( + query_union(target, RR_AAAA, resolvers) + ) + if gw_addrs and own_addrs & gw_addrs: + return True, "ok (address matches gateway)" + if cnames: + return False, f"CNAME points at {cnames!r}, expected {target!r}" + return False, ( + f"resolves to {sorted(own_addrs)!r} which does not match the gateway " + f"{sorted(gw_addrs)!r}" + ) + + +def _parse_caa(rdata: str) -> Optional[Tuple[int, str, str]]: + """Parse CAA rdata in either form a DoH resolver may hand back. + + Google returns presentation format -- `0 issue "letsencrypt.org;..."`. + Cloudflare returns RFC 3597 generic format -- `\\# 47 00 05 69 73 73 75 65 + ...` -- i.e. the length followed by hex octets of the wire encoding. Parsing + only the first shape silently degrades a "CAA forbids this" answer into "no + CAA found", which is the wrong direction to fail in. + """ + rdata = rdata.strip() + + if rdata.startswith("\\#"): + # wire format: flags(1) tag-length(1) tag(tag-length) value(rest) + try: + raw = bytes.fromhex("".join(rdata.split()[2:])) + except ValueError: + return None + if len(raw) < 2: + return None + flags, tag_len = raw[0], raw[1] + tag = raw[2 : 2 + tag_len].decode("ascii", "replace").lower() + value = raw[2 + tag_len :].decode("utf-8", "replace") + return flags, tag, value + + parts = rdata.split(None, 2) + if len(parts) != 3: + return None + flags_s, tag, value = parts + try: + flags = int(flags_s) + except ValueError: + return None + return flags, tag.lower(), value.strip().strip('"') + + +def _caa_permits(value: str, tag: str, want_method: str, account_uri: str) -> Tuple[bool, str]: + """Does one CAA value allow us to issue via want_method for account_uri?""" + fields = [f.strip() for f in value.split(";")] + issuer = _fqdn(fields[0]) if fields else "" + if issuer != LE_ISSUER: + return False, f"{tag} allows {issuer!r}, not {LE_ISSUER}" + + params: Dict[str, str] = {} + for field in fields[1:]: + if "=" in field: + k, v = field.split("=", 1) + params[k.strip().lower()] = v.strip() + + methods = params.get("validationmethods") + if methods is not None: + allowed = {m.strip() for m in methods.split(",")} + if want_method not in allowed: + return False, ( + f"{tag} restricts validationmethods to {sorted(allowed)}, " + f"which excludes {want_method}" + ) + + uri = params.get("accounturi") + if uri is not None and account_uri and uri != account_uri: + return False, f"{tag} pins accounturi={uri}, but this container's account is {account_uri}" + + return True, "ok" + + +def check_caa( + domain: str, want_tag: str, want_method: str, account_uri: str, resolvers: str +) -> Tuple[bool, str]: + """Check CAA for domain, walking up to the closest ancestor that has one. + + An absent CAA record set means every CA may issue (RFC 8659), so "no CAA + anywhere" is a pass, not a failure. + """ + labels = _fqdn(domain).split(".") + for i in range(len(labels) - 1): + candidate = ".".join(labels[i:]) + # Union: one resolver seeing a restriction is enough to honour it. + rdatas = query_union(candidate, RR_CAA, resolvers) + parsed = [p for p in (_parse_caa(r) for r in rdatas) if p is not None] + relevant = [p for p in parsed if p[1] == want_tag] + if not parsed: + continue # keep climbing; this level has no CAA record set + + # This is the closest CAA record set; it is the one that governs. + if not relevant: + return False, ( + f"{candidate} has a CAA record set but no {want_tag} tag, " + f"which forbids issuance" + ) + reasons = [] + for _flags, tag, value in relevant: + ok, reason = _caa_permits(value, tag, want_method, account_uri) + if ok: + return True, f"ok (from {candidate})" + reasons.append(reason) + return False, f"CAA at {candidate} does not permit issuance: {'; '.join(reasons)}" + + return True, "ok (no CAA record set; issuance is unrestricted)" + + +def render(records: Sequence[Record], domain: str) -> str: + width = 74 + lines = [ + "", + "=" * width, + f" DNS records required for {domain}", + "=" * width, + " This container holds no DNS credentials, so you must create these", + " records yourself. Certificate issuance waits until they are visible.", + "", + ] + for rec in records: + lines.append(f" {rec.type:<6} {rec.name}") + lines.append(f" {'':<6} -> {rec.value}") + if rec.note: + lines.append(f" {'':<6} ({rec.note})") + lines.append("") + lines.append("=" * width) + lines.append("") + return "\n".join(lines) + + +def verify_once( + records: Sequence[Record], + domain: str, + caa_tag: str, + caa_method: str, + account_uri: str, + resolvers: str, +) -> List[Tuple[str, bool, str]]: + results = [] + for rec in records: + if rec.type == "TXT": + ok, why = check_txt(rec, resolvers) + results.append((f"TXT {rec.name}", ok, why)) + elif rec.type == "CNAME": + ok, why = check_alias(rec, resolvers) + results.append((f"CNAME {rec.name}", ok, why)) + ok, why = check_caa(domain, caa_tag, caa_method, account_uri, resolvers) + results.append((f"CAA {domain}", ok, why)) + return results + + +def build_records(args: argparse.Namespace) -> List[Record]: + include = {part.strip() for part in args.include.split(",") if part.strip()} + records = [] + if "cname" in include: + records.append( + Record( + type="CNAME", + name=args.domain.lstrip("*."), + value=args.alias_target, + note="points the hostname at the dstack gateway", + ) + ) + if "txt" in include: + records.append( + Record( + type="TXT", + name=args.txt_name, + value=args.txt_value, + note="tells the gateway which instance to route this hostname to", + ) + ) + if "caa" in include and args.caa_value: + note = "optional; restricts issuance for this name to Let's Encrypt" + if args.account_uri: + note = "optional, but pins issuance to this enclave's ACME account" + records.append( + Record( + type="CAA", + name=args.caa_name, + value=f'0 {args.caa_tag} "{args.caa_value}"', + note=note, + ) + ) + return records + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--domain", required=True) + parser.add_argument("--alias-target", required=True, help="gateway domain") + parser.add_argument("--txt-name", required=True) + parser.add_argument("--txt-value", required=True) + parser.add_argument("--caa-name", default="") + parser.add_argument("--caa-tag", default="issue", choices=["issue", "issuewild"]) + parser.add_argument("--caa-value", default="") + parser.add_argument("--account-uri", default="") + parser.add_argument("--challenge", default="tls-alpn-01") + parser.add_argument( + "--mode", + default=os.environ.get("DNS_SETUP_MODE", "wait"), + choices=["wait", "print", "webhook"], + ) + parser.add_argument("--timeout", type=int, default=int(os.environ.get("DNS_SETUP_TIMEOUT", DEFAULT_TIMEOUT))) + parser.add_argument("--interval", type=int, default=int(os.environ.get("DNS_SETUP_INTERVAL", DEFAULT_INTERVAL))) + parser.add_argument( + "--resolver", + default=os.environ.get("DOH_RESOLVERS", os.environ.get("DOH_RESOLVER", DEFAULT_DOH)), + help="comma-separated DoH endpoints", + ) + parser.add_argument("--json", action="store_true", help="also emit the records as JSON") + parser.add_argument( + "--include", + default="cname,txt,caa", + help="comma-separated subset of records to handle (cname,txt,caa)", + ) + args = parser.parse_args() + + if not args.caa_name: + args.caa_name = args.domain.lstrip("*.") + + records = build_records(args) + print(render(records, args.domain), flush=True) + if args.json: + print(json.dumps([asdict(r) for r in records], indent=2), flush=True) + + if args.mode == "webhook": + # Imported lazily so `print`/`wait` do not depend on the hook module. + import dnshook + + try: + dnshook.notify(records, domain=args.domain, challenge=args.challenge) + except Exception as exc: # noqa: BLE001 - a hook failure must not be fatal + print(f"Warning: DNS webhook notification failed: {exc}", file=sys.stderr, flush=True) + print("Continuing; the records above still have to exist.", file=sys.stderr, flush=True) + + if args.mode == "print": + print("DNS_SETUP_MODE=print: not verifying, continuing immediately.", flush=True) + return 0 + + deadline = time.monotonic() + args.timeout + attempt = 0 + while True: + attempt += 1 + try: + results = verify_once( + records, args.domain.lstrip("*."), args.caa_tag, args.challenge, + args.account_uri, args.resolver, + ) + except ResolveError as exc: + print(f"[dns-check #{attempt}] resolver problem: {exc}", flush=True) + results = [] + + if results and all(ok for _, ok, _ in results): + print(f"[dns-check #{attempt}] all records verified:", flush=True) + for label, _, why in results: + print(f" OK {label}: {why}", flush=True) + return 0 + + for label, ok, why in results: + print(f"[dns-check #{attempt}] {'OK ' if ok else 'WAIT'} {label}: {why}", flush=True) + + remaining = deadline - time.monotonic() + if remaining <= 0: + print( + f"Error: DNS records were not visible within {args.timeout}s. " + f"Create the records above, or set DNS_SETUP_MODE=print to skip this check.", + file=sys.stderr, + flush=True, + ) + return 1 + time.sleep(min(args.interval, max(1, int(remaining)))) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/custom-domain/dstack-ingress/scripts/dnshook.py b/custom-domain/dstack-ingress/scripts/dnshook.py new file mode 100644 index 00000000..3dd45d8c --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/dnshook.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Push the required DNS records to an operator-supplied webhook. + +Because the instance ID is baked into the TXT record in tls-alpn-01 mode, that +record changes every time the CVM instance is replaced. Doing that by hand means +downtime on every redeploy, so this hook exists to let the operator automate it. + +That makes the hook security-relevant: whoever receives it is being asked to +point a hostname at whatever instance the caller names. An unauthenticated hook +would hand domain hijacking to anyone who can reach the endpoint. So every +request carries: + + * an HMAC-SHA256 over the exact payload string, keyed by a shared secret; and + * optionally a TDX quote whose report_data commits to that same payload. + +Verify both on the receiving end. The HMAC alone only proves the caller knows +the secret; the quote proves the caller is the enclave you expect, so check the +app_id / measurements in it before touching DNS. + +Payload is sent as a *string* field rather than a nested object on purpose: the +receiver signs and hashes exactly the bytes it was given, with no JSON +re-canonicalisation to disagree about. +""" + +import argparse +import hashlib +import hmac +import json +import os +import subprocess +import sys +import time +from dataclasses import asdict +from typing import Any, Dict, List, Optional, Sequence + +import requests + +DEFAULT_TIMEOUT = 15 +DEFAULT_RETRIES = 3 +DSTACK_SOCKETS = ( + ("/var/run/dstack.sock", "http://localhost/GetQuote?report_data={rd}"), + ("/var/run/tappd.sock", "http://localhost/prpc/Tappd.RawQuote?report_data={rd}"), +) + + +def _canonical(payload: Dict[str, Any]) -> str: + return json.dumps(payload, sort_keys=True, separators=(",", ":")) + + +def _sign(payload_str: str, secret: str) -> str: + return hmac.new(secret.encode(), payload_str.encode(), hashlib.sha256).hexdigest() + + +def get_quote(payload_str: str) -> Optional[Dict[str, Any]]: + """Fetch a TDX quote committing to payload_str, or None if unavailable.""" + report_data = hashlib.sha256(payload_str.encode()).hexdigest() + for sock, url_tpl in DSTACK_SOCKETS: + if not os.path.exists(sock): + continue + try: + out = subprocess.run( + ["curl", "-s", "--unix-socket", sock, url_tpl.format(rd=report_data)], + capture_output=True, + text=True, + timeout=30, + ) + if out.returncode != 0 or not out.stdout.strip(): + print( + f"Warning: quote fetch via {sock} failed (rc={out.returncode})", + file=sys.stderr, + ) + continue + quote = json.loads(out.stdout) + quote["report_data"] = report_data + return quote + except (subprocess.SubprocessError, ValueError) as exc: + print(f"Warning: quote fetch via {sock} failed: {exc}", file=sys.stderr) + return None + + +def build_envelope( + records: Sequence[Any], + domain: str, + challenge: str, + with_quote: bool = True, +) -> Dict[str, Any]: + payload = { + "version": 1, + "domain": domain, + "challenge": challenge, + "app_id": os.environ.get("DSTACK_APP_ID", ""), + "instance_id": os.environ.get("DSTACK_INSTANCE_ID", ""), + "timestamp": int(time.time()), + "records": [asdict(r) if hasattr(r, "__dataclass_fields__") else dict(r) for r in records], + } + payload_str = _canonical(payload) + envelope: Dict[str, Any] = {"payload": payload_str} + + secret = os.environ.get("DNS_WEBHOOK_TOKEN", "") + if secret: + envelope["hmac_sha256"] = _sign(payload_str, secret) + else: + print( + "Warning: DNS_WEBHOOK_TOKEN is unset, so the webhook request is " + "unauthenticated. Anyone who can reach your endpoint could redirect " + "your domain. Set a secret.", + file=sys.stderr, + ) + + if with_quote: + quote = get_quote(payload_str) + if quote is not None: + envelope["attestation"] = quote + return envelope + + +def notify( + records: Sequence[Any], + domain: str, + challenge: str, + url: Optional[str] = None, + retries: int = DEFAULT_RETRIES, +) -> None: + url = url or os.environ.get("DNS_WEBHOOK_URL", "") + if not url: + raise RuntimeError("DNS_SETUP_MODE=webhook but DNS_WEBHOOK_URL is not set") + + envelope = build_envelope(records, domain, challenge) + timeout = int(os.environ.get("DNS_WEBHOOK_TIMEOUT", DEFAULT_TIMEOUT)) + last_error: Optional[str] = None + + for attempt in range(1, retries + 1): + try: + resp = requests.post( + url, + json=envelope, + timeout=timeout, + headers={"user-agent": "dstack-ingress/dnshook"}, + ) + if 200 <= resp.status_code < 300: + print(f"DNS webhook accepted (HTTP {resp.status_code})", flush=True) + return + body = resp.text[:512] + last_error = f"HTTP {resp.status_code}: {body}" + except requests.RequestException as exc: + last_error = str(exc) + + print( + f"DNS webhook attempt {attempt}/{retries} failed: {last_error}", + file=sys.stderr, + flush=True, + ) + if attempt < retries: + time.sleep(2**attempt) + + raise RuntimeError(f"DNS webhook failed after {retries} attempts: {last_error}") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", default="", help="override DNS_WEBHOOK_URL") + parser.add_argument("--domain", required=True) + parser.add_argument("--challenge", default="tls-alpn-01") + parser.add_argument( + "--records", + required=True, + help="JSON array of {type,name,value} objects", + ) + parser.add_argument( + "--dry-run", action="store_true", help="print the envelope instead of sending" + ) + args = parser.parse_args() + + records: List[Dict[str, str]] = json.loads(args.records) + if args.dry_run: + print(json.dumps(build_envelope(records, args.domain, args.challenge), indent=2)) + return 0 + notify(records, args.domain, args.challenge, url=args.url or None) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index c1c88e34..55f4cc7d 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -13,10 +13,40 @@ TIMEOUT_SERVER=${TIMEOUT_SERVER:-86400s} EVIDENCE_SERVER=${EVIDENCE_SERVER:-true} EVIDENCE_PORT=${EVIDENCE_PORT:-80} ALPN=${ALPN:-} +CHALLENGE_TYPE=${CHALLENGE_TYPE:-dns-01} +# Loopback address/port lego's tls-alpn-01 responder binds to, and the loopback +# port the real TLS frontend moves to so the public port can be used for +# ALPN-based routing. Neither is reachable from outside the container. +TLSALPN_ADDRESS=${TLSALPN_ADDRESS:-127.0.0.1} +TLSALPN_PORT=${TLSALPN_PORT:-9443} +TLS_TERMINATE_PORT=${TLS_TERMINATE_PORT:-9444} +DNS_SETUP_MODE=${DNS_SETUP_MODE:-wait} + +case "$CHALLENGE_TYPE" in + dns-01|tls-alpn-01) ;; + *) + echo "Error: invalid CHALLENGE_TYPE '$CHALLENGE_TYPE' (expected dns-01 or tls-alpn-01)" >&2 + exit 1 + ;; +esac + +case "$DNS_SETUP_MODE" in + wait|print|webhook) ;; + *) + echo "Error: invalid DNS_SETUP_MODE '$DNS_SETUP_MODE' (expected wait, print or webhook)" >&2 + exit 1 + ;; +esac if ! PORT=$(sanitize_port "$PORT"); then exit 1 fi +if ! TLSALPN_PORT=$(sanitize_port "$TLSALPN_PORT"); then + exit 1 +fi +if ! TLS_TERMINATE_PORT=$(sanitize_port "$TLS_TERMINATE_PORT"); then + exit 1 +fi if ! DOMAIN=$(sanitize_domain "$DOMAIN"); then exit 1 fi @@ -45,6 +75,10 @@ if ! ALPN=$(sanitize_alpn "$ALPN"); then exit 1 fi +# renew-certificate.sh, renewal-daemon.sh and certman.py run as child processes +# and read these from the environment. +export CHALLENGE_TYPE TLSALPN_ADDRESS TLSALPN_PORT DNS_SETUP_MODE PORT + # Warn about deprecated L7 env vars for var in CLIENT_MAX_BODY_SIZE PROXY_READ_TIMEOUT PROXY_SEND_TIMEOUT PROXY_CONNECT_TIMEOUT PROXY_BUFFER_SIZE PROXY_BUFFERS PROXY_BUSY_BUFFERS_SIZE; do if [ -n "${!var}" ]; then @@ -79,13 +113,20 @@ setup_py_env() { source /opt/app-venv/bin/activate if [ ! -f /.venv_bootstrapped ]; then - echo "Bootstrapping certbot dependencies" pip install --upgrade pip - pip install certbot requests boto3 botocore + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then + # lego handles ACME here, so certbot and the cloud SDKs are dead + # weight: skip them and keep startup and attack surface smaller. + echo "Bootstrapping python dependencies (tls-alpn-01: no certbot)" + pip install requests + else + echo "Bootstrapping certbot dependencies" + pip install certbot requests boto3 botocore + fi touch /.venv_bootstrapped fi - ln -sf /opt/app-venv/bin/certbot /usr/local/bin/certbot + [ -x /opt/app-venv/bin/certbot ] && ln -sf /opt/app-venv/bin/certbot /usr/local/bin/certbot echo 'source /opt/app-venv/bin/activate' > /etc/profile.d/app-venv.sh } @@ -94,6 +135,17 @@ setup_certbot_env() { # shellcheck disable=SC1091 source /opt/app-venv/bin/activate + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then + # No DNS provider and no certbot plugin: lego handles this path and is + # baked into the image. + if [ ! -x "${LEGO_BIN:-/usr/local/bin/lego}" ]; then + echo "Error: lego binary not found at ${LEGO_BIN:-/usr/local/bin/lego}" >&2 + exit 1 + fi + echo "Using lego for tls-alpn-01: $(${LEGO_BIN:-/usr/local/bin/lego} --version)" + return + fi + if [ "${DNS_PROVIDER}" = "route53" ]; then mkdir -p /root/.aws @@ -128,6 +180,17 @@ setup_py_env # Emit common haproxy global/defaults/frontend preamble. # Both single-domain and multi-domain modes share this identical config. emit_haproxy_preamble() { + # In tls-alpn-01 mode the ACME responder has to complete the TLS handshake + # itself, but haproxy owns the public port. So the public port becomes a + # plain TCP frontend that peeks at the ClientHello and forwards only + # acme-tls/1 connections to lego; everything else goes to the real TLS + # frontend, which moves to loopback. PROXY protocol carries the client + # address across that extra hop so the backend still sees the real peer. + local tls_bind=":${PORT}" + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then + tls_bind="127.0.0.1:${TLS_TERMINATE_PORT} accept-proxy" + fi + # "crt " loads all PEM files from the directory. # ALPN is appended conditionally via ${ALPN:+ alpn ${ALPN}}. cat </etc/haproxy/haproxy.cfg @@ -148,8 +211,31 @@ defaults timeout client ${TIMEOUT_CLIENT} timeout server ${TIMEOUT_SERVER} +EOF + + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then + cat <>/etc/haproxy/haproxy.cfg +frontend tls_peek + bind :${PORT} + tcp-request inspect-delay 5s + tcp-request content accept if { req.ssl_hello_type 1 } + # Only the CA sends this ALPN protocol. A client that sends it anyway just + # reaches lego's responder, which refuses anything but acme-tls/1. + use_backend be_acme if { req.ssl_alpn -i acme-tls/1 } + default_backend be_tls_terminate + +backend be_acme + server lego ${TLSALPN_ADDRESS}:${TLSALPN_PORT} + +backend be_tls_terminate + server local 127.0.0.1:${TLS_TERMINATE_PORT} send-proxy-v2 + +EOF + fi + + cat <>/etc/haproxy/haproxy.cfg frontend tls_in - bind :${PORT} ssl crt /etc/haproxy/certs/${ALPN:+ alpn ${ALPN}} + bind ${tls_bind} ssl crt /etc/haproxy/certs/${ALPN:+ alpn ${ALPN}} EOF if [ "$EVIDENCE_SERVER" = "true" ]; then @@ -290,26 +376,67 @@ set_alias_record() { echo "Alias record set for $domain" } -set_txt_record() { - local domain="$1" - local APP_ID - +# Query the guest agent once for this app's identity. +load_dstack_identity() { + local info if [[ -S /var/run/dstack.sock ]]; then - DSTACK_APP_ID=$(curl -s --unix-socket /var/run/dstack.sock http://localhost/Info | jq -j .app_id) - export DSTACK_APP_ID + info=$(curl -s --unix-socket /var/run/dstack.sock http://localhost/Info) else - DSTACK_APP_ID=$(curl -s --unix-socket /var/run/tappd.sock http://localhost/prpc/Tappd.Info | jq -j .app_id) - export DSTACK_APP_ID + info=$(curl -s --unix-socket /var/run/tappd.sock http://localhost/prpc/Tappd.Info) fi - APP_ID=${APP_ID:-"$DSTACK_APP_ID"} - local txt_domain + DSTACK_APP_ID=$(echo "$info" | jq -j .app_id) + DSTACK_INSTANCE_ID=$(echo "$info" | jq -j .instance_id) + export DSTACK_APP_ID DSTACK_INSTANCE_ID + + if [ -z "$DSTACK_APP_ID" ] || [ "$DSTACK_APP_ID" = "null" ]; then + echo "Error: could not read app_id from the dstack guest agent" >&2 + exit 1 + fi + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ] && + { [ -z "$DSTACK_INSTANCE_ID" ] || [ "$DSTACK_INSTANCE_ID" = "null" ]; }; then + echo "Error: could not read instance_id from the dstack guest agent, which" >&2 + echo "tls-alpn-01 needs to pin gateway routing to this instance" >&2 + exit 1 + fi +} + +txt_record_name() { + local domain="$1" if [[ "$domain" == \*.* ]]; then # Wildcard domain: *.myapp.com → _dstack-app-address-wildcard.myapp.com - txt_domain="${TXT_PREFIX}-wildcard.${domain#\*.}" + echo "${TXT_PREFIX}-wildcard.${domain#\*.}" + else + echo "${TXT_PREFIX}.${domain}" + fi +} + +# What the gateway should route this hostname to. +# +# dns-01 uses the app id, so the gateway load-balances across every instance of +# the app. tls-alpn-01 cannot: the challenge is answered by whichever instance +# certbot runs on, and routing by app id makes the gateway race connections +# across the app's instances (select_top_n_hosts -> connect_multiple_hosts) +# while the CA validates from several vantage points at once. The challenge +# would land on the wrong replica. Pinning to the instance id makes validation +# deterministic -- at the cost of sending all traffic to this one instance, so +# tls-alpn-01 mode is effectively single-instance. +txt_record_value() { + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then + echo "${DSTACK_INSTANCE_ID}:${PORT}" else - txt_domain="${TXT_PREFIX}.${domain}" + echo "${APP_ID:-$DSTACK_APP_ID}:${PORT}" fi +} + +caa_tag_for() { + if [[ "$1" == \*.* ]]; then echo "issuewild"; else echo "issue"; fi +} + +set_txt_record() { + local domain="$1" + local txt_domain + txt_domain=$(txt_record_name "$domain") if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then echo "[challenge-delegation] Set this in your production zone yourself (static, one-time):" @@ -319,7 +446,7 @@ set_txt_record() { dnsman.py set_txt \ --domain "$txt_domain" \ - --content "$APP_ID:$PORT" + --content "$(txt_record_value)" if [ $? -ne 0 ]; then echo "Error: Failed to set TXT record for $domain" @@ -421,7 +548,7 @@ set_caa_record() { dnsman.py set_caa \ --domain "$caa_domain" \ --caa-tag "$caa_tag" \ - --caa-value "letsencrypt.org;validationmethods=dns-01;accounturi=$account_uri" + --caa-value "letsencrypt.org;validationmethods=${CHALLENGE_TYPE};accounturi=$account_uri" if [ $? -ne 0 ]; then echo "Warning: Failed to set CAA record for $domain" @@ -430,9 +557,8 @@ set_caa_record() { fi } -process_domain() { +process_domain_dns01() { local domain="$1" - echo "Processing domain: $domain" set_alias_record "$domain" set_txt_record "$domain" @@ -441,6 +567,66 @@ process_domain() { renew-certificate.sh "$domain" } +process_domain_tlsalpn() { + local domain="$1" + + if [[ "$domain" == \*.* ]]; then + echo "Error: cannot issue a wildcard certificate for $domain with tls-alpn-01." >&2 + echo "RFC 8737 forbids it; use CHALLENGE_TYPE=dns-01 for wildcards." >&2 + exit 1 + fi + + # Register the ACME account first so the CAA record we print can already + # pin accounturi. Without this the operator would have to add CAA in a + # second pass, after the account exists. + if ! legoman.py register --email "$CERTBOT_EMAIL"; then + echo "Error: failed to register the ACME account" >&2 + exit 1 + fi + + local account_uri caa_value + account_uri=$(legoman.py account-uri --email "$CERTBOT_EMAIL" 2>/dev/null || true) + if [ -n "$account_uri" ]; then + caa_value="letsencrypt.org;validationmethods=tls-alpn-01;accounturi=$account_uri" + else + echo "Warning: could not read the ACME account URI; the CAA record will not pin it" >&2 + caa_value="letsencrypt.org;validationmethods=tls-alpn-01" + fi + + # Wait for the records routing depends on. An existing CAA record is also + # checked for compatibility -- not because CAA is required (an absent CAA + # record set permits every CA), but because an incompatible one makes the CA + # refuse and burns Let's Encrypt's failed-validation budget: 5 per account + # per hostname per hour. + if ! dnsguide.py \ + --domain "$domain" \ + --alias-target "$GATEWAY_DOMAIN" \ + --txt-name "$(txt_record_name "$domain")" \ + --txt-value "$(txt_record_value)" \ + --caa-name "${domain#\*.}" \ + --caa-tag "$(caa_tag_for "$domain")" \ + --caa-value "$caa_value" \ + --account-uri "$account_uri" \ + --challenge tls-alpn-01 \ + --mode "$DNS_SETUP_MODE"; then + echo "Error: required DNS records for $domain are not in place" >&2 + exit 1 + fi + + renew-certificate.sh "$domain" +} + +process_domain() { + local domain="$1" + echo "Processing domain: $domain (challenge: $CHALLENGE_TYPE)" + + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then + process_domain_tlsalpn "$domain" + else + process_domain_dns01 "$domain" + fi +} + bootstrap() { echo "Bootstrap: Setting up domains" @@ -467,22 +653,68 @@ bootstrap() { touch /.bootstrapped } +# haproxy refuses to start when the crt directory has no PEM in it. In +# tls-alpn-01 mode the proxy has to be listening *before* the first certificate +# exists, because it is what forwards the ACME challenge to certbot, so seed a +# self-signed placeholder that the real certificate overwrites later. +ensure_placeholder_certs() { + local all_domains domain pem + all_domains=$(get-all-domains.sh) + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + pem="/etc/haproxy/certs/${domain}.pem" + [ -f "$pem" ] && continue + echo "Generating placeholder certificate for ${domain}" + openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -keyout /tmp/placeholder.key -out /tmp/placeholder.crt \ + -subj "/CN=${domain#\*.}" >/dev/null 2>&1 + cat /tmp/placeholder.crt /tmp/placeholder.key >"$pem" + chmod 600 "$pem" + rm -f /tmp/placeholder.key /tmp/placeholder.crt + done <<<"$all_domains" +} + +# Obtain certificates once the proxy is already serving, then swap them in. +bootstrap_and_reload() { + # Nested subshell so an `exit 1` from a process_domain helper is reported as + # a failed condition here rather than silently killing this background job. + if ( bootstrap ); then + build-combined-pems.sh || echo "Combined PEM build failed" >&2 + if [ -f /var/run/haproxy/haproxy.pid ]; then + kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)" && + echo "Certificates installed and HAProxy reloaded" + fi + else + echo "Bootstrap failed; the proxy keeps serving the placeholder certificate." >&2 + echo "Fix the DNS records above; the renewal daemon retries every 12 hours." >&2 + fi +} + # Credentials are now handled by certman.py setup # Setup certbot environment (venv is already created in Dockerfile) setup_certbot_env - -# Check if it's the first time the container is started -if [ ! -f "/.bootstrapped" ]; then - bootstrap +load_dstack_identity + +if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then + # Ordering is load-bearing here. The tls-alpn-01 challenge arrives on the + # public TLS port, so haproxy must already be routing acme-tls/1 to certbot + # before the first certificate can be issued. Start the proxy on a + # placeholder certificate and let issuance converge behind it. + ensure_placeholder_certs + build-combined-pems.sh || true else - echo "Certificate for $DOMAIN already exists" - generate-evidences.sh + # dns-01 needs no inbound traffic, so keep the original order: get the + # certificate first, start serving second. + if [ ! -f "/.bootstrapped" ]; then + bootstrap + else + echo "Certificate for $DOMAIN already exists" + generate-evidences.sh + fi + build-combined-pems.sh fi -# Build combined PEM files for haproxy -build-combined-pems.sh - # Generate haproxy config if [ -n "$ROUTING_MAP" ]; then setup_haproxy_cfg_multi @@ -496,6 +728,15 @@ if [ "$EVIDENCE_SERVER" = "true" ]; then echo "Evidence server started on port ${EVIDENCE_PORT} (mini_httpd)" fi +if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then + if [ ! -f "/.bootstrapped" ]; then + bootstrap_and_reload & + else + echo "Certificates already bootstrapped" + generate-evidences.sh || echo "Evidence generation failed" >&2 + fi +fi + renewal-daemon.sh & exec "$@" diff --git a/custom-domain/dstack-ingress/scripts/functions.sh b/custom-domain/dstack-ingress/scripts/functions.sh index 2ff37e43..c0e5df1e 100644 --- a/custom-domain/dstack-ingress/scripts/functions.sh +++ b/custom-domain/dstack-ingress/scripts/functions.sh @@ -187,3 +187,65 @@ get_letsencrypt_account_file() { echo "${account_files[0]}" } + +# --- ACME client layout ------------------------------------------------------ +# +# The two challenge types use different clients, and they lay their state out +# differently: +# +# certbot (dns-01) /etc/letsencrypt/live//{fullchain,privkey}.pem +# /etc/letsencrypt/accounts//directory/*/regr.json +# lego (tls-alpn-01) $LEGO_PATH/certificates/.{crt,key} +# $LEGO_PATH/accounts///account.json +# +# Everything downstream (combined PEMs, evidence) goes through these helpers so +# it does not have to care which client produced the files. + +lego_path() { + echo "${LEGO_PATH:-/etc/letsencrypt/lego}" +} + +using_lego() { + [[ "${CHALLENGE_TYPE:-dns-01}" == "tls-alpn-01" ]] +} + +# lego writes the full chain into the .crt file. +cert_fullchain_path() { + local domain="$1" + if using_lego; then + echo "$(lego_path)/certificates/${domain}.crt" + else + echo "/etc/letsencrypt/live/$(cert_dir_name "$domain")/fullchain.pem" + fi +} + +cert_privkey_path() { + local domain="$1" + if using_lego; then + echo "$(lego_path)/certificates/${domain}.key" + else + echo "/etc/letsencrypt/live/$(cert_dir_name "$domain")/privkey.pem" + fi +} + +get_lego_account_file() { + local pattern account_files + pattern="$(lego_path)/accounts/*/*/account.json" + account_files=( $pattern ) + + if [[ ! -f "${account_files[0]}" ]]; then + echo "Error: lego account file not found at $pattern" >&2 + return 1 + fi + + echo "${account_files[0]}" +} + +# Account registration document, whichever client produced it. +acme_account_file() { + if using_lego; then + get_lego_account_file + else + get_letsencrypt_account_file + fi +} diff --git a/custom-domain/dstack-ingress/scripts/generate-evidences.sh b/custom-domain/dstack-ingress/scripts/generate-evidences.sh index afc8bfa9..40d2e0b3 100644 --- a/custom-domain/dstack-ingress/scripts/generate-evidences.sh +++ b/custom-domain/dstack-ingress/scripts/generate-evidences.sh @@ -4,14 +4,21 @@ set -e source "/scripts/functions.sh" -if ! ACME_ACCOUNT_FILE=$(get_letsencrypt_account_file); then - echo "Error: Cannot generate evidences without Let's Encrypt account file" +if ! ACME_ACCOUNT_FILE=$(acme_account_file); then + echo "Error: Cannot generate evidences without the ACME account file" exit 1 fi mkdir -p /evidences cd /evidences || exit cp "${ACME_ACCOUNT_FILE}" acme-account.json +# These files exist to be published, so force them world-readable. lego writes +# its account document and certificates 0600 and cp keeps that mode, which makes +# the evidence server answer 403. (certbot's regr.json/fullchain.pem are 0644, +# so this only ever bit the lego path.) Neither file carries a private key: the +# account document holds the account URL and status, and a certificate chain is +# public by construction. +chmod 644 acme-account.json # Get all domains and copy their certificates all_domains=$(get-all-domains.sh) @@ -23,9 +30,10 @@ fi # Copy all certificate files while IFS= read -r domain; do [[ -n "$domain" ]] || continue - cert_file="/etc/letsencrypt/live/$(cert_dir_name "$domain")/fullchain.pem" + cert_file=$(cert_fullchain_path "$domain") if [ -f "$cert_file" ]; then cp "$cert_file" "cert-${domain}.pem" + chmod 644 "cert-${domain}.pem" else echo "Warning: Certificate not found for domain: $domain" fi diff --git a/custom-domain/dstack-ingress/scripts/legoman.py b/custom-domain/dstack-ingress/scripts/legoman.py new file mode 100644 index 00000000..92b6f562 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/legoman.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Certificate management for the tls-alpn-01 path, backed by lego. + +certbot is not usable here. Its standalone plugin is HTTP-01 only, the +maintainers declined to add tls-alpn-01 (certbot/certbot#6724), and the acme +library dropped the challenge entirely in 4.2.0 -- the last release carrying +`acme.standalone.TLSALPN01Server` is 4.1.1, where it is already marked +deprecated. Building on that would mean pinning both `certbot` and `acme` to a +deleted, unmaintained API inside a component whose whole job is TLS. + +lego implements tls-alpn-01 as a first-class challenge and ships as a single +static binary. It binds a loopback address (`--tls.address`); haproxy forwards +only `acme-tls/1` ClientHellos there, so issuance never disturbs serving +traffic. + +Targets the lego 5.x CLI, which differs from 4.x in ways that matter here: +`renew` folded into `run --renew-days`, `--tls.port` became `--tls.address`, +flags moved from global to per-command, and the account document renamed +`registration.uri` to `registration.accountURL`. 5.x also added +`accounts register`, which lets us create the account -- and therefore know its +URI -- before the first certificate exists, so the CAA record can be printed +with `accounturi` up front. + +The dns-01 path still runs certbot via certman.py; nothing here touches it. +""" + +import argparse +import json +import os +import subprocess +import sys +from typing import List, Optional, Tuple +from urllib.parse import urlparse + +LEGO_BIN = os.environ.get("LEGO_BIN", "/usr/local/bin/lego") +LEGO_PATH = os.environ.get("LEGO_PATH", "/etc/letsencrypt/lego") +ACME_PROD = "https://acme-v02.api.letsencrypt.org/directory" +ACME_STAGING = "https://acme-staging-v02.api.letsencrypt.org/directory" + +# Exit codes, matching the protocol renew-certificate.sh expects. +EXIT_CHANGED = 0 +EXIT_ERROR = 1 +EXIT_UNCHANGED = 2 + +RUN_TIMEOUT = 600 +REGISTER_TIMEOUT = 120 + + +def acme_server() -> str: + if os.environ.get("CERTBOT_STAGING", "false") == "true": + return ACME_STAGING + return ACME_PROD + + +def server_dir() -> str: + """lego namespaces account storage by the CA host (plus port, if any).""" + parsed = urlparse(acme_server()) + host = parsed.hostname or "acme" + return f"{host}_{parsed.port}" if parsed.port else host + + +def account_file(email: str) -> Optional[str]: + path = os.path.join(LEGO_PATH, "accounts", server_dir(), email, "account.json") + return path if os.path.isfile(path) else None + + +def account_uri(email: str) -> Optional[str]: + path = account_file(email) + if not path: + return None + try: + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError) as exc: + print(f"Warning: cannot read lego account file {path}: {exc}", file=sys.stderr) + return None + registration = data.get("registration") or {} + # lego 5.x renamed this from "uri"; accept both so a directory written by an + # older binary still resolves. + return registration.get("accountURL") or registration.get("uri") + + +def cert_paths(domain: str) -> Tuple[str, str]: + base = os.path.join(LEGO_PATH, "certificates", domain) + return f"{base}.crt", f"{base}.key" + + +def certificate_exists(domain: str) -> bool: + crt, key = cert_paths(domain) + return os.path.isfile(crt) and os.path.isfile(key) + + +def _common_flags(email: str) -> List[str]: + return [ + "--path", + LEGO_PATH, + "--server", + acme_server(), + "--email", + email, + "--accept-tos", + ] + + +def _run(cmd: List[str], timeout: int = RUN_TIMEOUT) -> Tuple[int, str]: + masked = [ + arg if not (i > 0 and cmd[i - 1] in ("--email", "-m")) else "" + for i, arg in enumerate(cmd) + ] + print(f"Executing: {' '.join(masked)}", flush=True) + try: + result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + print(f"lego timed out after {timeout}s", file=sys.stderr) + return 124, "" + output = (result.stdout or "") + (result.stderr or "") + for line in output.strip().splitlines(): + print(f" lego| {line}", flush=True) + return result.returncode, output + + +def register(email: str) -> int: + """Create the ACME account without issuing anything.""" + if account_file(email): + print("ACME account already exists") + return EXIT_UNCHANGED + + print("Registering ACME account") + code, _ = _run( + [LEGO_BIN, "accounts", "register"] + _common_flags(email), + timeout=REGISTER_TIMEOUT, + ) + if code == 0 and account_file(email): + print("✓ ACME account registered") + return EXIT_CHANGED + print(f"✗ ACME account registration failed (exit code {code})", file=sys.stderr) + return EXIT_ERROR + + +def _challenge_flags() -> List[str]: + address = os.environ.get("TLSALPN_ADDRESS", "127.0.0.1") + port = os.environ.get("TLSALPN_PORT", "9443") + return ["--tls", "--tls.address", f"{address}:{port}"] + + +def run_cert(domain: str, email: str) -> int: + """`lego run` both obtains and renews, deciding by remaining lifetime.""" + if domain.startswith("*."): + print( + f"Error: cannot issue a wildcard certificate for {domain} with tls-alpn-01. " + f"RFC 8737 forbids it; use CHALLENGE_TYPE=dns-01 for wildcards.", + file=sys.stderr, + ) + return EXIT_ERROR + + existed = certificate_exists(domain) + cmd = ( + [LEGO_BIN, "run"] + + _common_flags(email) + + ["--domains", domain] + + _challenge_flags() + ) + renew_days = os.environ.get("RENEW_DAYS_BEFORE", "") + if renew_days: + cmd += ["--renew-days", renew_days] + + print(f"{'Renewing' if existed else 'Obtaining'} certificate for {domain} via tls-alpn-01") + code, output = _run(cmd) + if code != 0: + print(f"✗ lego failed for {domain} (exit code {code})", file=sys.stderr) + return EXIT_ERROR + if not certificate_exists(domain): + print(f"✗ lego reported success but no certificate for {domain}", file=sys.stderr) + return EXIT_ERROR + + # lego exits 0 whether or not it actually replaced anything, so the log line + # is the only signal that a renewal was skipped. + lowered = output.lower() + if existed and ("no renewal" in lowered or "not yet due for renewal" in lowered): + print("No certificates need renewal") + return EXIT_UNCHANGED + + print(f"✓ Certificate ready for {domain}") + return EXIT_CHANGED + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "action", + choices=["obtain", "renew", "auto", "register", "account-uri", "cert-path"], + ) + parser.add_argument("--domain", help="Domain name") + parser.add_argument("--email", help="Email for ACME registration") + args = parser.parse_args() + + email = args.email or os.environ.get("CERTBOT_EMAIL", "") + + if args.action == "account-uri": + if not email: + print("Error: --email or CERTBOT_EMAIL is required", file=sys.stderr) + return EXIT_ERROR + uri = account_uri(email) + if not uri: + return EXIT_ERROR + print(uri) + return EXIT_CHANGED + + if args.action == "cert-path": + if not args.domain: + print("Error: --domain is required", file=sys.stderr) + return EXIT_ERROR + crt, key = cert_paths(args.domain) + print(f"{crt} {key}") + return EXIT_CHANGED + + if not email: + print("Error: --email or CERTBOT_EMAIL is required", file=sys.stderr) + return EXIT_ERROR + if not os.path.isfile(LEGO_BIN): + print(f"Error: lego binary not found at {LEGO_BIN}", file=sys.stderr) + return EXIT_ERROR + + os.makedirs(LEGO_PATH, exist_ok=True) + + if args.action == "register": + code = register(email) + # "already registered" is success for callers that just need it to exist. + return EXIT_CHANGED if code == EXIT_UNCHANGED else code + + if not args.domain: + print("Error: --domain is required", file=sys.stderr) + return EXIT_ERROR + + # obtain / renew / auto all map onto `lego run`, which decides for itself. + return run_cert(args.domain, email) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/custom-domain/dstack-ingress/scripts/renew-certificate.sh b/custom-domain/dstack-ingress/scripts/renew-certificate.sh index 8c822cbd..8ae97d4d 100755 --- a/custom-domain/dstack-ingress/scripts/renew-certificate.sh +++ b/custom-domain/dstack-ingress/scripts/renew-certificate.sh @@ -3,9 +3,15 @@ source /opt/app-venv/bin/activate DOMAIN=$1 -# Use the unified certbot manager SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" -python3 "$SCRIPT_DIR/certman.py" auto --domain "$DOMAIN" --email "$CERTBOT_EMAIL" + +# dns-01 runs certbot with a DNS provider plugin. tls-alpn-01 runs lego: certbot +# never implemented the challenge and the acme library removed it in 4.2.0. +if [ "${CHALLENGE_TYPE:-dns-01}" = "tls-alpn-01" ]; then + python3 "$SCRIPT_DIR/legoman.py" auto --domain "$DOMAIN" --email "$CERTBOT_EMAIL" +else + python3 "$SCRIPT_DIR/certman.py" auto --domain "$DOMAIN" --email "$CERTBOT_EMAIL" +fi CERT_STATUS=$? if [ $CERT_STATUS -eq 1 ]; then diff --git a/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py b/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py new file mode 100644 index 00000000..0ba4db9e --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Unit tests for dnsguide's record parsing and CAA evaluation. + +Run: python3 scripts/tests/test_dnsguide.py +""" + +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) + +import dnsguide # noqa: E402 + +# Same record, as each resolver actually returns it. Google answers in +# presentation format; Cloudflare answers in RFC 3597 generic format. +CAA_PRESENTATION = '0 issue "letsencrypt.org;validationmethods=dns-01"' +CAA_GENERIC = ( + "\\# 47 00 05 69 73 73 75 65 6c 65 74 73 65 6e 63 72 79 70 74 2e 6f 72 67 3b " + "76 61 6c 69 64 61 74 69 6f 6e 6d 65 74 68 6f 64 73 3d 64 6e 73 2d 30 31" +) + + +class TestCaaParsing(unittest.TestCase): + def test_presentation_format(self): + self.assertEqual( + dnsguide._parse_caa(CAA_PRESENTATION), + (0, "issue", "letsencrypt.org;validationmethods=dns-01"), + ) + + def test_generic_format_matches_presentation(self): + """Both resolvers must yield the same tuple, or the check is resolver-dependent.""" + self.assertEqual( + dnsguide._parse_caa(CAA_GENERIC), dnsguide._parse_caa(CAA_PRESENTATION) + ) + + def test_issuewild_tag(self): + parsed = dnsguide._parse_caa('0 issuewild "letsencrypt.org"') + self.assertEqual(parsed, (0, "issuewild", "letsencrypt.org")) + + def test_garbage_is_rejected(self): + for junk in ("", "nonsense", "\\# zz", "\\# 2"): + self.assertIsNone(dnsguide._parse_caa(junk), junk) + + +class TestCaaPermits(unittest.TestCase): + ACCOUNT = "https://acme-staging-v02.api.letsencrypt.org/acme/acct/1" + + def test_method_restriction_blocks_other_method(self): + ok, why = dnsguide._caa_permits( + "letsencrypt.org;validationmethods=dns-01", "issue", "tls-alpn-01", "" + ) + self.assertFalse(ok) + self.assertIn("tls-alpn-01", why) + + def test_method_restriction_allows_listed_method(self): + ok, _ = dnsguide._caa_permits( + "letsencrypt.org;validationmethods=dns-01,tls-alpn-01", "issue", "tls-alpn-01", "" + ) + self.assertTrue(ok) + + def test_no_parameters_allows_any_method(self): + ok, _ = dnsguide._caa_permits("letsencrypt.org", "issue", "tls-alpn-01", "") + self.assertTrue(ok) + + def test_other_issuer_blocks(self): + ok, why = dnsguide._caa_permits("digicert.com", "issue", "tls-alpn-01", "") + self.assertFalse(ok) + self.assertIn("digicert.com", why) + + def test_account_mismatch_blocks(self): + ok, why = dnsguide._caa_permits( + f"letsencrypt.org;accounturi={self.ACCOUNT}", "issue", "tls-alpn-01", + "https://acme-staging-v02.api.letsencrypt.org/acme/acct/999", + ) + self.assertFalse(ok) + self.assertIn("accounturi", why) + + def test_account_unknown_skips_the_comparison(self): + """Before our account exists we cannot compare it, so do not block on it.""" + ok, _ = dnsguide._caa_permits( + f"letsencrypt.org;accounturi={self.ACCOUNT}", "issue", "tls-alpn-01", "" + ) + self.assertTrue(ok) + + +class TestTxtNormalisation(unittest.TestCase): + def test_strips_quotes(self): + self.assertEqual(dnsguide._unquote_txt('"abc:443"'), "abc:443") + + def test_joins_split_chunks(self): + self.assertEqual(dnsguide._unquote_txt('"aaa" "bbb"'), "aaabbb") + + def test_passes_through_unquoted(self): + self.assertEqual(dnsguide._unquote_txt("abc:443"), "abc:443") + + +class TestRecordBuilding(unittest.TestCase): + def _args(self, **over): + import argparse + + base = dict( + domain="app.example.com", + alias_target="gw.example.com", + txt_name="_dstack-app-address.app.example.com", + txt_value="deadbeef:443", + caa_name="app.example.com", + caa_tag="issue", + caa_value="letsencrypt.org;validationmethods=tls-alpn-01", + account_uri="", + include="cname,txt,caa", + ) + base.update(over) + return argparse.Namespace(**base) + + def test_all_records(self): + types = [r.type for r in dnsguide.build_records(self._args())] + self.assertEqual(types, ["CNAME", "TXT", "CAA"]) + + def test_include_filters(self): + types = [r.type for r in dnsguide.build_records(self._args(include="txt"))] + self.assertEqual(types, ["TXT"]) + + def test_caa_omitted_without_value(self): + types = [r.type for r in dnsguide.build_records(self._args(caa_value=""))] + self.assertEqual(types, ["CNAME", "TXT"]) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From ecf89abe7365b40d930b69ccc725d11f56a49b47 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 18:41:44 -0700 Subject: [PATCH 02/16] fix(dstack-ingress): serialize bootstrap with the renewal daemon and let the gateway's DNS cache expire --- custom-domain/dstack-ingress/README.md | 1 + .../dstack-ingress/scripts/entrypoint.sh | 46 ++++++++++++++----- .../dstack-ingress/scripts/renewal-daemon.sh | 12 +++++ 3 files changed, 48 insertions(+), 11 deletions(-) diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index e840336f..2b7c6510 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -182,6 +182,7 @@ environment: | `TLSALPN_PORT` | `9443` | Loopback port lego's ACME responder binds to | | `TLS_TERMINATE_PORT` | `9444` | Loopback port the TLS frontend moves to in tls-alpn-01 mode | | `RENEW_DAYS_BEFORE` | lego default | Days of remaining lifetime that trigger renewal | +| `DNS_SETTLE_SECONDS` | `30` | Wait after DNS verifies, so the gateway's own TXT cache expires | | `MAXCONN` | `4096` | HAProxy max connections | | `TIMEOUT_CONNECT` | `10s` | Backend connect timeout | | `TIMEOUT_CLIENT` | `86400s` | Client-side timeout (24h for long-lived connections) | diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index 55f4cc7d..1cb4d9b3 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -613,6 +613,18 @@ process_domain_tlsalpn() { exit 1 fi + # Public DNS being correct is not the same as the gateway acting on it. The + # gateway resolves the app-address TXT itself and caches the answer for the + # record's TTL, so right after the value changes -- which is every time the + # CVM instance is replaced, since the record names the instance -- it can + # still route the challenge to the previous target. Observed exactly that: + # dnsguide passed, then the CA got "Error getting validation data" because + # the gateway was still sending it to the old instance. + if [ "${DNS_SETTLE_SECONDS:-30}" -gt 0 ]; then + echo "Waiting ${DNS_SETTLE_SECONDS:-30}s for the gateway's DNS cache to expire" + sleep "${DNS_SETTLE_SECONDS:-30}" + fi + renew-certificate.sh "$domain" } @@ -676,18 +688,30 @@ ensure_placeholder_certs() { # Obtain certificates once the proxy is already serving, then swap them in. bootstrap_and_reload() { - # Nested subshell so an `exit 1` from a process_domain helper is reported as - # a failed condition here rather than silently killing this background job. - if ( bootstrap ); then - build-combined-pems.sh || echo "Combined PEM build failed" >&2 - if [ -f /var/run/haproxy/haproxy.pid ]; then - kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)" && - echo "Certificates installed and HAProxy reloaded" + local attempt=0 delay + while true; do + attempt=$((attempt + 1)) + # Nested subshell so an `exit 1` from a process_domain helper is + # reported as a failed condition here rather than silently killing + # this background job. + if ( bootstrap ); then + build-combined-pems.sh || echo "Combined PEM build failed" >&2 + if [ -f /var/run/haproxy/haproxy.pid ]; then + kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)" && + echo "Certificates installed and HAProxy reloaded" + fi + return 0 fi - else - echo "Bootstrap failed; the proxy keeps serving the placeholder certificate." >&2 - echo "Fix the DNS records above; the renewal daemon retries every 12 hours." >&2 - fi + + # Falling through to the 12-hour renewal daemon would be a terrible + # retry interval for a flow whose normal state is "waiting for the + # operator to create a DNS record". Back off, but keep trying. + delay=$((60 * attempt)) + [ "$delay" -gt 1800 ] && delay=1800 + echo "Bootstrap attempt ${attempt} failed; the proxy keeps serving the" >&2 + echo "placeholder certificate. Retrying in ${delay}s." >&2 + sleep "$delay" + done } # Credentials are now handled by certman.py setup diff --git a/custom-domain/dstack-ingress/scripts/renewal-daemon.sh b/custom-domain/dstack-ingress/scripts/renewal-daemon.sh index d20614d0..4784cc0a 100755 --- a/custom-domain/dstack-ingress/scripts/renewal-daemon.sh +++ b/custom-domain/dstack-ingress/scripts/renewal-daemon.sh @@ -1,6 +1,18 @@ #!/bin/bash while true; do + # Never run concurrently with the initial bootstrap. Under tls-alpn-01 the + # proxy has to be listening before the first certificate can be issued, so + # bootstrap runs in the background while this daemon is already up. Two + # ACME clients at once fight over the challenge port ("bind: Address + # already in use") and, worse, the loser still spends one of Let's + # Encrypt's five failed validations per hostname per hour. + if [ ! -f /.bootstrapped ]; then + echo "[$(date)] Waiting for bootstrap to finish before checking renewals" + sleep 30 + continue + fi + echo "[$(date)] Checking for certificate renewal" all_domains=$(get-all-domains.sh) From 55fd225c15fb001ddfccaefdbaf14e9710cd834c Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 19:15:18 -0700 Subject: [PATCH 03/16] refactor(dstack-ingress): make one process own the certificate lifecycle --- custom-domain/dstack-ingress/README.md | 1 + .../dstack-ingress/scripts/cert-manager.sh | 118 ++++++ .../dstack-ingress/scripts/domains.sh | 228 +++++++++++ .../dstack-ingress/scripts/entrypoint.sh | 379 +----------------- .../scripts/renew-certificate.sh | 7 +- .../dstack-ingress/scripts/renewal-daemon.sh | 54 --- 6 files changed, 366 insertions(+), 421 deletions(-) create mode 100644 custom-domain/dstack-ingress/scripts/cert-manager.sh create mode 100644 custom-domain/dstack-ingress/scripts/domains.sh delete mode 100755 custom-domain/dstack-ingress/scripts/renewal-daemon.sh diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index 2b7c6510..8c76f59c 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -182,6 +182,7 @@ environment: | `TLSALPN_PORT` | `9443` | Loopback port lego's ACME responder binds to | | `TLS_TERMINATE_PORT` | `9444` | Loopback port the TLS frontend moves to in tls-alpn-01 mode | | `RENEW_DAYS_BEFORE` | lego default | Days of remaining lifetime that trigger renewal | +| `RENEW_INTERVAL` | `43200` | Seconds between successful certificate passes | | `DNS_SETTLE_SECONDS` | `30` | Wait after DNS verifies, so the gateway's own TXT cache expires | | `MAXCONN` | `4096` | HAProxy max connections | | `TIMEOUT_CONNECT` | `10s` | Backend connect timeout | diff --git a/custom-domain/dstack-ingress/scripts/cert-manager.sh b/custom-domain/dstack-ingress/scripts/cert-manager.sh new file mode 100644 index 00000000..9c18eb80 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/cert-manager.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# cert-manager.sh - the single owner of the certificate lifecycle. +# +# There used to be two drivers: a `bootstrap` that ran once at startup and a +# renewal daemon that woke every 12 hours. Both ended up calling +# renew-certificate.sh, so they were the same operation with different +# preludes. That was harmless only because bootstrap finished before the daemon +# started -- an ordering that stopped holding once tls-alpn-01 forced bootstrap +# into the background (haproxy has to be listening before the first certificate +# can be issued). The two then raced: both ACME clients tried to bind the same +# challenge port, and the loser still spent one of Let's Encrypt's five failed +# validations per hostname per hour. +# +# So there is now one pass, used for both jobs. process_domain is idempotent -- +# the DNS setters short-circuit when a record already holds the right value, the +# ACME account registration returns early when it exists, and the ACME client +# reports "no renewal needed" when the certificate is still fresh -- so the +# first pass and the ten-thousandth are the same code. +# +# Usage: +# cert-manager.sh --once run a single pass, exit with its status +# cert-manager.sh run passes forever + +set -uo pipefail + +source /scripts/functions.sh +source /scripts/domains.sh + +# Self-sufficient rather than trusting the parent's exports, for the same +# reason domains.sh carries its own defaults. +if [ -z "${DSTACK_INSTANCE_ID:-}" ] || [ -z "${DSTACK_APP_ID:-}" ]; then + load_dstack_identity +fi + +RENEW_INTERVAL=${RENEW_INTERVAL:-43200} # 12h between successful passes +RETRY_INTERVAL_MIN=${RETRY_INTERVAL_MIN:-60} # backoff floor after a failed pass +RETRY_INTERVAL_MAX=${RETRY_INTERVAL_MAX:-1800} + +reload_haproxy() { + if [ ! -f /var/run/haproxy/haproxy.pid ]; then + # Normal during the dns-01 startup pass: haproxy has not been exec'd + # yet, and it will read the fresh certificates when it starts. + echo "HAProxy is not running yet; certificates will be picked up at start" + return 0 + fi + if kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)"; then + echo "HAProxy reloaded with the new certificates" + else + echo "HAProxy reload failed: SIGUSR2 to PID $(cat /var/run/haproxy/haproxy.pid) failed" >&2 + return 1 + fi +} + +# One pass over every configured domain. Returns non-zero if any domain failed. +run_pass() { + local all_domains domain failed=0 changed=0 + + all_domains=$(get-all-domains.sh) + if [ -z "$all_domains" ]; then + echo "Error: No domains found. Set either DOMAIN or DOMAINS" >&2 + return 1 + fi + + local status + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + # Subshell: the process_domain helpers `exit` on fatal misconfiguration, + # which should fail this domain rather than kill the manager. + ( process_domain "$domain" ) + status=$? + # renew-certificate.sh reports 0 = certificate changed, 2 = nothing to + # do, 1 = failure, and process_domain passes that through. Reloading + # only on 0 is what keeps a quiet pass from churning haproxy. + case $status in + 0) changed=1 ;; + 2) ;; + *) + echo "Certificate management failed for $domain" >&2 + failed=1 + ;; + esac + done <<<"$all_domains" + + if [ "$changed" -eq 1 ]; then + generate-evidences.sh || echo "Evidence generation failed" >&2 + build-combined-pems.sh || echo "Combined PEM build failed" >&2 + reload_haproxy || true + fi + + if [ "$failed" -eq 0 ]; then + touch /.bootstrapped + return 0 + fi + return 1 +} + +if [ "${1:-}" = "--once" ]; then + run_pass + exit $? +fi + +attempt=0 +while true; do + if run_pass; then + attempt=0 + delay=$RENEW_INTERVAL + else + # Falling back to the 12-hour interval would be a terrible retry rate + # for a flow whose normal state is "waiting for an operator to create a + # DNS record". Back off, but stay responsive. + attempt=$((attempt + 1)) + delay=$((RETRY_INTERVAL_MIN * attempt)) + [ "$delay" -gt "$RETRY_INTERVAL_MAX" ] && delay=$RETRY_INTERVAL_MAX + echo "[$(date)] Pass failed (attempt ${attempt}); retrying in ${delay}s" + fi + echo "[$(date)] Next certificate check in ${delay}s" + sleep "$delay" +done diff --git a/custom-domain/dstack-ingress/scripts/domains.sh b/custom-domain/dstack-ingress/scripts/domains.sh new file mode 100644 index 00000000..4d3474d8 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/domains.sh @@ -0,0 +1,228 @@ +#!/bin/bash +# domains.sh - what DNS records each domain needs, and how to make them exist. +# +# Sourced by both entrypoint.sh and cert-manager.sh so there is exactly one +# definition of "process this domain", whether it is the first pass at startup +# or the periodic renewal pass. + +# This file is sourced by two independent processes (entrypoint.sh and +# cert-manager.sh), so it owns its own defaults rather than trusting the caller +# to have computed and exported them. Getting that wrong is quiet and nasty: an +# unset TXT_PREFIX builds a TXT name that can never exist, and the DNS wait then +# blocks forever on a record nobody could create. +# +# GATEWAY_DOMAIN, CERTBOT_EMAIL and SET_CAA are genuinely external (compose +# supplies them) and are deliberately not defaulted here. +PORT=${PORT:-443} +TXT_PREFIX=${TXT_PREFIX:-_dstack-app-address} +CHALLENGE_TYPE=${CHALLENGE_TYPE:-dns-01} +DNS_SETUP_MODE=${DNS_SETUP_MODE:-wait} +DNS_SETTLE_SECONDS=${DNS_SETTLE_SECONDS:-30} +SET_CAA=${SET_CAA:-false} +APP_ID=${APP_ID:-} + +set_alias_record() { + local domain="$1" + echo "Setting alias record for $domain" + dnsman.py set_alias \ + --domain "$domain" \ + --content "$GATEWAY_DOMAIN" + + if [ $? -ne 0 ]; then + echo "Error: Failed to set alias record for $domain" + exit 1 + fi + echo "Alias record set for $domain" +} + +# Query the guest agent once for this app's identity. +load_dstack_identity() { + local info + if [[ -S /var/run/dstack.sock ]]; then + info=$(curl -s --unix-socket /var/run/dstack.sock http://localhost/Info) + else + info=$(curl -s --unix-socket /var/run/tappd.sock http://localhost/prpc/Tappd.Info) + fi + + DSTACK_APP_ID=$(echo "$info" | jq -j .app_id) + DSTACK_INSTANCE_ID=$(echo "$info" | jq -j .instance_id) + export DSTACK_APP_ID DSTACK_INSTANCE_ID + + if [ -z "$DSTACK_APP_ID" ] || [ "$DSTACK_APP_ID" = "null" ]; then + echo "Error: could not read app_id from the dstack guest agent" >&2 + exit 1 + fi + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ] && + { [ -z "$DSTACK_INSTANCE_ID" ] || [ "$DSTACK_INSTANCE_ID" = "null" ]; }; then + echo "Error: could not read instance_id from the dstack guest agent, which" >&2 + echo "tls-alpn-01 needs to pin gateway routing to this instance" >&2 + exit 1 + fi +} + +txt_record_name() { + local domain="$1" + if [[ "$domain" == \*.* ]]; then + # Wildcard domain: *.myapp.com → _dstack-app-address-wildcard.myapp.com + echo "${TXT_PREFIX}-wildcard.${domain#\*.}" + else + echo "${TXT_PREFIX}.${domain}" + fi +} + +# What the gateway should route this hostname to. +# +# dns-01 uses the app id, so the gateway load-balances across every instance of +# the app. tls-alpn-01 cannot: the challenge is answered by whichever instance +# certbot runs on, and routing by app id makes the gateway race connections +# across the app's instances (select_top_n_hosts -> connect_multiple_hosts) +# while the CA validates from several vantage points at once. The challenge +# would land on the wrong replica. Pinning to the instance id makes validation +# deterministic -- at the cost of sending all traffic to this one instance, so +# tls-alpn-01 mode is effectively single-instance. +txt_record_value() { + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then + echo "${DSTACK_INSTANCE_ID}:${PORT}" + else + echo "${APP_ID:-$DSTACK_APP_ID}:${PORT}" + fi +} + +caa_tag_for() { + if [[ "$1" == \*.* ]]; then echo "issuewild"; else echo "issue"; fi +} + +set_txt_record() { + local domain="$1" + local txt_domain + txt_domain=$(txt_record_name "$domain") + + dnsman.py set_txt \ + --domain "$txt_domain" \ + --content "$(txt_record_value)" + + if [ $? -ne 0 ]; then + echo "Error: Failed to set TXT record for $domain" + exit 1 + fi +} + +set_caa_record() { + local domain="$1" + if [ "$SET_CAA" != "true" ]; then + echo "Skipping CAA record setup" + return + fi + + local ACCOUNT_URI + local account_file + + if ! account_file=$(get_letsencrypt_account_file); then + echo "Warning: Cannot set CAA record - account file not found" + echo "This is not critical - certificates can still be issued without CAA records" + return + fi + + local caa_domain caa_tag + caa_domain="${domain#\*.}" + caa_tag=$(caa_tag_for "$domain") + + ACCOUNT_URI=$(jq -j '.uri' "$account_file") + echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$ACCOUNT_URI" + dnsman.py set_caa \ + --domain "$caa_domain" \ + --caa-tag "$caa_tag" \ + --caa-value "letsencrypt.org;validationmethods=${CHALLENGE_TYPE};accounturi=$ACCOUNT_URI" + + if [ $? -ne 0 ]; then + echo "Warning: Failed to set CAA record for $domain" + echo "This is not critical - certificates can still be issued without CAA records" + echo "Consider disabling CAA records by setting SET_CAA=false if this continues to fail" + fi +} + +process_domain_dns01() { + local domain="$1" + + set_alias_record "$domain" + set_txt_record "$domain" + renew-certificate.sh "$domain" || echo "First certificate renewal failed for $domain, will retry after set CAA record" + set_caa_record "$domain" + renew-certificate.sh "$domain" +} + +process_domain_tlsalpn() { + local domain="$1" + + if [[ "$domain" == \*.* ]]; then + echo "Error: cannot issue a wildcard certificate for $domain with tls-alpn-01." >&2 + echo "RFC 8737 forbids it; use CHALLENGE_TYPE=dns-01 for wildcards." >&2 + exit 1 + fi + + # Register the ACME account first so the CAA record we print can already + # pin accounturi. Without this the operator would have to add CAA in a + # second pass, after the account exists. + if ! legoman.py register --email "$CERTBOT_EMAIL"; then + echo "Error: failed to register the ACME account" >&2 + exit 1 + fi + + local account_uri caa_value + account_uri=$(legoman.py account-uri --email "$CERTBOT_EMAIL" 2>/dev/null || true) + if [ -n "$account_uri" ]; then + caa_value="letsencrypt.org;validationmethods=tls-alpn-01;accounturi=$account_uri" + else + echo "Warning: could not read the ACME account URI; the CAA record will not pin it" >&2 + caa_value="letsencrypt.org;validationmethods=tls-alpn-01" + fi + + # Wait for the records routing depends on. An existing CAA record is also + # checked for compatibility -- not because CAA is required (an absent CAA + # record set permits every CA), but because an incompatible one makes the CA + # refuse and burns Let's Encrypt's failed-validation budget: 5 per account + # per hostname per hour. + if ! dnsguide.py \ + --domain "$domain" \ + --alias-target "$GATEWAY_DOMAIN" \ + --txt-name "$(txt_record_name "$domain")" \ + --txt-value "$(txt_record_value)" \ + --caa-name "${domain#\*.}" \ + --caa-tag "$(caa_tag_for "$domain")" \ + --caa-value "$caa_value" \ + --account-uri "$account_uri" \ + --challenge tls-alpn-01 \ + --mode "$DNS_SETUP_MODE"; then + echo "Error: required DNS records for $domain are not in place" >&2 + exit 1 + fi + + # Public DNS being correct is not the same as the gateway acting on it. The + # gateway resolves the app-address TXT itself and caches the answer for the + # record's TTL, so right after the value changes -- which is every time the + # CVM instance is replaced, since the record names the instance -- it can + # still route the challenge to the previous target. Observed exactly that: + # dnsguide passed, then the CA got "Error getting validation data" because + # the gateway was still sending it to the old instance. + # Only on the first pass: the settle wait exists because the TXT value has + # just changed, which is true at bootstrap (and after an instance + # replacement, which is a fresh container and therefore also a first pass). + # Renewal passes reuse a record the gateway resolved long ago. + if [ ! -f /.bootstrapped ] && [ "${DNS_SETTLE_SECONDS}" -gt 0 ]; then + echo "Waiting ${DNS_SETTLE_SECONDS}s for the gateway's DNS cache to expire" + sleep "${DNS_SETTLE_SECONDS}" + fi + + renew-certificate.sh "$domain" +} + +process_domain() { + local domain="$1" + echo "Processing domain: $domain (challenge: $CHALLENGE_TYPE)" + + if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then + process_domain_tlsalpn "$domain" + else + process_domain_dns01 "$domain" + fi +} diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index 1cb4d9b3..21d776c3 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -3,6 +3,7 @@ set -e source "/scripts/functions.sh" +source "/scripts/domains.sh" PORT=${PORT:-443} TXT_PREFIX=${TXT_PREFIX:-"_dstack-app-address"} @@ -75,8 +76,8 @@ if ! ALPN=$(sanitize_alpn "$ALPN"); then exit 1 fi -# renew-certificate.sh, renewal-daemon.sh and certman.py run as child processes -# and read these from the environment. +# cert-manager.sh, renew-certificate.sh and the ACME clients run as child +# processes and read these from the environment. export CHALLENGE_TYPE TLSALPN_ADDRESS TLSALPN_PORT DNS_SETUP_MODE PORT # Warn about deprecated L7 env vars @@ -352,319 +353,6 @@ backend ${be_name} emit_evidence_backend } -set_alias_record() { - local domain="$1" - if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then - # certbot validates the challenge at the base name even for a wildcard, - # so the _acme-challenge CNAME must use the base domain (strip "*."). - local base="${domain#\*.}" - echo "[challenge-delegation] Not touching ${domain}'s own zone (token is scoped to the delegated zone)." - echo "[challenge-delegation] Set these in your production zone yourself (static, one-time):" - echo " ${domain} CNAME ${GATEWAY_DOMAIN}" - echo " _acme-challenge.${base} CNAME _acme-challenge.${base}.${ACME_CHALLENGE_ALIAS}" - return - fi - echo "Setting alias record for $domain" - dnsman.py set_alias \ - --domain "$domain" \ - --content "$GATEWAY_DOMAIN" - - if [ $? -ne 0 ]; then - echo "Error: Failed to set alias record for $domain" - exit 1 - fi - echo "Alias record set for $domain" -} - -# Query the guest agent once for this app's identity. -load_dstack_identity() { - local info - if [[ -S /var/run/dstack.sock ]]; then - info=$(curl -s --unix-socket /var/run/dstack.sock http://localhost/Info) - else - info=$(curl -s --unix-socket /var/run/tappd.sock http://localhost/prpc/Tappd.Info) - fi - - DSTACK_APP_ID=$(echo "$info" | jq -j .app_id) - DSTACK_INSTANCE_ID=$(echo "$info" | jq -j .instance_id) - export DSTACK_APP_ID DSTACK_INSTANCE_ID - - if [ -z "$DSTACK_APP_ID" ] || [ "$DSTACK_APP_ID" = "null" ]; then - echo "Error: could not read app_id from the dstack guest agent" >&2 - exit 1 - fi - if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ] && - { [ -z "$DSTACK_INSTANCE_ID" ] || [ "$DSTACK_INSTANCE_ID" = "null" ]; }; then - echo "Error: could not read instance_id from the dstack guest agent, which" >&2 - echo "tls-alpn-01 needs to pin gateway routing to this instance" >&2 - exit 1 - fi -} - -txt_record_name() { - local domain="$1" - if [[ "$domain" == \*.* ]]; then - # Wildcard domain: *.myapp.com → _dstack-app-address-wildcard.myapp.com - echo "${TXT_PREFIX}-wildcard.${domain#\*.}" - else - echo "${TXT_PREFIX}.${domain}" - fi -} - -# What the gateway should route this hostname to. -# -# dns-01 uses the app id, so the gateway load-balances across every instance of -# the app. tls-alpn-01 cannot: the challenge is answered by whichever instance -# certbot runs on, and routing by app id makes the gateway race connections -# across the app's instances (select_top_n_hosts -> connect_multiple_hosts) -# while the CA validates from several vantage points at once. The challenge -# would land on the wrong replica. Pinning to the instance id makes validation -# deterministic -- at the cost of sending all traffic to this one instance, so -# tls-alpn-01 mode is effectively single-instance. -txt_record_value() { - if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then - echo "${DSTACK_INSTANCE_ID}:${PORT}" - else - echo "${APP_ID:-$DSTACK_APP_ID}:${PORT}" - fi -} - -caa_tag_for() { - if [[ "$1" == \*.* ]]; then echo "issuewild"; else echo "issue"; fi -} - -set_txt_record() { - local domain="$1" - local txt_domain - txt_domain=$(txt_record_name "$domain") - - if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then - echo "[challenge-delegation] Set this in your production zone yourself (static, one-time):" - echo " ${txt_domain} TXT \"$APP_ID:$PORT\"" - return - fi - - dnsman.py set_txt \ - --domain "$txt_domain" \ - --content "$(txt_record_value)" - - if [ $? -ne 0 ]; then - echo "Error: Failed to set TXT record for $domain" - exit 1 - fi -} - -# caa_domain_and_tag DOMAIN -> prints "caa_domain caa_tag" (strips a wildcard). -caa_domain_and_tag() { - local domain="$1" - if [[ "$domain" == \*.* ]]; then - echo "${domain#\*.} issuewild" - else - echo "$domain issue" - fi -} - -# In delegation mode we have NO token for the served domain's zone, so we cannot -# set the accounturi CAA ourselves. That CAA is the forge-prevention (only this -# enclave's ACME account may issue), so this path is deliberately NOT gated by -# SET_CAA and fails closed if the record is confirmed absent — otherwise anyone -# who can satisfy the delegated DNS-01 challenge could obtain a cert for the -# domain. Set ALLOW_MISSING_CAA=true to override (accept the risk). -verify_delegation_caa() { - local domain="$1" - local account_file account_uri caa_domain caa_tag caa_value resp status found - - if ! account_file=$(get_letsencrypt_account_file); then - echo "ERROR: cannot read the Let's Encrypt account file to determine the required accounturi CAA" >&2 - caa_fail_or_allow "$domain" - return - fi - read -r caa_domain caa_tag < <(caa_domain_and_tag "$domain") - account_uri=$(jq -j '.uri' "$account_file") - caa_value="letsencrypt.org;validationmethods=dns-01;accounturi=$account_uri" - - echo "[challenge-delegation] Set this CAA in your production zone (static, one-time):" - echo " ${caa_domain} CAA 0 ${caa_tag} \"${caa_value}\"" - - # Verify via DNS-over-HTTPS (dig is not installed in the image; curl+jq are). - resp=$(curl -s --max-time 10 "https://dns.google/resolve?name=${caa_domain}&type=257" 2>/dev/null || true) - if [ -z "$resp" ]; then - echo "WARNING: could not reach dns.google to verify the CAA (network issue) — NOT confirmed; continuing" - return - fi - status=$(echo "$resp" | jq -r '.Status // empty' 2>/dev/null || true) - if [ "$status" != "0" ]; then - echo "WARNING: CAA DoH query for ${caa_domain} returned status ${status:-unknown} — NOT confirmed; continuing" - return - fi - # Literal (grep -F) match on the CAA data fields — the account URI contains - # '/' and '.', which must not be treated as regex. - found=$(echo "$resp" | jq -r '.Answer // [] | .[] | .data' 2>/dev/null | grep -F "accounturi=$account_uri" || true) - if [ -n "$found" ]; then - echo "[challenge-delegation] Verified: accounturi CAA is present for $caa_domain" - return - fi - echo "ERROR: the accounturi CAA is NOT present for $caa_domain." >&2 - echo "ERROR: without it, anyone who can satisfy the delegated DNS-01 challenge could obtain a" >&2 - echo "ERROR: publicly-trusted certificate for this domain (forged TLS termination)." >&2 - caa_fail_or_allow "$domain" -} - -caa_fail_or_allow() { - local domain="$1" - if [ "${ALLOW_MISSING_CAA:-false}" = "true" ]; then - echo "WARNING: ALLOW_MISSING_CAA=true — continuing without a verified accounturi CAA for $domain" - return 0 - fi - echo "ERROR: refusing to continue without the accounturi CAA for $domain." >&2 - echo "ERROR: set the CAA record shown above, or ALLOW_MISSING_CAA=true to override." >&2 - exit 1 -} - -set_caa_record() { - local domain="$1" - - # Delegation mode is handled separately and is NOT gated by SET_CAA. - if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then - verify_delegation_caa "$domain" - return - fi - - if [ "$SET_CAA" != "true" ]; then - echo "Skipping CAA record setup" - return - fi - - local account_file account_uri caa_domain caa_tag - if ! account_file=$(get_letsencrypt_account_file); then - echo "Warning: Cannot set CAA record - account file not found" - echo "This is not critical - certificates can still be issued without CAA records" - return - fi - read -r caa_domain caa_tag < <(caa_domain_and_tag "$domain") - account_uri=$(jq -j '.uri' "$account_file") - - echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$account_uri" - dnsman.py set_caa \ - --domain "$caa_domain" \ - --caa-tag "$caa_tag" \ - --caa-value "letsencrypt.org;validationmethods=${CHALLENGE_TYPE};accounturi=$account_uri" - - if [ $? -ne 0 ]; then - echo "Warning: Failed to set CAA record for $domain" - echo "This is not critical - certificates can still be issued without CAA records" - echo "Consider disabling CAA records by setting SET_CAA=false if this continues to fail" - fi -} - -process_domain_dns01() { - local domain="$1" - - set_alias_record "$domain" - set_txt_record "$domain" - renew-certificate.sh "$domain" || echo "First certificate renewal failed for $domain, will retry after set CAA record" - set_caa_record "$domain" - renew-certificate.sh "$domain" -} - -process_domain_tlsalpn() { - local domain="$1" - - if [[ "$domain" == \*.* ]]; then - echo "Error: cannot issue a wildcard certificate for $domain with tls-alpn-01." >&2 - echo "RFC 8737 forbids it; use CHALLENGE_TYPE=dns-01 for wildcards." >&2 - exit 1 - fi - - # Register the ACME account first so the CAA record we print can already - # pin accounturi. Without this the operator would have to add CAA in a - # second pass, after the account exists. - if ! legoman.py register --email "$CERTBOT_EMAIL"; then - echo "Error: failed to register the ACME account" >&2 - exit 1 - fi - - local account_uri caa_value - account_uri=$(legoman.py account-uri --email "$CERTBOT_EMAIL" 2>/dev/null || true) - if [ -n "$account_uri" ]; then - caa_value="letsencrypt.org;validationmethods=tls-alpn-01;accounturi=$account_uri" - else - echo "Warning: could not read the ACME account URI; the CAA record will not pin it" >&2 - caa_value="letsencrypt.org;validationmethods=tls-alpn-01" - fi - - # Wait for the records routing depends on. An existing CAA record is also - # checked for compatibility -- not because CAA is required (an absent CAA - # record set permits every CA), but because an incompatible one makes the CA - # refuse and burns Let's Encrypt's failed-validation budget: 5 per account - # per hostname per hour. - if ! dnsguide.py \ - --domain "$domain" \ - --alias-target "$GATEWAY_DOMAIN" \ - --txt-name "$(txt_record_name "$domain")" \ - --txt-value "$(txt_record_value)" \ - --caa-name "${domain#\*.}" \ - --caa-tag "$(caa_tag_for "$domain")" \ - --caa-value "$caa_value" \ - --account-uri "$account_uri" \ - --challenge tls-alpn-01 \ - --mode "$DNS_SETUP_MODE"; then - echo "Error: required DNS records for $domain are not in place" >&2 - exit 1 - fi - - # Public DNS being correct is not the same as the gateway acting on it. The - # gateway resolves the app-address TXT itself and caches the answer for the - # record's TTL, so right after the value changes -- which is every time the - # CVM instance is replaced, since the record names the instance -- it can - # still route the challenge to the previous target. Observed exactly that: - # dnsguide passed, then the CA got "Error getting validation data" because - # the gateway was still sending it to the old instance. - if [ "${DNS_SETTLE_SECONDS:-30}" -gt 0 ]; then - echo "Waiting ${DNS_SETTLE_SECONDS:-30}s for the gateway's DNS cache to expire" - sleep "${DNS_SETTLE_SECONDS:-30}" - fi - - renew-certificate.sh "$domain" -} - -process_domain() { - local domain="$1" - echo "Processing domain: $domain (challenge: $CHALLENGE_TYPE)" - - if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then - process_domain_tlsalpn "$domain" - else - process_domain_dns01 "$domain" - fi -} - -bootstrap() { - echo "Bootstrap: Setting up domains" - - local all_domains - all_domains=$(get-all-domains.sh) - - if [ -z "$all_domains" ]; then - echo "Error: No domains found. Set either DOMAIN or DOMAINS environment variable" - exit 1 - fi - - echo "Found domains:" - echo "$all_domains" - - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - process_domain "$domain" - done <<<"$all_domains" - - # Generate evidences after all certificates are obtained - echo "Generating evidence files for all domains..." - generate-evidences.sh - - touch /.bootstrapped -} - # haproxy refuses to start when the crt directory has no PEM in it. In # tls-alpn-01 mode the proxy has to be listening *before* the first certificate # exists, because it is what forwards the ACME challenge to certbot, so seed a @@ -687,32 +375,6 @@ ensure_placeholder_certs() { } # Obtain certificates once the proxy is already serving, then swap them in. -bootstrap_and_reload() { - local attempt=0 delay - while true; do - attempt=$((attempt + 1)) - # Nested subshell so an `exit 1` from a process_domain helper is - # reported as a failed condition here rather than silently killing - # this background job. - if ( bootstrap ); then - build-combined-pems.sh || echo "Combined PEM build failed" >&2 - if [ -f /var/run/haproxy/haproxy.pid ]; then - kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)" && - echo "Certificates installed and HAProxy reloaded" - fi - return 0 - fi - - # Falling through to the 12-hour renewal daemon would be a terrible - # retry interval for a flow whose normal state is "waiting for the - # operator to create a DNS record". Back off, but keep trying. - delay=$((60 * attempt)) - [ "$delay" -gt 1800 ] && delay=1800 - echo "Bootstrap attempt ${attempt} failed; the proxy keeps serving the" >&2 - echo "placeholder certificate. Retrying in ${delay}s." >&2 - sleep "$delay" - done -} # Credentials are now handled by certman.py setup @@ -721,23 +383,17 @@ setup_certbot_env load_dstack_identity if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then - # Ordering is load-bearing here. The tls-alpn-01 challenge arrives on the - # public TLS port, so haproxy must already be routing acme-tls/1 to certbot - # before the first certificate can be issued. Start the proxy on a - # placeholder certificate and let issuance converge behind it. + # Ordering is load-bearing. The tls-alpn-01 challenge arrives on the public + # TLS port, so haproxy has to be forwarding acme-tls/1 before the first + # certificate can exist. Start on a placeholder and let the certificate + # manager converge behind it. ensure_placeholder_certs - build-combined-pems.sh || true else - # dns-01 needs no inbound traffic, so keep the original order: get the - # certificate first, start serving second. - if [ ! -f "/.bootstrapped" ]; then - bootstrap - else - echo "Certificate for $DOMAIN already exists" - generate-evidences.sh - fi - build-combined-pems.sh + # dns-01 needs no inbound traffic, so keep the original order: certificate + # first, serving second, and a failure here is fatal as it always was. + cert-manager.sh --once fi +build-combined-pems.sh || true # Generate haproxy config if [ -n "$ROUTING_MAP" ]; then @@ -752,15 +408,8 @@ if [ "$EVIDENCE_SERVER" = "true" ]; then echo "Evidence server started on port ${EVIDENCE_PORT} (mini_httpd)" fi -if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then - if [ ! -f "/.bootstrapped" ]; then - bootstrap_and_reload & - else - echo "Certificates already bootstrapped" - generate-evidences.sh || echo "Evidence generation failed" >&2 - fi -fi - -renewal-daemon.sh & +# One process owns certificates from here on: first pass (tls-alpn-01, where it +# has to run behind a live proxy) and every renewal after that. +cert-manager.sh & exec "$@" diff --git a/custom-domain/dstack-ingress/scripts/renew-certificate.sh b/custom-domain/dstack-ingress/scripts/renew-certificate.sh index 8ae97d4d..aa746ef4 100755 --- a/custom-domain/dstack-ingress/scripts/renew-certificate.sh +++ b/custom-domain/dstack-ingress/scripts/renew-certificate.sh @@ -16,9 +16,12 @@ CERT_STATUS=$? if [ $CERT_STATUS -eq 1 ]; then echo "Certificate management failed" >&2 - exit 1 elif [ $CERT_STATUS -eq 2 ]; then echo "No certificates need renewal, skipping evidence generation" fi -exit 0 \ No newline at end of file +# Propagate the tri-state: 0 = certificate changed, 2 = nothing to do, 1 = +# failure. This used to always exit 0, which told the caller "something +# changed" on every single pass -- so evidences were regenerated and haproxy +# reloaded every 12 hours whether or not a certificate had moved. +exit $CERT_STATUS \ No newline at end of file diff --git a/custom-domain/dstack-ingress/scripts/renewal-daemon.sh b/custom-domain/dstack-ingress/scripts/renewal-daemon.sh deleted file mode 100755 index 4784cc0a..00000000 --- a/custom-domain/dstack-ingress/scripts/renewal-daemon.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash - -while true; do - # Never run concurrently with the initial bootstrap. Under tls-alpn-01 the - # proxy has to be listening before the first certificate can be issued, so - # bootstrap runs in the background while this daemon is already up. Two - # ACME clients at once fight over the challenge port ("bind: Address - # already in use") and, worse, the loser still spends one of Let's - # Encrypt's five failed validations per hostname per hour. - if [ ! -f /.bootstrapped ]; then - echo "[$(date)] Waiting for bootstrap to finish before checking renewals" - sleep 30 - continue - fi - - echo "[$(date)] Checking for certificate renewal" - - all_domains=$(get-all-domains.sh) - - if [ -n "$all_domains" ]; then - renewal_occurred=false - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - echo "[$(date)] Checking renewal for domain: $domain" - if renew-certificate.sh "$domain"; then - renewal_occurred=true - else - echo "Certificate renewal check failed for $domain with status $?" - fi - done <<<"$all_domains" - - if [ "$renewal_occurred" = true ]; then - echo "[$(date)] Generating evidence files after renewals..." - generate-evidences.sh || echo "Evidence generation failed" - - # Rebuild combined PEM files for haproxy - build-combined-pems.sh || echo "Combined PEM build failed" - - # Graceful reload: send SIGUSR2 to haproxy master process - if [ ! -f /var/run/haproxy/haproxy.pid ]; then - echo "HAProxy reload failed: PID file /var/run/haproxy/haproxy.pid not found" >&2 - elif ! kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)"; then - echo "HAProxy reload failed: SIGUSR2 to PID $(cat /var/run/haproxy/haproxy.pid) failed" >&2 - else - echo "Certificate renewed and HAProxy reloaded successfully" - fi - fi - else - echo "[$(date)] No domains configured for renewal" - fi - - echo "[$(date)] Next renewal check in 12 hours" - sleep 43200 -done From dfbc15d9c9e1a8ed63c6c123f04125903bd6f587 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 19:28:01 -0700 Subject: [PATCH 04/16] fix(dstack-ingress): bind the evidence server to loopback --- .../dstack-ingress/scripts/entrypoint.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index 21d776c3..7c7af916 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -404,8 +404,22 @@ fi # Start evidence HTTP server if enabled if [ "$EVIDENCE_SERVER" = "true" ]; then - mini_httpd -d /evidences -p "${EVIDENCE_PORT}" -D -l /dev/stderr & - echo "Evidence server started on port ${EVIDENCE_PORT} (mini_httpd)" + # Bind loopback explicitly, for two reasons. + # + # Correctness of the logs: left to itself mini_httpd binds [::]:port first, + # and since the container has net.ipv6.bindv6only=0 that socket already + # covers IPv4, so its second bind on 0.0.0.0:port fails with EADDRINUSE. It + # logs "bind: Address already in use", keeps serving on the surviving + # socket, and leaves a scary line in the log forever. + # + # Reach: the only consumer is haproxy's be_evidence backend, which connects + # to 127.0.0.1:${EVIDENCE_PORT}. Listening on every interface additionally + # exposed the evidence server to any other container on the compose + # network at ingress:${EVIDENCE_PORT}, bypassing the proxy. Nothing + # documented depends on that; evidence is meant to be read through + # https:///evidences/ or the shared /evidences volume. + mini_httpd -d /evidences -p "${EVIDENCE_PORT}" -h 127.0.0.1 -D -l /dev/stderr & + echo "Evidence server started on 127.0.0.1:${EVIDENCE_PORT} (mini_httpd)" fi # One process owns certificates from here on: first pass (tls-alpn-01, where it From ef6c76051f25ff5c1fcfe5e57834d6985e18a0b0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 20:54:26 -0700 Subject: [PATCH 05/16] refactor(dstack-ingress): give each challenge type its own top-level script --- .../scripts/build-combined-pems.sh | 26 -- .../dstack-ingress/scripts/cert-manager.sh | 118 ------ custom-domain/dstack-ingress/scripts/dns01.sh | 246 +++++++++++ .../dstack-ingress/scripts/domains.sh | 228 ---------- .../dstack-ingress/scripts/entrypoint.sh | 388 ++---------------- .../dstack-ingress/scripts/evidence-lib.sh | 75 ++++ .../dstack-ingress/scripts/functions.sh | 69 +--- .../scripts/generate-evidences.sh | 65 --- .../dstack-ingress/scripts/haproxy-lib.sh | 187 +++++++++ .../dstack-ingress/scripts/legoman.py | 2 +- .../scripts/renew-certificate.sh | 27 -- .../dstack-ingress/scripts/tlsalpn.sh | 308 ++++++++++++++ 12 files changed, 860 insertions(+), 879 deletions(-) delete mode 100644 custom-domain/dstack-ingress/scripts/build-combined-pems.sh delete mode 100644 custom-domain/dstack-ingress/scripts/cert-manager.sh create mode 100644 custom-domain/dstack-ingress/scripts/dns01.sh delete mode 100644 custom-domain/dstack-ingress/scripts/domains.sh create mode 100644 custom-domain/dstack-ingress/scripts/evidence-lib.sh delete mode 100644 custom-domain/dstack-ingress/scripts/generate-evidences.sh create mode 100644 custom-domain/dstack-ingress/scripts/haproxy-lib.sh delete mode 100755 custom-domain/dstack-ingress/scripts/renew-certificate.sh create mode 100644 custom-domain/dstack-ingress/scripts/tlsalpn.sh diff --git a/custom-domain/dstack-ingress/scripts/build-combined-pems.sh b/custom-domain/dstack-ingress/scripts/build-combined-pems.sh deleted file mode 100644 index 8b0a31c6..00000000 --- a/custom-domain/dstack-ingress/scripts/build-combined-pems.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# build-combined-pems.sh - Concatenate Let's Encrypt cert files into -# HAProxy combined PEM format (fullchain + privkey in one file). - -set -e - -source /scripts/functions.sh - -CERT_DIR="/etc/haproxy/certs" -mkdir -p "$CERT_DIR" - -all_domains=$(get-all-domains.sh) - -while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - fullchain=$(cert_fullchain_path "$domain") - privkey=$(cert_privkey_path "$domain") - combined="${CERT_DIR}/${domain}.pem" - if [ -f "$fullchain" ] && [ -f "$privkey" ]; then - cat "$fullchain" "$privkey" > "$combined" - chmod 600 "$combined" - echo "Combined PEM created: ${combined}" - else - echo "Warning: Cert files missing for ${domain}, skipping" - fi -done <<< "$all_domains" diff --git a/custom-domain/dstack-ingress/scripts/cert-manager.sh b/custom-domain/dstack-ingress/scripts/cert-manager.sh deleted file mode 100644 index 9c18eb80..00000000 --- a/custom-domain/dstack-ingress/scripts/cert-manager.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash -# cert-manager.sh - the single owner of the certificate lifecycle. -# -# There used to be two drivers: a `bootstrap` that ran once at startup and a -# renewal daemon that woke every 12 hours. Both ended up calling -# renew-certificate.sh, so they were the same operation with different -# preludes. That was harmless only because bootstrap finished before the daemon -# started -- an ordering that stopped holding once tls-alpn-01 forced bootstrap -# into the background (haproxy has to be listening before the first certificate -# can be issued). The two then raced: both ACME clients tried to bind the same -# challenge port, and the loser still spent one of Let's Encrypt's five failed -# validations per hostname per hour. -# -# So there is now one pass, used for both jobs. process_domain is idempotent -- -# the DNS setters short-circuit when a record already holds the right value, the -# ACME account registration returns early when it exists, and the ACME client -# reports "no renewal needed" when the certificate is still fresh -- so the -# first pass and the ten-thousandth are the same code. -# -# Usage: -# cert-manager.sh --once run a single pass, exit with its status -# cert-manager.sh run passes forever - -set -uo pipefail - -source /scripts/functions.sh -source /scripts/domains.sh - -# Self-sufficient rather than trusting the parent's exports, for the same -# reason domains.sh carries its own defaults. -if [ -z "${DSTACK_INSTANCE_ID:-}" ] || [ -z "${DSTACK_APP_ID:-}" ]; then - load_dstack_identity -fi - -RENEW_INTERVAL=${RENEW_INTERVAL:-43200} # 12h between successful passes -RETRY_INTERVAL_MIN=${RETRY_INTERVAL_MIN:-60} # backoff floor after a failed pass -RETRY_INTERVAL_MAX=${RETRY_INTERVAL_MAX:-1800} - -reload_haproxy() { - if [ ! -f /var/run/haproxy/haproxy.pid ]; then - # Normal during the dns-01 startup pass: haproxy has not been exec'd - # yet, and it will read the fresh certificates when it starts. - echo "HAProxy is not running yet; certificates will be picked up at start" - return 0 - fi - if kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)"; then - echo "HAProxy reloaded with the new certificates" - else - echo "HAProxy reload failed: SIGUSR2 to PID $(cat /var/run/haproxy/haproxy.pid) failed" >&2 - return 1 - fi -} - -# One pass over every configured domain. Returns non-zero if any domain failed. -run_pass() { - local all_domains domain failed=0 changed=0 - - all_domains=$(get-all-domains.sh) - if [ -z "$all_domains" ]; then - echo "Error: No domains found. Set either DOMAIN or DOMAINS" >&2 - return 1 - fi - - local status - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - # Subshell: the process_domain helpers `exit` on fatal misconfiguration, - # which should fail this domain rather than kill the manager. - ( process_domain "$domain" ) - status=$? - # renew-certificate.sh reports 0 = certificate changed, 2 = nothing to - # do, 1 = failure, and process_domain passes that through. Reloading - # only on 0 is what keeps a quiet pass from churning haproxy. - case $status in - 0) changed=1 ;; - 2) ;; - *) - echo "Certificate management failed for $domain" >&2 - failed=1 - ;; - esac - done <<<"$all_domains" - - if [ "$changed" -eq 1 ]; then - generate-evidences.sh || echo "Evidence generation failed" >&2 - build-combined-pems.sh || echo "Combined PEM build failed" >&2 - reload_haproxy || true - fi - - if [ "$failed" -eq 0 ]; then - touch /.bootstrapped - return 0 - fi - return 1 -} - -if [ "${1:-}" = "--once" ]; then - run_pass - exit $? -fi - -attempt=0 -while true; do - if run_pass; then - attempt=0 - delay=$RENEW_INTERVAL - else - # Falling back to the 12-hour interval would be a terrible retry rate - # for a flow whose normal state is "waiting for an operator to create a - # DNS record". Back off, but stay responsive. - attempt=$((attempt + 1)) - delay=$((RETRY_INTERVAL_MIN * attempt)) - [ "$delay" -gt "$RETRY_INTERVAL_MAX" ] && delay=$RETRY_INTERVAL_MAX - echo "[$(date)] Pass failed (attempt ${attempt}); retrying in ${delay}s" - fi - echo "[$(date)] Next certificate check in ${delay}s" - sleep "$delay" -done diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh new file mode 100644 index 00000000..7f39e561 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -0,0 +1,246 @@ +#!/bin/bash +# dns01.sh - certificates via certbot and a DNS provider API. +# +# entrypoint.sh validates the shared settings and then execs this file. There is +# no shared orchestration above it and no interface it has to implement -- this +# is simply the program that runs for dns-01, top to bottom. + +set -e + +source /scripts/functions.sh +source /scripts/haproxy-lib.sh +source /scripts/evidence-lib.sh + +# certbot stores certificates under /etc/letsencrypt/live//. +cert_fullchain_path() { echo "/etc/letsencrypt/live/$(cert_dir_name "$1")/fullchain.pem"; } +cert_privkey_path() { echo "/etc/letsencrypt/live/$(cert_dir_name "$1")/privkey.pem"; } + +setup_py_env() { + if [ ! -d /opt/app-venv ]; then + echo "Creating application virtual environment" + python3 -m venv --system-site-packages /opt/app-venv + fi + + # Activate venv for subsequent steps + # shellcheck disable=SC1091 + source /opt/app-venv/bin/activate + + if [ ! -f /.venv_bootstrapped ]; then + echo "Bootstrapping certbot dependencies" + pip install --upgrade pip + pip install certbot requests boto3 botocore + touch /.venv_bootstrapped + fi + + ln -sf /opt/app-venv/bin/certbot /usr/local/bin/certbot + echo 'source /opt/app-venv/bin/activate' > /etc/profile.d/app-venv.sh +} + +setup_certbot_env() { + # Ensure the virtual environment is active for certbot configuration + # shellcheck disable=SC1091 + source /opt/app-venv/bin/activate + + if [ "${DNS_PROVIDER}" = "route53" ]; then + mkdir -p /root/.aws + + cat </root/.aws/config +[profile certbot] +role_arn=${AWS_ROLE_ARN} +source_profile=certbot-source +region=${AWS_REGION:-us-east-1} +EOF + + cat </root/.aws/credentials +[certbot-source] +aws_access_key_id=${AWS_ACCESS_KEY_ID} +aws_secret_access_key=${AWS_SECRET_ACCESS_KEY} +EOF + + unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN + export AWS_PROFILE=certbot + fi + + # Use the unified certbot manager to install plugins and setup credentials + echo "Installing DNS plugins and setting up credentials" + certman.py setup + if [ $? -ne 0 ]; then + echo "Error: Failed to setup certbot environment" + exit 1 + fi +} + +set_alias_record() { + local domain="$1" + echo "Setting alias record for $domain" + dnsman.py set_alias \ + --domain "$domain" \ + --content "$GATEWAY_DOMAIN" + + if [ $? -ne 0 ]; then + echo "Error: Failed to set alias record for $domain" + exit 1 + fi + echo "Alias record set for $domain" +} + +txt_record_value() { echo "${APP_ID:-$DSTACK_APP_ID}:${PORT}"; } + +set_txt_record() { + local domain="$1" + local txt_domain + txt_domain=$(txt_record_name "$domain") + + dnsman.py set_txt \ + --domain "$txt_domain" \ + --content "$(txt_record_value)" + + if [ $? -ne 0 ]; then + echo "Error: Failed to set TXT record for $domain" + exit 1 + fi +} + +set_caa_record() { + local domain="$1" + if [ "$SET_CAA" != "true" ]; then + echo "Skipping CAA record setup" + return + fi + + local ACCOUNT_URI + local account_file + + if ! account_file=$(get_letsencrypt_account_file); then + echo "Warning: Cannot set CAA record - account file not found" + echo "This is not critical - certificates can still be issued without CAA records" + return + fi + + local caa_domain caa_tag + caa_domain="${domain#\*.}" + caa_tag=$(caa_tag_for "$domain") + + ACCOUNT_URI=$(jq -j '.uri' "$account_file") + echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$ACCOUNT_URI" + dnsman.py set_caa \ + --domain "$caa_domain" \ + --caa-tag "$caa_tag" \ + --caa-value "letsencrypt.org;validationmethods=dns-01;accounturi=$ACCOUNT_URI" + + if [ $? -ne 0 ]; then + echo "Warning: Failed to set CAA record for $domain" + echo "This is not critical - certificates can still be issued without CAA records" + echo "Consider disabling CAA records by setting SET_CAA=false if this continues to fail" + fi +} + +process_domain() { + local domain="$1" + + set_alias_record "$domain" + set_txt_record "$domain" + # The CAA record names our ACME account, so the account has to exist first: + # try once (expected to fail on a fresh deployment), write CAA, try again. + issue_certificate "$domain" || echo "First certificate attempt failed for $domain, retrying after the CAA record is set" + set_caa_record "$domain" + issue_certificate "$domain" +} + +issue_certificate() { + source /opt/app-venv/bin/activate + certman.py auto --domain "$1" --email "$CERTBOT_EMAIL" +} + +build_combined_pems() { + local domain fullchain privkey combined + mkdir -p /etc/haproxy/certs + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + fullchain=$(cert_fullchain_path "$domain") + privkey=$(cert_privkey_path "$domain") + combined="/etc/haproxy/certs/${domain}.pem" + if [ -f "$fullchain" ] && [ -f "$privkey" ]; then + cat "$fullchain" "$privkey" > "$combined" + chmod 600 "$combined" + echo "Combined PEM created: ${combined}" + else + echo "Warning: Cert files missing for ${domain}, skipping" + fi + done <<<"$(get-all-domains.sh)" +} + +collect_evidence() { + local account domain + if ! account=$(get_letsencrypt_account_file); then + echo "Error: cannot generate evidence without the ACME account file" >&2 + return 1 + fi + evidence_reset + evidence_collect_account "$account" + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + evidence_collect_cert "$domain" "$(cert_fullchain_path "$domain")" + done <<<"$(get-all-domains.sh)" + evidence_finalize +} + +# One pass over every domain: make the DNS records right, then issue or renew. +# Idempotent, so the first pass and every later one are the same code. +run_pass() { + local domain status failed=0 changed=0 + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + # `if` context, not a bare command: process_domain returns 2 for + # "nothing to do", and under `set -e` a bare non-zero would kill the + # whole script on every quiet renewal pass. + if ( process_domain "$domain" ); then status=0; else status=$?; fi + case $status in + 0) changed=1 ;; + 2) ;; + *) echo "Certificate management failed for $domain" >&2; failed=1 ;; + esac + done <<<"$(get-all-domains.sh)" + + if [ "$changed" -eq 1 ]; then + collect_evidence || echo "Evidence generation failed" >&2 + build_combined_pems || echo "Combined PEM build failed" >&2 + haproxy_reload || true + fi + [ "$failed" -eq 0 ] +} + +cert_loop() { + local attempt=0 delay + while true; do + if run_pass; then + attempt=0 + delay=${RENEW_INTERVAL:-43200} + else + attempt=$((attempt + 1)) + delay=$((60 * attempt)) + [ "$delay" -gt 1800 ] && delay=1800 + echo "[$(date)] Pass failed (attempt ${attempt}); retrying in ${delay}s" + fi + echo "[$(date)] Next certificate check in ${delay}s" + sleep "$delay" + done +} + +setup_py_env +setup_certbot_env +load_dstack_identity + +# dns-01 needs no inbound traffic, so keep the original order: certificate +# first, serving second. A failure here is fatal, as it always was. +run_pass +build_combined_pems + +haproxy_emit_global +haproxy_emit_tls_frontend ":${PORT}" +haproxy_emit_backends + +evidence_start_server +cert_loop & + +exec "$@" diff --git a/custom-domain/dstack-ingress/scripts/domains.sh b/custom-domain/dstack-ingress/scripts/domains.sh deleted file mode 100644 index 4d3474d8..00000000 --- a/custom-domain/dstack-ingress/scripts/domains.sh +++ /dev/null @@ -1,228 +0,0 @@ -#!/bin/bash -# domains.sh - what DNS records each domain needs, and how to make them exist. -# -# Sourced by both entrypoint.sh and cert-manager.sh so there is exactly one -# definition of "process this domain", whether it is the first pass at startup -# or the periodic renewal pass. - -# This file is sourced by two independent processes (entrypoint.sh and -# cert-manager.sh), so it owns its own defaults rather than trusting the caller -# to have computed and exported them. Getting that wrong is quiet and nasty: an -# unset TXT_PREFIX builds a TXT name that can never exist, and the DNS wait then -# blocks forever on a record nobody could create. -# -# GATEWAY_DOMAIN, CERTBOT_EMAIL and SET_CAA are genuinely external (compose -# supplies them) and are deliberately not defaulted here. -PORT=${PORT:-443} -TXT_PREFIX=${TXT_PREFIX:-_dstack-app-address} -CHALLENGE_TYPE=${CHALLENGE_TYPE:-dns-01} -DNS_SETUP_MODE=${DNS_SETUP_MODE:-wait} -DNS_SETTLE_SECONDS=${DNS_SETTLE_SECONDS:-30} -SET_CAA=${SET_CAA:-false} -APP_ID=${APP_ID:-} - -set_alias_record() { - local domain="$1" - echo "Setting alias record for $domain" - dnsman.py set_alias \ - --domain "$domain" \ - --content "$GATEWAY_DOMAIN" - - if [ $? -ne 0 ]; then - echo "Error: Failed to set alias record for $domain" - exit 1 - fi - echo "Alias record set for $domain" -} - -# Query the guest agent once for this app's identity. -load_dstack_identity() { - local info - if [[ -S /var/run/dstack.sock ]]; then - info=$(curl -s --unix-socket /var/run/dstack.sock http://localhost/Info) - else - info=$(curl -s --unix-socket /var/run/tappd.sock http://localhost/prpc/Tappd.Info) - fi - - DSTACK_APP_ID=$(echo "$info" | jq -j .app_id) - DSTACK_INSTANCE_ID=$(echo "$info" | jq -j .instance_id) - export DSTACK_APP_ID DSTACK_INSTANCE_ID - - if [ -z "$DSTACK_APP_ID" ] || [ "$DSTACK_APP_ID" = "null" ]; then - echo "Error: could not read app_id from the dstack guest agent" >&2 - exit 1 - fi - if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ] && - { [ -z "$DSTACK_INSTANCE_ID" ] || [ "$DSTACK_INSTANCE_ID" = "null" ]; }; then - echo "Error: could not read instance_id from the dstack guest agent, which" >&2 - echo "tls-alpn-01 needs to pin gateway routing to this instance" >&2 - exit 1 - fi -} - -txt_record_name() { - local domain="$1" - if [[ "$domain" == \*.* ]]; then - # Wildcard domain: *.myapp.com → _dstack-app-address-wildcard.myapp.com - echo "${TXT_PREFIX}-wildcard.${domain#\*.}" - else - echo "${TXT_PREFIX}.${domain}" - fi -} - -# What the gateway should route this hostname to. -# -# dns-01 uses the app id, so the gateway load-balances across every instance of -# the app. tls-alpn-01 cannot: the challenge is answered by whichever instance -# certbot runs on, and routing by app id makes the gateway race connections -# across the app's instances (select_top_n_hosts -> connect_multiple_hosts) -# while the CA validates from several vantage points at once. The challenge -# would land on the wrong replica. Pinning to the instance id makes validation -# deterministic -- at the cost of sending all traffic to this one instance, so -# tls-alpn-01 mode is effectively single-instance. -txt_record_value() { - if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then - echo "${DSTACK_INSTANCE_ID}:${PORT}" - else - echo "${APP_ID:-$DSTACK_APP_ID}:${PORT}" - fi -} - -caa_tag_for() { - if [[ "$1" == \*.* ]]; then echo "issuewild"; else echo "issue"; fi -} - -set_txt_record() { - local domain="$1" - local txt_domain - txt_domain=$(txt_record_name "$domain") - - dnsman.py set_txt \ - --domain "$txt_domain" \ - --content "$(txt_record_value)" - - if [ $? -ne 0 ]; then - echo "Error: Failed to set TXT record for $domain" - exit 1 - fi -} - -set_caa_record() { - local domain="$1" - if [ "$SET_CAA" != "true" ]; then - echo "Skipping CAA record setup" - return - fi - - local ACCOUNT_URI - local account_file - - if ! account_file=$(get_letsencrypt_account_file); then - echo "Warning: Cannot set CAA record - account file not found" - echo "This is not critical - certificates can still be issued without CAA records" - return - fi - - local caa_domain caa_tag - caa_domain="${domain#\*.}" - caa_tag=$(caa_tag_for "$domain") - - ACCOUNT_URI=$(jq -j '.uri' "$account_file") - echo "Adding CAA record ($caa_tag) for $caa_domain, accounturi=$ACCOUNT_URI" - dnsman.py set_caa \ - --domain "$caa_domain" \ - --caa-tag "$caa_tag" \ - --caa-value "letsencrypt.org;validationmethods=${CHALLENGE_TYPE};accounturi=$ACCOUNT_URI" - - if [ $? -ne 0 ]; then - echo "Warning: Failed to set CAA record for $domain" - echo "This is not critical - certificates can still be issued without CAA records" - echo "Consider disabling CAA records by setting SET_CAA=false if this continues to fail" - fi -} - -process_domain_dns01() { - local domain="$1" - - set_alias_record "$domain" - set_txt_record "$domain" - renew-certificate.sh "$domain" || echo "First certificate renewal failed for $domain, will retry after set CAA record" - set_caa_record "$domain" - renew-certificate.sh "$domain" -} - -process_domain_tlsalpn() { - local domain="$1" - - if [[ "$domain" == \*.* ]]; then - echo "Error: cannot issue a wildcard certificate for $domain with tls-alpn-01." >&2 - echo "RFC 8737 forbids it; use CHALLENGE_TYPE=dns-01 for wildcards." >&2 - exit 1 - fi - - # Register the ACME account first so the CAA record we print can already - # pin accounturi. Without this the operator would have to add CAA in a - # second pass, after the account exists. - if ! legoman.py register --email "$CERTBOT_EMAIL"; then - echo "Error: failed to register the ACME account" >&2 - exit 1 - fi - - local account_uri caa_value - account_uri=$(legoman.py account-uri --email "$CERTBOT_EMAIL" 2>/dev/null || true) - if [ -n "$account_uri" ]; then - caa_value="letsencrypt.org;validationmethods=tls-alpn-01;accounturi=$account_uri" - else - echo "Warning: could not read the ACME account URI; the CAA record will not pin it" >&2 - caa_value="letsencrypt.org;validationmethods=tls-alpn-01" - fi - - # Wait for the records routing depends on. An existing CAA record is also - # checked for compatibility -- not because CAA is required (an absent CAA - # record set permits every CA), but because an incompatible one makes the CA - # refuse and burns Let's Encrypt's failed-validation budget: 5 per account - # per hostname per hour. - if ! dnsguide.py \ - --domain "$domain" \ - --alias-target "$GATEWAY_DOMAIN" \ - --txt-name "$(txt_record_name "$domain")" \ - --txt-value "$(txt_record_value)" \ - --caa-name "${domain#\*.}" \ - --caa-tag "$(caa_tag_for "$domain")" \ - --caa-value "$caa_value" \ - --account-uri "$account_uri" \ - --challenge tls-alpn-01 \ - --mode "$DNS_SETUP_MODE"; then - echo "Error: required DNS records for $domain are not in place" >&2 - exit 1 - fi - - # Public DNS being correct is not the same as the gateway acting on it. The - # gateway resolves the app-address TXT itself and caches the answer for the - # record's TTL, so right after the value changes -- which is every time the - # CVM instance is replaced, since the record names the instance -- it can - # still route the challenge to the previous target. Observed exactly that: - # dnsguide passed, then the CA got "Error getting validation data" because - # the gateway was still sending it to the old instance. - # Only on the first pass: the settle wait exists because the TXT value has - # just changed, which is true at bootstrap (and after an instance - # replacement, which is a fresh container and therefore also a first pass). - # Renewal passes reuse a record the gateway resolved long ago. - if [ ! -f /.bootstrapped ] && [ "${DNS_SETTLE_SECONDS}" -gt 0 ]; then - echo "Waiting ${DNS_SETTLE_SECONDS}s for the gateway's DNS cache to expire" - sleep "${DNS_SETTLE_SECONDS}" - fi - - renew-certificate.sh "$domain" -} - -process_domain() { - local domain="$1" - echo "Processing domain: $domain (challenge: $CHALLENGE_TYPE)" - - if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then - process_domain_tlsalpn "$domain" - else - process_domain_dns01 "$domain" - fi -} diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index 7c7af916..587bf830 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -1,9 +1,15 @@ #!/bin/bash +# entrypoint.sh - validate the settings both modes share, then hand over. +# +# There is no shared orchestration below this point: dns-01 and tls-alpn-01 have +# genuinely little in common (different ACME client, different on-disk layout, +# different DNS story, different proxy topology, different startup order), so +# each is its own program. What they truly share lives in the *-lib.sh files and +# is composed by the mode, not dispatched to. set -e -source "/scripts/functions.sh" -source "/scripts/domains.sh" +source /scripts/functions.sh PORT=${PORT:-443} TXT_PREFIX=${TXT_PREFIX:-"_dstack-app-address"} @@ -15,13 +21,6 @@ EVIDENCE_SERVER=${EVIDENCE_SERVER:-true} EVIDENCE_PORT=${EVIDENCE_PORT:-80} ALPN=${ALPN:-} CHALLENGE_TYPE=${CHALLENGE_TYPE:-dns-01} -# Loopback address/port lego's tls-alpn-01 responder binds to, and the loopback -# port the real TLS frontend moves to so the public port can be used for -# ALPN-based routing. Neither is reachable from outside the container. -TLSALPN_ADDRESS=${TLSALPN_ADDRESS:-127.0.0.1} -TLSALPN_PORT=${TLSALPN_PORT:-9443} -TLS_TERMINATE_PORT=${TLS_TERMINATE_PORT:-9444} -DNS_SETUP_MODE=${DNS_SETUP_MODE:-wait} case "$CHALLENGE_TYPE" in dns-01|tls-alpn-01) ;; @@ -31,23 +30,9 @@ case "$CHALLENGE_TYPE" in ;; esac -case "$DNS_SETUP_MODE" in - wait|print|webhook) ;; - *) - echo "Error: invalid DNS_SETUP_MODE '$DNS_SETUP_MODE' (expected wait, print or webhook)" >&2 - exit 1 - ;; -esac - if ! PORT=$(sanitize_port "$PORT"); then exit 1 fi -if ! TLSALPN_PORT=$(sanitize_port "$TLSALPN_PORT"); then - exit 1 -fi -if ! TLS_TERMINATE_PORT=$(sanitize_port "$TLS_TERMINATE_PORT"); then - exit 1 -fi if ! DOMAIN=$(sanitize_domain "$DOMAIN"); then exit 1 fi @@ -76,10 +61,6 @@ if ! ALPN=$(sanitize_alpn "$ALPN"); then exit 1 fi -# cert-manager.sh, renew-certificate.sh and the ACME clients run as child -# processes and read these from the environment. -export CHALLENGE_TYPE TLSALPN_ADDRESS TLSALPN_PORT DNS_SETUP_MODE PORT - # Warn about deprecated L7 env vars for var in CLIENT_MAX_BODY_SIZE PROXY_READ_TIMEOUT PROXY_SEND_TIMEOUT PROXY_CONNECT_TIMEOUT PROXY_BUFFER_SIZE PROXY_BUFFERS PROXY_BUSY_BUFFERS_SIZE; do if [ -n "${!var}" ]; then @@ -87,343 +68,20 @@ for var in CLIENT_MAX_BODY_SIZE PROXY_READ_TIMEOUT PROXY_SEND_TIMEOUT PROXY_CONN fi done -# Parse TARGET_ENDPOINT into host:port for haproxy backend -parse_target_endpoint() { - local endpoint="$1" - # Strip protocol prefix if present (http://, https://, grpc://) - local hostport="${endpoint#*://}" - # If no protocol was stripped, use as-is - if [ "$hostport" = "$endpoint" ]; then - hostport="$endpoint" - fi - # Strip any trailing path - hostport="${hostport%%/*}" - echo "$hostport" -} - -echo "Setting up certbot environment" - -setup_py_env() { - if [ ! -d /opt/app-venv ]; then - echo "Creating application virtual environment" - python3 -m venv --system-site-packages /opt/app-venv - fi - - # Activate venv for subsequent steps - # shellcheck disable=SC1091 - source /opt/app-venv/bin/activate - - if [ ! -f /.venv_bootstrapped ]; then - pip install --upgrade pip - if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then - # lego handles ACME here, so certbot and the cloud SDKs are dead - # weight: skip them and keep startup and attack surface smaller. - echo "Bootstrapping python dependencies (tls-alpn-01: no certbot)" - pip install requests - else - echo "Bootstrapping certbot dependencies" - pip install certbot requests boto3 botocore - fi - touch /.venv_bootstrapped - fi - - [ -x /opt/app-venv/bin/certbot ] && ln -sf /opt/app-venv/bin/certbot /usr/local/bin/certbot - echo 'source /opt/app-venv/bin/activate' > /etc/profile.d/app-venv.sh -} - -setup_certbot_env() { - # Ensure the virtual environment is active for certbot configuration - # shellcheck disable=SC1091 - source /opt/app-venv/bin/activate - - if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then - # No DNS provider and no certbot plugin: lego handles this path and is - # baked into the image. - if [ ! -x "${LEGO_BIN:-/usr/local/bin/lego}" ]; then - echo "Error: lego binary not found at ${LEGO_BIN:-/usr/local/bin/lego}" >&2 - exit 1 - fi - echo "Using lego for tls-alpn-01: $(${LEGO_BIN:-/usr/local/bin/lego} --version)" - return - fi - - if [ "${DNS_PROVIDER}" = "route53" ]; then - mkdir -p /root/.aws +# Everything from here on belongs to one mode. Exported so the mode script and +# the helpers it invokes see the sanitized values. +export PORT DOMAIN DOMAINS TARGET_ENDPOINT ROUTING_MAP TXT_PREFIX MAXCONN +export TIMEOUT_CONNECT TIMEOUT_CLIENT TIMEOUT_SERVER EVIDENCE_SERVER EVIDENCE_PORT ALPN - cat </root/.aws/config -[profile certbot] -role_arn=${AWS_ROLE_ARN} -source_profile=certbot-source -region=${AWS_REGION:-us-east-1} -EOF - - cat </root/.aws/credentials -[certbot-source] -aws_access_key_id=${AWS_ACCESS_KEY_ID} -aws_secret_access_key=${AWS_SECRET_ACCESS_KEY} -EOF - - unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN - export AWS_PROFILE=certbot - fi - - # Use the unified certbot manager to install plugins and setup credentials - echo "Installing DNS plugins and setting up credentials" - certman.py setup - if [ $? -ne 0 ]; then - echo "Error: Failed to setup certbot environment" +case "$CHALLENGE_TYPE" in + dns-01) + exec /scripts/dns01.sh "$@" + ;; + tls-alpn-01) + exec /scripts/tlsalpn.sh "$@" + ;; + *) + echo "Error: invalid CHALLENGE_TYPE '$CHALLENGE_TYPE' (expected dns-01 or tls-alpn-01)" >&2 exit 1 - fi -} - -setup_py_env - -# Emit common haproxy global/defaults/frontend preamble. -# Both single-domain and multi-domain modes share this identical config. -emit_haproxy_preamble() { - # In tls-alpn-01 mode the ACME responder has to complete the TLS handshake - # itself, but haproxy owns the public port. So the public port becomes a - # plain TCP frontend that peeks at the ClientHello and forwards only - # acme-tls/1 connections to lego; everything else goes to the real TLS - # frontend, which moves to loopback. PROXY protocol carries the client - # address across that extra hop so the backend still sees the real peer. - local tls_bind=":${PORT}" - if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then - tls_bind="127.0.0.1:${TLS_TERMINATE_PORT} accept-proxy" - fi - - # "crt " loads all PEM files from the directory. - # ALPN is appended conditionally via ${ALPN:+ alpn ${ALPN}}. - cat </etc/haproxy/haproxy.cfg -global - log stdout format raw local0 - maxconn ${MAXCONN} - pidfile /var/run/haproxy/haproxy.pid - ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305 - ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 - ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets - ssl-default-bind-curves secp384r1 - -defaults - log global - mode tcp - option tcplog - timeout connect ${TIMEOUT_CONNECT} - timeout client ${TIMEOUT_CLIENT} - timeout server ${TIMEOUT_SERVER} - -EOF - - if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then - cat <>/etc/haproxy/haproxy.cfg -frontend tls_peek - bind :${PORT} - tcp-request inspect-delay 5s - tcp-request content accept if { req.ssl_hello_type 1 } - # Only the CA sends this ALPN protocol. A client that sends it anyway just - # reaches lego's responder, which refuses anything but acme-tls/1. - use_backend be_acme if { req.ssl_alpn -i acme-tls/1 } - default_backend be_tls_terminate - -backend be_acme - server lego ${TLSALPN_ADDRESS}:${TLSALPN_PORT} - -backend be_tls_terminate - server local 127.0.0.1:${TLS_TERMINATE_PORT} send-proxy-v2 - -EOF - fi - - cat <>/etc/haproxy/haproxy.cfg -frontend tls_in - bind ${tls_bind} ssl crt /etc/haproxy/certs/${ALPN:+ alpn ${ALPN}} -EOF - - if [ "$EVIDENCE_SERVER" = "true" ]; then - cat <<'EVIDENCE_BLOCK' >>/etc/haproxy/haproxy.cfg - - # Route /evidences requests to the local evidence HTTP server. - # accept fires once 16 bytes have arrived — enough for the - # longest prefix we match ("HEAD /evidences" = 16 chars). - # Using req.len with a concrete threshold is critical: the - # previous payload(0,0) (length 0 = "whole buffer") deferred - # evaluation until the full inspect-delay because HAProxy - # cannot know when a TCP stream ends. - tcp-request inspect-delay 5s - tcp-request content accept if { req.len ge 16 } - acl is_evidence payload(0,16) -m beg "GET /evidences" - acl is_evidence payload(0,16) -m beg "HEAD /evidences" - use_backend be_evidence if is_evidence -EVIDENCE_BLOCK - fi -} - -# Append the evidence backend block to haproxy.cfg -emit_evidence_backend() { - if [ "$EVIDENCE_SERVER" = "true" ]; then - cat <>/etc/haproxy/haproxy.cfg - -backend be_evidence - mode http - http-request replace-path /evidences(.*) \1 - server evidence 127.0.0.1:${EVIDENCE_PORT} -EOF - fi -} - -# Generate haproxy.cfg for single-domain mode (DOMAIN + TARGET_ENDPOINT) -setup_haproxy_cfg() { - local target_hostport - target_hostport=$(parse_target_endpoint "$TARGET_ENDPOINT") - - emit_haproxy_preamble - - cat <>/etc/haproxy/haproxy.cfg - - default_backend be_upstream - -backend be_upstream - server app1 ${target_hostport} -EOF - - emit_evidence_backend -} - -# Generate haproxy.cfg for multi-domain mode (ROUTING_MAP) -setup_haproxy_cfg_multi() { - emit_haproxy_preamble - - # Parse ROUTING_MAP and generate use_backend rules + backend sections - # Support both newline-separated and comma-separated formats - local routing_map_normalized - routing_map_normalized=$(echo "$ROUTING_MAP" | tr ',' '\n') - - local backend_rules="" - local backend_sections="" - local first_be_name="" - local domain target be_name - - while IFS= read -r line; do - [[ -n "$line" ]] || continue - [[ "$line" == \#* ]] && continue - domain="${line%%=*}" - target="${line#*=}" - domain=$(echo "$domain" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - target=$(echo "$target" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') - [[ -n "$domain" && -n "$target" ]] || continue - - # Validate domain and target to prevent config injection - if ! domain=$(sanitize_domain "$domain"); then - echo "Error: Invalid domain in ROUTING_MAP: ${line}" >&2 - exit 1 - fi - if ! target=$(sanitize_target_endpoint "$target"); then - echo "Error: Invalid target in ROUTING_MAP: ${line}" >&2 - exit 1 - fi - - # Strip protocol prefix from target if present - target=$(parse_target_endpoint "$target") - - # Generate safe backend name from domain - be_name="be_$(echo "$domain" | sed 's/[^A-Za-z0-9]/_/g')" - - if [ -z "$first_be_name" ]; then - first_be_name="$be_name" - fi - - backend_rules="${backend_rules} - use_backend ${be_name} if { ssl_fc_sni -i ${domain} }" - backend_sections="${backend_sections} - -backend ${be_name} - server s1 ${target}" - done <<< "$routing_map_normalized" - - echo "$backend_rules" >> /etc/haproxy/haproxy.cfg - - # Default to first backend in ROUTING_MAP - if [ -n "$first_be_name" ]; then - echo "" >> /etc/haproxy/haproxy.cfg - echo " default_backend ${first_be_name}" >> /etc/haproxy/haproxy.cfg - fi - - echo "$backend_sections" >> /etc/haproxy/haproxy.cfg - - emit_evidence_backend -} - -# haproxy refuses to start when the crt directory has no PEM in it. In -# tls-alpn-01 mode the proxy has to be listening *before* the first certificate -# exists, because it is what forwards the ACME challenge to certbot, so seed a -# self-signed placeholder that the real certificate overwrites later. -ensure_placeholder_certs() { - local all_domains domain pem - all_domains=$(get-all-domains.sh) - while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - pem="/etc/haproxy/certs/${domain}.pem" - [ -f "$pem" ] && continue - echo "Generating placeholder certificate for ${domain}" - openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ - -keyout /tmp/placeholder.key -out /tmp/placeholder.crt \ - -subj "/CN=${domain#\*.}" >/dev/null 2>&1 - cat /tmp/placeholder.crt /tmp/placeholder.key >"$pem" - chmod 600 "$pem" - rm -f /tmp/placeholder.key /tmp/placeholder.crt - done <<<"$all_domains" -} - -# Obtain certificates once the proxy is already serving, then swap them in. - -# Credentials are now handled by certman.py setup - -# Setup certbot environment (venv is already created in Dockerfile) -setup_certbot_env -load_dstack_identity - -if [ "$CHALLENGE_TYPE" = "tls-alpn-01" ]; then - # Ordering is load-bearing. The tls-alpn-01 challenge arrives on the public - # TLS port, so haproxy has to be forwarding acme-tls/1 before the first - # certificate can exist. Start on a placeholder and let the certificate - # manager converge behind it. - ensure_placeholder_certs -else - # dns-01 needs no inbound traffic, so keep the original order: certificate - # first, serving second, and a failure here is fatal as it always was. - cert-manager.sh --once -fi -build-combined-pems.sh || true - -# Generate haproxy config -if [ -n "$ROUTING_MAP" ]; then - setup_haproxy_cfg_multi -elif [ -n "$DOMAIN" ] && [ -n "$TARGET_ENDPOINT" ]; then - setup_haproxy_cfg -fi - -# Start evidence HTTP server if enabled -if [ "$EVIDENCE_SERVER" = "true" ]; then - # Bind loopback explicitly, for two reasons. - # - # Correctness of the logs: left to itself mini_httpd binds [::]:port first, - # and since the container has net.ipv6.bindv6only=0 that socket already - # covers IPv4, so its second bind on 0.0.0.0:port fails with EADDRINUSE. It - # logs "bind: Address already in use", keeps serving on the surviving - # socket, and leaves a scary line in the log forever. - # - # Reach: the only consumer is haproxy's be_evidence backend, which connects - # to 127.0.0.1:${EVIDENCE_PORT}. Listening on every interface additionally - # exposed the evidence server to any other container on the compose - # network at ingress:${EVIDENCE_PORT}, bypassing the proxy. Nothing - # documented depends on that; evidence is meant to be read through - # https:///evidences/ or the shared /evidences volume. - mini_httpd -d /evidences -p "${EVIDENCE_PORT}" -h 127.0.0.1 -D -l /dev/stderr & - echo "Evidence server started on 127.0.0.1:${EVIDENCE_PORT} (mini_httpd)" -fi - -# One process owns certificates from here on: first pass (tls-alpn-01, where it -# has to run behind a live proxy) and every renewal after that. -cert-manager.sh & - -exec "$@" + ;; +esac diff --git a/custom-domain/dstack-ingress/scripts/evidence-lib.sh b/custom-domain/dstack-ingress/scripts/evidence-lib.sh new file mode 100644 index 00000000..644200a3 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/evidence-lib.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# evidence-lib.sh - the evidence file format and its TDX quote. +# +# The modes disagree about where the ACME account document and the certificates +# live on disk, so each mode hands its own paths to evidence_collect_*. What the +# evidence *is* -- the file names, the manifest, and the report_data binding -- +# is one definition, here. + +EVIDENCE_DIR=${EVIDENCE_DIR:-/evidences} + +evidence_reset() { + mkdir -p "$EVIDENCE_DIR" +} + +# These files exist to be published, so force them world-readable. lego writes +# its account document and certificates 0600 and cp keeps that mode, which makes +# the evidence server answer 403. Neither carries a private key: the account +# document holds the account URL and status, and a certificate chain is public +# by construction. +evidence_collect_account() { + cp "$1" "$EVIDENCE_DIR/acme-account.json" + chmod 644 "$EVIDENCE_DIR/acme-account.json" +} + +evidence_collect_cert() { + local domain="$1" path="$2" + if [ ! -f "$path" ]; then + echo "Warning: Certificate not found for domain: $domain" + return 0 + fi + cp "$path" "$EVIDENCE_DIR/cert-${domain}.pem" + chmod 644 "$EVIDENCE_DIR/cert-${domain}.pem" +} + +# Hash whatever was collected and bind it into a TDX quote. +evidence_finalize() { + cd "$EVIDENCE_DIR" || return 1 +# Generate checksum for all files + sha256sum acme-account.json cert-*.pem > sha256sum.txt 2>/dev/null || { + echo "Error: No certificate files found" + return 1 + } + + QUOTED_HASH=$(sha256sum sha256sum.txt | awk '{print $1}') + + PADDED_HASH="${QUOTED_HASH}" + while [ ${#PADDED_HASH} -lt 128 ]; do + PADDED_HASH="${PADDED_HASH}0" + done + QUOTED_HASH="${PADDED_HASH}" + + if [[ -S /var/run/dstack.sock ]]; then + curl -s --unix-socket /var/run/dstack.sock "http://localhost/GetQuote?report_data=${QUOTED_HASH}" > quote.json + else + curl -s --unix-socket /var/run/tappd.sock "http://localhost/prpc/Tappd.RawQuote?report_data=${QUOTED_HASH}" > quote.json + fi + if [ $? -ne 0 ]; then + echo "Error: Failed to generate evidences" + return 1 + fi + echo "Generated evidences successfully" +} + +evidence_start_server() { + [ "${EVIDENCE_SERVER:-true}" = "true" ] || return 0 + # Bind loopback explicitly. Left to itself mini_httpd binds [::]:port first, + # and since the container has net.ipv6.bindv6only=0 that socket already + # covers IPv4, so its second bind fails with EADDRINUSE and it logs a + # permanent "bind: Address already in use". The only consumer is haproxy's + # be_evidence backend on 127.0.0.1 anyway; listening on every interface also + # exposed this to other containers on the compose network, bypassing the + # proxy. + mini_httpd -d "$EVIDENCE_DIR" -p "${EVIDENCE_PORT:-80}" -h 127.0.0.1 -D -l /dev/stderr & + echo "Evidence server started on 127.0.0.1:${EVIDENCE_PORT:-80} (mini_httpd)" +} diff --git a/custom-domain/dstack-ingress/scripts/functions.sh b/custom-domain/dstack-ingress/scripts/functions.sh index c0e5df1e..23d0df85 100644 --- a/custom-domain/dstack-ingress/scripts/functions.sh +++ b/custom-domain/dstack-ingress/scripts/functions.sh @@ -188,64 +188,35 @@ get_letsencrypt_account_file() { echo "${account_files[0]}" } -# --- ACME client layout ------------------------------------------------------ -# -# The two challenge types use different clients, and they lay their state out -# differently: -# -# certbot (dns-01) /etc/letsencrypt/live//{fullchain,privkey}.pem -# /etc/letsencrypt/accounts//directory/*/regr.json -# lego (tls-alpn-01) $LEGO_PATH/certificates/.{crt,key} -# $LEGO_PATH/accounts///account.json -# -# Everything downstream (combined PEMs, evidence) goes through these helpers so -# it does not have to care which client produced the files. - -lego_path() { - echo "${LEGO_PATH:-/etc/letsencrypt/lego}" -} - -using_lego() { - [[ "${CHALLENGE_TYPE:-dns-01}" == "tls-alpn-01" ]] -} - -# lego writes the full chain into the .crt file. -cert_fullchain_path() { +txt_record_name() { local domain="$1" - if using_lego; then - echo "$(lego_path)/certificates/${domain}.crt" + if [[ "$domain" == \*.* ]]; then + # Wildcard domain: *.myapp.com → _dstack-app-address-wildcard.myapp.com + echo "${TXT_PREFIX}-wildcard.${domain#\*.}" else - echo "/etc/letsencrypt/live/$(cert_dir_name "$domain")/fullchain.pem" + echo "${TXT_PREFIX}.${domain}" fi } -cert_privkey_path() { - local domain="$1" - if using_lego; then - echo "$(lego_path)/certificates/${domain}.key" - else - echo "/etc/letsencrypt/live/$(cert_dir_name "$domain")/privkey.pem" - fi +caa_tag_for() { + if [[ "$1" == \*.* ]]; then echo "issuewild"; else echo "issue"; fi } -get_lego_account_file() { - local pattern account_files - pattern="$(lego_path)/accounts/*/*/account.json" - account_files=( $pattern ) - - if [[ ! -f "${account_files[0]}" ]]; then - echo "Error: lego account file not found at $pattern" >&2 - return 1 +# Query the guest agent once for this app's identity. +load_dstack_identity() { + local info + if [[ -S /var/run/dstack.sock ]]; then + info=$(curl -s --unix-socket /var/run/dstack.sock http://localhost/Info) + else + info=$(curl -s --unix-socket /var/run/tappd.sock http://localhost/prpc/Tappd.Info) fi - echo "${account_files[0]}" -} + DSTACK_APP_ID=$(echo "$info" | jq -j .app_id) + DSTACK_INSTANCE_ID=$(echo "$info" | jq -j .instance_id) + export DSTACK_APP_ID DSTACK_INSTANCE_ID -# Account registration document, whichever client produced it. -acme_account_file() { - if using_lego; then - get_lego_account_file - else - get_letsencrypt_account_file + if [ -z "$DSTACK_APP_ID" ] || [ "$DSTACK_APP_ID" = "null" ]; then + echo "Error: could not read app_id from the dstack guest agent" >&2 + exit 1 fi } diff --git a/custom-domain/dstack-ingress/scripts/generate-evidences.sh b/custom-domain/dstack-ingress/scripts/generate-evidences.sh deleted file mode 100644 index 40d2e0b3..00000000 --- a/custom-domain/dstack-ingress/scripts/generate-evidences.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -set -e - -source "/scripts/functions.sh" - -if ! ACME_ACCOUNT_FILE=$(acme_account_file); then - echo "Error: Cannot generate evidences without the ACME account file" - exit 1 -fi - -mkdir -p /evidences -cd /evidences || exit -cp "${ACME_ACCOUNT_FILE}" acme-account.json -# These files exist to be published, so force them world-readable. lego writes -# its account document and certificates 0600 and cp keeps that mode, which makes -# the evidence server answer 403. (certbot's regr.json/fullchain.pem are 0644, -# so this only ever bit the lego path.) Neither file carries a private key: the -# account document holds the account URL and status, and a certificate chain is -# public by construction. -chmod 644 acme-account.json - -# Get all domains and copy their certificates -all_domains=$(get-all-domains.sh) -if [ -z "$all_domains" ]; then - echo "Error: No domains found for evidence generation" - exit 1 -fi - -# Copy all certificate files -while IFS= read -r domain; do - [[ -n "$domain" ]] || continue - cert_file=$(cert_fullchain_path "$domain") - if [ -f "$cert_file" ]; then - cp "$cert_file" "cert-${domain}.pem" - chmod 644 "cert-${domain}.pem" - else - echo "Warning: Certificate not found for domain: $domain" - fi -done <<< "$all_domains" - -# Generate checksum for all files -sha256sum acme-account.json cert-*.pem > sha256sum.txt 2>/dev/null || { - echo "Error: No certificate files found" - exit 1 -} - -QUOTED_HASH=$(sha256sum sha256sum.txt | awk '{print $1}') - -PADDED_HASH="${QUOTED_HASH}" -while [ ${#PADDED_HASH} -lt 128 ]; do - PADDED_HASH="${PADDED_HASH}0" -done -QUOTED_HASH="${PADDED_HASH}" - -if [[ -S /var/run/dstack.sock ]]; then - curl -s --unix-socket /var/run/dstack.sock "http://localhost/GetQuote?report_data=${QUOTED_HASH}" > quote.json -else - curl -s --unix-socket /var/run/tappd.sock "http://localhost/prpc/Tappd.RawQuote?report_data=${QUOTED_HASH}" > quote.json -fi -if [ $? -ne 0 ]; then - echo "Error: Failed to generate evidences" - exit 1 -fi -echo "Generated evidences successfully" diff --git a/custom-domain/dstack-ingress/scripts/haproxy-lib.sh b/custom-domain/dstack-ingress/scripts/haproxy-lib.sh new file mode 100644 index 00000000..8d506fce --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/haproxy-lib.sh @@ -0,0 +1,187 @@ +#!/bin/bash +# haproxy-lib.sh - the haproxy config pieces that are identical in every mode. +# +# The modes differ only in their frontends: dns-01 terminates TLS on the public +# port, tls-alpn-01 puts a ClientHello-peeking frontend there and moves the TLS +# frontend to loopback. global/defaults, the evidence ACL, the backends and the +# reload are the same either way, so they live here and each mode composes them. +# Nothing in this file branches on the mode; callers pass what differs. + +# Parse TARGET_ENDPOINT into host:port for haproxy backend +parse_target_endpoint() { + local endpoint="$1" + # Strip protocol prefix if present (http://, https://, grpc://) + local hostport="${endpoint#*://}" + # If no protocol was stripped, use as-is + if [ "$hostport" = "$endpoint" ]; then + hostport="$endpoint" + fi + # Strip any trailing path + hostport="${hostport%%/*}" + echo "$hostport" +} + +# global + defaults. Always the first thing written to haproxy.cfg. +haproxy_emit_global() { + cat </etc/haproxy/haproxy.cfg +global + log stdout format raw local0 + maxconn ${MAXCONN} + pidfile /var/run/haproxy/haproxy.pid + ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305 + ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets + ssl-default-bind-curves secp384r1 + +defaults + log global + mode tcp + option tcplog + timeout connect ${TIMEOUT_CONNECT} + timeout client ${TIMEOUT_CLIENT} + timeout server ${TIMEOUT_SERVER} + +EOF +} + +# The TLS-terminating frontend. $1 is the bind spec -- the one part the modes +# disagree about. +haproxy_emit_tls_frontend() { + local bind_spec="$1" + # "crt " loads all PEM files from the directory. + # ALPN is appended conditionally via ${ALPN:+ alpn ${ALPN}}. + cat <>/etc/haproxy/haproxy.cfg +frontend tls_in + bind ${bind_spec} ssl crt /etc/haproxy/certs/${ALPN:+ alpn ${ALPN}} +EOF + + if [ "$EVIDENCE_SERVER" = "true" ]; then + cat <<'EVIDENCE_BLOCK' >>/etc/haproxy/haproxy.cfg + + # Route /evidences requests to the local evidence HTTP server. + # accept fires once 16 bytes have arrived — enough for the + # longest prefix we match ("HEAD /evidences" = 16 chars). + # Using req.len with a concrete threshold is critical: the + # previous payload(0,0) (length 0 = "whole buffer") deferred + # evaluation until the full inspect-delay because HAProxy + # cannot know when a TCP stream ends. + tcp-request inspect-delay 5s + tcp-request content accept if { req.len ge 16 } + acl is_evidence payload(0,16) -m beg "GET /evidences" + acl is_evidence payload(0,16) -m beg "HEAD /evidences" + use_backend be_evidence if is_evidence +EVIDENCE_BLOCK + fi +} + +# Append the evidence backend block to haproxy.cfg +haproxy_emit_evidence_backend() { + if [ "$EVIDENCE_SERVER" = "true" ]; then + cat <>/etc/haproxy/haproxy.cfg + +backend be_evidence + mode http + http-request replace-path /evidences(.*) \1 + server evidence 127.0.0.1:${EVIDENCE_PORT} +EOF + fi +} + +# Generate haproxy.cfg for single-domain mode (DOMAIN + TARGET_ENDPOINT) +haproxy_emit_backends_single() { + local target_hostport + target_hostport=$(parse_target_endpoint "$TARGET_ENDPOINT") + + cat <>/etc/haproxy/haproxy.cfg + + default_backend be_upstream + +backend be_upstream + server app1 ${target_hostport} +EOF +} + +# Generate haproxy.cfg for multi-domain mode (ROUTING_MAP) +haproxy_emit_backends_multi() { + # Parse ROUTING_MAP and generate use_backend rules + backend sections + # Support both newline-separated and comma-separated formats + local routing_map_normalized + routing_map_normalized=$(echo "$ROUTING_MAP" | tr ',' '\n') + + local backend_rules="" + local backend_sections="" + local first_be_name="" + local domain target be_name + + while IFS= read -r line; do + [[ -n "$line" ]] || continue + [[ "$line" == \#* ]] && continue + domain="${line%%=*}" + target="${line#*=}" + domain=$(echo "$domain" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + target=$(echo "$target" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [[ -n "$domain" && -n "$target" ]] || continue + + # Validate domain and target to prevent config injection + if ! domain=$(sanitize_domain "$domain"); then + echo "Error: Invalid domain in ROUTING_MAP: ${line}" >&2 + exit 1 + fi + if ! target=$(sanitize_target_endpoint "$target"); then + echo "Error: Invalid target in ROUTING_MAP: ${line}" >&2 + exit 1 + fi + + # Strip protocol prefix from target if present + target=$(parse_target_endpoint "$target") + + # Generate safe backend name from domain + be_name="be_$(echo "$domain" | sed 's/[^A-Za-z0-9]/_/g')" + + if [ -z "$first_be_name" ]; then + first_be_name="$be_name" + fi + + backend_rules="${backend_rules} + use_backend ${be_name} if { ssl_fc_sni -i ${domain} }" + backend_sections="${backend_sections} + +backend ${be_name} + server s1 ${target}" + done <<< "$routing_map_normalized" + + echo "$backend_rules" >> /etc/haproxy/haproxy.cfg + + # Default to first backend in ROUTING_MAP + if [ -n "$first_be_name" ]; then + echo "" >> /etc/haproxy/haproxy.cfg + echo " default_backend ${first_be_name}" >> /etc/haproxy/haproxy.cfg + fi + + echo "$backend_sections" >> /etc/haproxy/haproxy.cfg +} + +# Pick the backend layout from the environment, then append the evidence backend. +haproxy_emit_backends() { + if [ -n "${ROUTING_MAP:-}" ]; then + haproxy_emit_backends_multi + elif [ -n "${DOMAIN:-}" ] && [ -n "${TARGET_ENDPOINT:-}" ]; then + haproxy_emit_backends_single + fi + haproxy_emit_evidence_backend +} + +haproxy_reload() { + if [ ! -f /var/run/haproxy/haproxy.pid ]; then + # Normal during a startup pass: haproxy has not been exec'd yet and will + # read the fresh certificates when it starts. + echo "HAProxy is not running yet; certificates will be picked up at start" + return 0 + fi + if kill -USR2 "$(cat /var/run/haproxy/haproxy.pid)"; then + echo "HAProxy reloaded with the new certificates" + else + echo "HAProxy reload failed: SIGUSR2 to PID $(cat /var/run/haproxy/haproxy.pid) failed" >&2 + return 1 + fi +} diff --git a/custom-domain/dstack-ingress/scripts/legoman.py b/custom-domain/dstack-ingress/scripts/legoman.py index 92b6f562..dd01c76a 100644 --- a/custom-domain/dstack-ingress/scripts/legoman.py +++ b/custom-domain/dstack-ingress/scripts/legoman.py @@ -37,7 +37,7 @@ ACME_PROD = "https://acme-v02.api.letsencrypt.org/directory" ACME_STAGING = "https://acme-staging-v02.api.letsencrypt.org/directory" -# Exit codes, matching the protocol renew-certificate.sh expects. +# Exit codes: 0 changed, 2 nothing to do, 1 failed. The caller reloads on 0. EXIT_CHANGED = 0 EXIT_ERROR = 1 EXIT_UNCHANGED = 2 diff --git a/custom-domain/dstack-ingress/scripts/renew-certificate.sh b/custom-domain/dstack-ingress/scripts/renew-certificate.sh deleted file mode 100755 index aa746ef4..00000000 --- a/custom-domain/dstack-ingress/scripts/renew-certificate.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -source /opt/app-venv/bin/activate - -DOMAIN=$1 - -SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" - -# dns-01 runs certbot with a DNS provider plugin. tls-alpn-01 runs lego: certbot -# never implemented the challenge and the acme library removed it in 4.2.0. -if [ "${CHALLENGE_TYPE:-dns-01}" = "tls-alpn-01" ]; then - python3 "$SCRIPT_DIR/legoman.py" auto --domain "$DOMAIN" --email "$CERTBOT_EMAIL" -else - python3 "$SCRIPT_DIR/certman.py" auto --domain "$DOMAIN" --email "$CERTBOT_EMAIL" -fi -CERT_STATUS=$? - -if [ $CERT_STATUS -eq 1 ]; then - echo "Certificate management failed" >&2 -elif [ $CERT_STATUS -eq 2 ]; then - echo "No certificates need renewal, skipping evidence generation" -fi - -# Propagate the tri-state: 0 = certificate changed, 2 = nothing to do, 1 = -# failure. This used to always exit 0, which told the caller "something -# changed" on every single pass -- so evidences were regenerated and haproxy -# reloaded every 12 hours whether or not a certificate had moved. -exit $CERT_STATUS \ No newline at end of file diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh new file mode 100644 index 00000000..e0d12fc6 --- /dev/null +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -0,0 +1,308 @@ +#!/bin/bash +# tlsalpn.sh - certificates via lego and the TLS-ALPN-01 challenge, with no DNS +# credentials in the container. +# +# entrypoint.sh validates the shared settings and then execs this file. There is +# no shared orchestration above it and no interface it has to implement -- this +# is simply the program that runs for tls-alpn-01, top to bottom. + +set -e + +source /scripts/functions.sh +source /scripts/haproxy-lib.sh +source /scripts/evidence-lib.sh + +LEGO_BIN=${LEGO_BIN:-/usr/local/bin/lego} +LEGO_PATH=${LEGO_PATH:-/etc/letsencrypt/lego} +TLSALPN_ADDRESS=${TLSALPN_ADDRESS:-127.0.0.1} +TLSALPN_PORT=${TLSALPN_PORT:-9443} +TLS_TERMINATE_PORT=${TLS_TERMINATE_PORT:-9444} +DNS_SETUP_MODE=${DNS_SETUP_MODE:-wait} +DNS_SETTLE_SECONDS=${DNS_SETTLE_SECONDS:-30} +export LEGO_BIN LEGO_PATH TLSALPN_ADDRESS TLSALPN_PORT DNS_SETUP_MODE + +case "$DNS_SETUP_MODE" in + wait|print|webhook) ;; + *) + echo "Error: invalid DNS_SETUP_MODE '$DNS_SETUP_MODE' (expected wait, print or webhook)" >&2 + exit 1 + ;; +esac + +# lego stores certificates under $LEGO_PATH/certificates/.{crt,key}; +# the .crt already holds the full chain. +cert_fullchain_path() { echo "${LEGO_PATH}/certificates/${1}.crt"; } +cert_privkey_path() { echo "${LEGO_PATH}/certificates/${1}.key"; } + +acme_account_file() { + local files=( "${LEGO_PATH}"/accounts/*/*/account.json ) + if [[ ! -f "${files[0]}" ]]; then + echo "Error: lego account file not found under ${LEGO_PATH}/accounts" >&2 + return 1 + fi + echo "${files[0]}" +} + +setup_py_env() { + if [ ! -d /opt/app-venv ]; then + echo "Creating application virtual environment" + python3 -m venv --system-site-packages /opt/app-venv + fi + # shellcheck disable=SC1091 + source /opt/app-venv/bin/activate + + if [ ! -f /.venv_bootstrapped ]; then + # lego handles ACME here, so certbot and the cloud SDKs are dead weight. + echo "Bootstrapping python dependencies (no certbot on this path)" + pip install --upgrade pip + pip install requests + touch /.venv_bootstrapped + fi + echo 'source /opt/app-venv/bin/activate' > /etc/profile.d/app-venv.sh +} + +check_lego() { + if [ ! -x "$LEGO_BIN" ]; then + echo "Error: lego binary not found at $LEGO_BIN" >&2 + exit 1 + fi + echo "Using lego for tls-alpn-01: $($LEGO_BIN --version)" +} + +require_instance_id() { + if [ -z "${DSTACK_INSTANCE_ID:-}" ] || [ "$DSTACK_INSTANCE_ID" = "null" ]; then + echo "Error: could not read instance_id from the dstack guest agent, which" >&2 + echo "tls-alpn-01 needs to pin gateway routing to this instance" >&2 + exit 1 + fi +} + +# What the gateway should route this hostname to. +# +# dns-01 uses the app id, so the gateway load-balances across every instance of +# the app. tls-alpn-01 cannot: the challenge is answered by whichever instance +# lego runs on, and routing by app id makes the gateway race connections across +# the app's instances while the CA validates from several vantage points at +# once, so the challenge would land on the wrong replica. Pinning to the +# instance id makes validation deterministic -- at the cost of sending all +# traffic to this one instance, so this mode is effectively single-instance. +txt_record_value() { echo "${DSTACK_INSTANCE_ID}:${PORT}"; } + +# haproxy owns the public port, but tls-alpn-01 needs the ACME responder to +# complete the TLS handshake itself. So the public port is a plain TCP frontend +# that peeks at the ClientHello and forwards only acme-tls/1 to lego; everything +# else goes to the real TLS frontend on loopback. PROXY protocol carries the +# client address across that hop so the backend still sees the real peer. +emit_peek_frontend() { + cat <>/etc/haproxy/haproxy.cfg +frontend tls_peek + bind :${PORT} + tcp-request inspect-delay 5s + tcp-request content accept if { req.ssl_hello_type 1 } + # Only the CA sends this ALPN protocol. A client that sends it anyway just + # reaches lego's responder, which refuses anything but acme-tls/1. + use_backend be_acme if { req.ssl_alpn -i acme-tls/1 } + default_backend be_tls_terminate + +backend be_acme + server lego ${TLSALPN_ADDRESS}:${TLSALPN_PORT} + +backend be_tls_terminate + server local 127.0.0.1:${TLS_TERMINATE_PORT} send-proxy-v2 + +EOF +} + +# haproxy refuses to start when the crt directory has no PEM in it, and here it +# has to be listening before the first certificate can exist -- it is what +# forwards the challenge to lego. Seed a placeholder the real cert overwrites. +# haproxy refuses to start when the crt directory has no PEM in it. In +# tls-alpn-01 mode the proxy has to be listening *before* the first certificate +# exists, because it is what forwards the ACME challenge to certbot, so seed a +# self-signed placeholder that the real certificate overwrites later. +ensure_placeholder_certs() { + local all_domains domain pem + all_domains=$(get-all-domains.sh) + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + pem="/etc/haproxy/certs/${domain}.pem" + [ -f "$pem" ] && continue + echo "Generating placeholder certificate for ${domain}" + openssl req -x509 -newkey rsa:2048 -nodes -days 1 \ + -keyout /tmp/placeholder.key -out /tmp/placeholder.crt \ + -subj "/CN=${domain#\*.}" >/dev/null 2>&1 + cat /tmp/placeholder.crt /tmp/placeholder.key >"$pem" + chmod 600 "$pem" + rm -f /tmp/placeholder.key /tmp/placeholder.crt + done <<<"$all_domains" +} + +build_combined_pems() { + local domain fullchain privkey combined + mkdir -p /etc/haproxy/certs + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + fullchain=$(cert_fullchain_path "$domain") + privkey=$(cert_privkey_path "$domain") + combined="/etc/haproxy/certs/${domain}.pem" + if [ -f "$fullchain" ] && [ -f "$privkey" ]; then + cat "$fullchain" "$privkey" > "$combined" + chmod 600 "$combined" + echo "Combined PEM created: ${combined}" + else + echo "Warning: Cert files missing for ${domain}, skipping" + fi + done <<<"$(get-all-domains.sh)" +} + +collect_evidence() { + local account domain + if ! account=$(acme_account_file); then + echo "Error: cannot generate evidence without the ACME account file" >&2 + return 1 + fi + evidence_reset + evidence_collect_account "$account" + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + evidence_collect_cert "$domain" "$(cert_fullchain_path "$domain")" + done <<<"$(get-all-domains.sh)" + evidence_finalize +} + +process_domain() { + local domain="$1" + + if [[ "$domain" == \*.* ]]; then + echo "Error: cannot issue a wildcard certificate for $domain with tls-alpn-01." >&2 + echo "RFC 8737 forbids it; use CHALLENGE_TYPE=dns-01 for wildcards." >&2 + exit 1 + fi + + # Register the ACME account first so the CAA record we print can already + # pin accounturi. Without this the operator would have to add CAA in a + # second pass, after the account exists. + if ! legoman.py register --email "$CERTBOT_EMAIL"; then + echo "Error: failed to register the ACME account" >&2 + exit 1 + fi + + local account_uri caa_value + account_uri=$(legoman.py account-uri --email "$CERTBOT_EMAIL" 2>/dev/null || true) + if [ -n "$account_uri" ]; then + caa_value="letsencrypt.org;validationmethods=tls-alpn-01;accounturi=$account_uri" + else + echo "Warning: could not read the ACME account URI; the CAA record will not pin it" >&2 + caa_value="letsencrypt.org;validationmethods=tls-alpn-01" + fi + + # Wait for the records routing depends on. An existing CAA record is also + # checked for compatibility -- not because CAA is required (an absent CAA + # record set permits every CA), but because an incompatible one makes the CA + # refuse and burns Let's Encrypt's failed-validation budget: 5 per account + # per hostname per hour. + if ! dnsguide.py \ + --domain "$domain" \ + --alias-target "$GATEWAY_DOMAIN" \ + --txt-name "$(txt_record_name "$domain")" \ + --txt-value "$(txt_record_value)" \ + --caa-name "${domain#\*.}" \ + --caa-tag "$(caa_tag_for "$domain")" \ + --caa-value "$caa_value" \ + --account-uri "$account_uri" \ + --challenge tls-alpn-01 \ + --mode "$DNS_SETUP_MODE"; then + echo "Error: required DNS records for $domain are not in place" >&2 + exit 1 + fi + + # Public DNS being correct is not the same as the gateway acting on it. The + # gateway resolves the app-address TXT itself and caches the answer for the + # record's TTL, so right after the value changes -- which is every time the + # CVM instance is replaced, since the record names the instance -- it can + # still route the challenge to the previous target. Observed exactly that: + # dnsguide passed, then the CA got "Error getting validation data" because + # the gateway was still sending it to the old instance. + # Only on the first pass. The settle wait exists because the TXT value has + # just changed -- true at startup, and after an instance replacement, which + # is a fresh container and therefore also a first pass. Renewal passes reuse + # a record the gateway resolved long ago. + if [ "$FIRST_PASS" = "true" ] && [ "${DNS_SETTLE_SECONDS}" -gt 0 ]; then + echo "Waiting ${DNS_SETTLE_SECONDS}s for the gateway's DNS cache to expire" + sleep "${DNS_SETTLE_SECONDS}" + fi + + legoman.py auto --domain "$domain" --email "$CERTBOT_EMAIL" +} + +# One pass over every domain: make sure the records exist, then issue or renew. +# Idempotent, so the first pass and every later one are the same code. There is +# deliberately only one of these -- an earlier split between a one-shot +# bootstrap and a renewal daemon raced for lego's challenge port, and the loser +# still spent one of Let's Encrypt's five failed validations per hostname/hour. +FIRST_PASS=true + +run_pass() { + local domain status failed=0 changed=0 + while IFS= read -r domain; do + [[ -n "$domain" ]] || continue + # `if` context, not a bare command: process_domain returns 2 for + # "nothing to do", and under `set -e` a bare non-zero would kill the + # whole script on every quiet renewal pass. + if ( process_domain "$domain" ); then status=0; else status=$?; fi + case $status in + 0) changed=1 ;; + 2) ;; + *) echo "Certificate management failed for $domain" >&2; failed=1 ;; + esac + done <<<"$(get-all-domains.sh)" + + if [ "$changed" -eq 1 ]; then + collect_evidence || echo "Evidence generation failed" >&2 + build_combined_pems || echo "Combined PEM build failed" >&2 + haproxy_reload || true + fi + FIRST_PASS=false + [ "$failed" -eq 0 ] +} + +cert_loop() { + local attempt=0 delay + while true; do + if run_pass; then + attempt=0 + delay=${RENEW_INTERVAL:-43200} + else + # The normal state here is "waiting for an operator to create a DNS + # record", so falling back to the 12-hour interval would be a + # terrible retry rate. + attempt=$((attempt + 1)) + delay=$((60 * attempt)) + [ "$delay" -gt 1800 ] && delay=1800 + echo "[$(date)] Pass failed (attempt ${attempt}); retrying in ${delay}s" + fi + echo "[$(date)] Next certificate check in ${delay}s" + sleep "$delay" + done +} + +setup_py_env +check_lego +load_dstack_identity +require_instance_id + +# Ordering is load-bearing: haproxy has to be forwarding acme-tls/1 before the +# first certificate can exist, so start on a placeholder and let the loop +# converge behind it. +ensure_placeholder_certs +build_combined_pems + +haproxy_emit_global +emit_peek_frontend +haproxy_emit_tls_frontend "127.0.0.1:${TLS_TERMINATE_PORT} accept-proxy" +haproxy_emit_backends + +evidence_start_server +cert_loop & + +exec "$@" From 6bc8a86a878d48c9aa4944497b3849fba571088e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 21:18:27 -0700 Subject: [PATCH 06/16] fix(dstack-ingress): let lego report renewals instead of inferring them --- .../dstack-ingress/scripts/legoman.py | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/custom-domain/dstack-ingress/scripts/legoman.py b/custom-domain/dstack-ingress/scripts/legoman.py index dd01c76a..bfb97906 100644 --- a/custom-domain/dstack-ingress/scripts/legoman.py +++ b/custom-domain/dstack-ingress/scripts/legoman.py @@ -90,6 +90,10 @@ def certificate_exists(domain: str) -> bool: return os.path.isfile(crt) and os.path.isfile(key) +def _renewed_marker(domain: str) -> str: + return os.path.join(LEGO_PATH, f".renewed-{domain}") + + def _common_flags(email: str) -> List[str]: return [ "--path", @@ -153,19 +157,33 @@ def run_cert(domain: str, email: str) -> int: ) return EXIT_ERROR - existed = certificate_exists(domain) + # Let lego decide whether a renewal is due: it consults the CA's ARI + # endpoint (RFC 9773) as well as the local window, and the CA can ask for + # early renewal during an incident. Deciding here from notAfter would + # override that. + # + # We only need to know whether it acted, and lego says so itself: + # --deploy-hook runs "in cases where a certificate is successfully + # created/renewed" and stays silent otherwise. That beats inferring it -- + # matching log text is not stable (4.x wrote "no renewal", 5.x writes + # "Skip renewal"), and the exit code is 0 either way. + marker = _renewed_marker(domain) + if os.path.exists(marker): + os.remove(marker) + cmd = ( [LEGO_BIN, "run"] + _common_flags(email) + ["--domains", domain] + _challenge_flags() + + ["--deploy-hook", f"touch {marker}"] ) renew_days = os.environ.get("RENEW_DAYS_BEFORE", "") if renew_days: cmd += ["--renew-days", renew_days] - print(f"{'Renewing' if existed else 'Obtaining'} certificate for {domain} via tls-alpn-01") - code, output = _run(cmd) + print(f"{'Renewing' if certificate_exists(domain) else 'Obtaining'} certificate for {domain} via tls-alpn-01") + code, _ = _run(cmd) if code != 0: print(f"✗ lego failed for {domain} (exit code {code})", file=sys.stderr) return EXIT_ERROR @@ -173,12 +191,10 @@ def run_cert(domain: str, email: str) -> int: print(f"✗ lego reported success but no certificate for {domain}", file=sys.stderr) return EXIT_ERROR - # lego exits 0 whether or not it actually replaced anything, so the log line - # is the only signal that a renewal was skipped. - lowered = output.lower() - if existed and ("no renewal" in lowered or "not yet due for renewal" in lowered): - print("No certificates need renewal") + if not os.path.exists(marker): + print(f"Certificate for {domain} is still current; nothing to do") return EXIT_UNCHANGED + os.remove(marker) print(f"✓ Certificate ready for {domain}") return EXIT_CHANGED From 7413f43cb73073a0d10e5e233390fbf43fb957f1 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 22:17:45 -0700 Subject: [PATCH 07/16] docs(dstack-ingress): name the ACME account settings after ACME, not certbot --- custom-domain/dstack-ingress/README.md | 9 ++++++--- .../dstack-ingress/scripts/legoman.py | 13 +++++++------ .../dstack-ingress/scripts/tlsalpn.sh | 18 +++++++++++++++--- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index 8c76f59c..251d3b26 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -159,7 +159,7 @@ environment: | `DOMAIN` | Your domain (single-domain mode). Supports wildcards (`*.example.com`) | | `TARGET_ENDPOINT` | Backend address, e.g. `app:80` or `http://app:80` | | `GATEWAY_DOMAIN` | dstack gateway domain (e.g. `_.dstack-prod5.phala.network`) | -| `CERTBOT_EMAIL` | Email for Let's Encrypt registration | +| `CERTBOT_EMAIL` | Email for Let's Encrypt registration. In tls-alpn-01 mode the preferred name is `ACME_EMAIL` | | `DNS_PROVIDER` | DNS provider (`cloudflare`, `linode`, `namecheap`) | ### Optional @@ -171,7 +171,8 @@ environment: | `ROUTING_MAP` | | Multi-domain routing: `domain=host:port` per line | | `SET_CAA` | `false` | Enable CAA DNS record (dns-01 only; tls-alpn-01 cannot write DNS) | | `TXT_PREFIX` | `_dstack-app-address` | DNS TXT record prefix | -| `CERTBOT_STAGING` | `false` | Use Let's Encrypt staging server | +| `CERTBOT_STAGING` | `false` | Use Let's Encrypt staging server. `ACME_STAGING` is the preferred name in tls-alpn-01 mode | +| `ACME_EMAIL` | | ACME account email in tls-alpn-01 mode. Falls back to `CERTBOT_EMAIL`; there is no certbot on that path | | `CHALLENGE_TYPE` | `dns-01` | `dns-01` (certbot + DNS credentials) or `tls-alpn-01` (lego, no DNS credentials) | | `DNS_SETUP_MODE` | `wait` | tls-alpn-01 only: `wait`, `print` or `webhook` — see below | | `DNS_SETUP_TIMEOUT` | `1800` | tls-alpn-01 only: seconds to wait for the records to appear | @@ -294,8 +295,10 @@ services: - CHALLENGE_TYPE=tls-alpn-01 - DOMAIN=app.example.com - TARGET_ENDPOINT=http://app:80 + # Printed as the CNAME target, and used to verify that the hostname + # really resolves to the gateway before issuance starts. - GATEWAY_DOMAIN=_.dstack-prod5.phala.network - - CERTBOT_EMAIL=you@example.com + - ACME_EMAIL=you@example.com # - DNS_SETUP_MODE=wait # default; blocks until the records exist ports: - "443:443" diff --git a/custom-domain/dstack-ingress/scripts/legoman.py b/custom-domain/dstack-ingress/scripts/legoman.py index bfb97906..9bb32495 100644 --- a/custom-domain/dstack-ingress/scripts/legoman.py +++ b/custom-domain/dstack-ingress/scripts/legoman.py @@ -47,9 +47,10 @@ def acme_server() -> str: - if os.environ.get("CERTBOT_STAGING", "false") == "true": - return ACME_STAGING - return ACME_PROD + # CERTBOT_STAGING is the historical name and still works, but there is no + # certbot on this path: the ACME client here is lego. + staging = os.environ.get("ACME_STAGING") or os.environ.get("CERTBOT_STAGING", "false") + return ACME_STAGING if staging == "true" else ACME_PROD def server_dir() -> str: @@ -210,11 +211,11 @@ def main() -> int: parser.add_argument("--email", help="Email for ACME registration") args = parser.parse_args() - email = args.email or os.environ.get("CERTBOT_EMAIL", "") + email = args.email or os.environ.get("ACME_EMAIL") or os.environ.get("CERTBOT_EMAIL", "") if args.action == "account-uri": if not email: - print("Error: --email or CERTBOT_EMAIL is required", file=sys.stderr) + print("Error: --email or ACME_EMAIL is required", file=sys.stderr) return EXIT_ERROR uri = account_uri(email) if not uri: @@ -231,7 +232,7 @@ def main() -> int: return EXIT_CHANGED if not email: - print("Error: --email or CERTBOT_EMAIL is required", file=sys.stderr) + print("Error: --email or ACME_EMAIL is required", file=sys.stderr) return EXIT_ERROR if not os.path.isfile(LEGO_BIN): print(f"Error: lego binary not found at {LEGO_BIN}", file=sys.stderr) diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh index e0d12fc6..b05f8be9 100644 --- a/custom-domain/dstack-ingress/scripts/tlsalpn.sh +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -12,6 +12,18 @@ source /scripts/functions.sh source /scripts/haproxy-lib.sh source /scripts/evidence-lib.sh +# ACME account settings. CERTBOT_EMAIL / CERTBOT_STAGING are the historical +# names, kept working for anyone migrating a compose file from dns-01, but there +# is no certbot on this path -- the ACME client here is lego. +ACME_EMAIL=${ACME_EMAIL:-${CERTBOT_EMAIL:-}} +ACME_STAGING=${ACME_STAGING:-${CERTBOT_STAGING:-false}} +export ACME_EMAIL ACME_STAGING + +if [ -z "$ACME_EMAIL" ]; then + echo "Error: ACME_EMAIL must be set (CERTBOT_EMAIL is accepted as an alias)" >&2 + exit 1 +fi + LEGO_BIN=${LEGO_BIN:-/usr/local/bin/lego} LEGO_PATH=${LEGO_PATH:-/etc/letsencrypt/lego} TLSALPN_ADDRESS=${TLSALPN_ADDRESS:-127.0.0.1} @@ -182,13 +194,13 @@ process_domain() { # Register the ACME account first so the CAA record we print can already # pin accounturi. Without this the operator would have to add CAA in a # second pass, after the account exists. - if ! legoman.py register --email "$CERTBOT_EMAIL"; then + if ! legoman.py register --email "$ACME_EMAIL"; then echo "Error: failed to register the ACME account" >&2 exit 1 fi local account_uri caa_value - account_uri=$(legoman.py account-uri --email "$CERTBOT_EMAIL" 2>/dev/null || true) + account_uri=$(legoman.py account-uri --email "$ACME_EMAIL" 2>/dev/null || true) if [ -n "$account_uri" ]; then caa_value="letsencrypt.org;validationmethods=tls-alpn-01;accounturi=$account_uri" else @@ -232,7 +244,7 @@ process_domain() { sleep "${DNS_SETTLE_SECONDS}" fi - legoman.py auto --domain "$domain" --email "$CERTBOT_EMAIL" + legoman.py auto --domain "$domain" --email "$ACME_EMAIL" } # One pass over every domain: make sure the records exist, then issue or renew. From 2306f2cab76c9357a2e4e9a4970f88bf3057d5c5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 22:27:35 -0700 Subject: [PATCH 08/16] feat(dstack-ingress): make the ACME contact address optional --- custom-domain/dstack-ingress/README.md | 17 +++++-- .../dstack-ingress/scripts/legoman.py | 47 ++++++++++--------- .../dstack-ingress/scripts/tlsalpn.sh | 16 +++---- 3 files changed, 46 insertions(+), 34 deletions(-) diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index 251d3b26..5af2cc95 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -159,7 +159,7 @@ environment: | `DOMAIN` | Your domain (single-domain mode). Supports wildcards (`*.example.com`) | | `TARGET_ENDPOINT` | Backend address, e.g. `app:80` or `http://app:80` | | `GATEWAY_DOMAIN` | dstack gateway domain (e.g. `_.dstack-prod5.phala.network`) | -| `CERTBOT_EMAIL` | Email for Let's Encrypt registration. In tls-alpn-01 mode the preferred name is `ACME_EMAIL` | +| `CERTBOT_EMAIL` | Email for Let's Encrypt registration. Not required in tls-alpn-01 mode, where the preferred name is `ACME_EMAIL` | | `DNS_PROVIDER` | DNS provider (`cloudflare`, `linode`, `namecheap`) | ### Optional @@ -172,7 +172,7 @@ environment: | `SET_CAA` | `false` | Enable CAA DNS record (dns-01 only; tls-alpn-01 cannot write DNS) | | `TXT_PREFIX` | `_dstack-app-address` | DNS TXT record prefix | | `CERTBOT_STAGING` | `false` | Use Let's Encrypt staging server. `ACME_STAGING` is the preferred name in tls-alpn-01 mode | -| `ACME_EMAIL` | | ACME account email in tls-alpn-01 mode. Falls back to `CERTBOT_EMAIL`; there is no certbot on that path | +| `ACME_EMAIL` | | Optional ACME contact address in tls-alpn-01 mode. Falls back to `CERTBOT_EMAIL`; there is no certbot on that path | | `CHALLENGE_TYPE` | `dns-01` | `dns-01` (certbot + DNS credentials) or `tls-alpn-01` (lego, no DNS credentials) | | `DNS_SETUP_MODE` | `wait` | tls-alpn-01 only: `wait`, `print` or `webhook` — see below | | `DNS_SETUP_TIMEOUT` | `1800` | tls-alpn-01 only: seconds to wait for the records to appear | @@ -298,7 +298,7 @@ services: # Printed as the CNAME target, and used to verify that the hostname # really resolves to the gateway before issuance starts. - GATEWAY_DOMAIN=_.dstack-prod5.phala.network - - ACME_EMAIL=you@example.com + # - ACME_EMAIL=you@example.com # optional, and published (see below) # - DNS_SETUP_MODE=wait # default; blocks until the records exist ports: - "443:443" @@ -348,6 +348,17 @@ Two consequences: means updating DNS. `DNS_SETUP_MODE=webhook` exists so this can be automated; doing it by hand means downtime on every redeploy. +### The ACME contact address is optional, and public + +`ACME_EMAIL` may be left unset. RFC 8555 makes the ACME `contact` field +optional, and Let's Encrypt stopped sending expiry notification mail in 2025, so +setting one buys little. + +It also does not stay private. The ACME account document is published as +attestation evidence at `/evidences/acme-account.json`, so an address set here +is readable by anyone who fetches the evidence endpoint. Leave it unset unless +you specifically want a contact on the account. + ### Limitations - **No wildcards.** RFC 8737 forbids tls-alpn-01 for wildcard identifiers, and diff --git a/custom-domain/dstack-ingress/scripts/legoman.py b/custom-domain/dstack-ingress/scripts/legoman.py index 9bb32495..2880803f 100644 --- a/custom-domain/dstack-ingress/scripts/legoman.py +++ b/custom-domain/dstack-ingress/scripts/legoman.py @@ -25,6 +25,7 @@ """ import argparse +import glob import json import os import subprocess @@ -60,13 +61,20 @@ def server_dir() -> str: return f"{host}_{parsed.port}" if parsed.port else host -def account_file(email: str) -> Optional[str]: - path = os.path.join(LEGO_PATH, "accounts", server_dir(), email, "account.json") - return path if os.path.isfile(path) else None +def account_file() -> Optional[str]: + """Find the account document by globbing rather than by building the path. + lego names the directory after the account email, and after a placeholder of + its own choosing when there is no email. Globbing means we neither have to + know that placeholder nor track it across lego versions. + """ + pattern = os.path.join(LEGO_PATH, "accounts", server_dir(), "*", "account.json") + matches = sorted(glob.glob(pattern)) + return matches[0] if matches else None -def account_uri(email: str) -> Optional[str]: - path = account_file(email) + +def account_uri() -> Optional[str]: + path = account_file() if not path: return None try: @@ -96,15 +104,14 @@ def _renewed_marker(domain: str) -> str: def _common_flags(email: str) -> List[str]: - return [ - "--path", - LEGO_PATH, - "--server", - acme_server(), - "--email", - email, - "--accept-tos", - ] + flags = ["--path", LEGO_PATH, "--server", acme_server(), "--accept-tos"] + # The ACME contact address is optional (RFC 8555 §7.3), and Let's Encrypt + # stopped sending expiry notifications in 2025, so it buys little. It is + # also published: the account document is served as evidence, so an address + # set here becomes public. Omit it unless the operator wants it. + if email: + flags += ["--email", email] + return flags def _run(cmd: List[str], timeout: int = RUN_TIMEOUT) -> Tuple[int, str]: @@ -126,7 +133,7 @@ def _run(cmd: List[str], timeout: int = RUN_TIMEOUT) -> Tuple[int, str]: def register(email: str) -> int: """Create the ACME account without issuing anything.""" - if account_file(email): + if account_file(): print("ACME account already exists") return EXIT_UNCHANGED @@ -135,7 +142,7 @@ def register(email: str) -> int: [LEGO_BIN, "accounts", "register"] + _common_flags(email), timeout=REGISTER_TIMEOUT, ) - if code == 0 and account_file(email): + if code == 0 and account_file(): print("✓ ACME account registered") return EXIT_CHANGED print(f"✗ ACME account registration failed (exit code {code})", file=sys.stderr) @@ -214,10 +221,7 @@ def main() -> int: email = args.email or os.environ.get("ACME_EMAIL") or os.environ.get("CERTBOT_EMAIL", "") if args.action == "account-uri": - if not email: - print("Error: --email or ACME_EMAIL is required", file=sys.stderr) - return EXIT_ERROR - uri = account_uri(email) + uri = account_uri() if not uri: return EXIT_ERROR print(uri) @@ -231,9 +235,6 @@ def main() -> int: print(f"{crt} {key}") return EXIT_CHANGED - if not email: - print("Error: --email or ACME_EMAIL is required", file=sys.stderr) - return EXIT_ERROR if not os.path.isfile(LEGO_BIN): print(f"Error: lego binary not found at {LEGO_BIN}", file=sys.stderr) return EXIT_ERROR diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh index b05f8be9..9c12d882 100644 --- a/custom-domain/dstack-ingress/scripts/tlsalpn.sh +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -15,15 +15,15 @@ source /scripts/evidence-lib.sh # ACME account settings. CERTBOT_EMAIL / CERTBOT_STAGING are the historical # names, kept working for anyone migrating a compose file from dns-01, but there # is no certbot on this path -- the ACME client here is lego. +# +# The contact address is optional. RFC 8555 makes it optional, Let's Encrypt +# stopped sending expiry notifications in 2025, and it does not stay private: +# the account document is published at /evidences/acme-account.json, so an +# address set here is readable by anyone who fetches the evidence. ACME_EMAIL=${ACME_EMAIL:-${CERTBOT_EMAIL:-}} ACME_STAGING=${ACME_STAGING:-${CERTBOT_STAGING:-false}} export ACME_EMAIL ACME_STAGING -if [ -z "$ACME_EMAIL" ]; then - echo "Error: ACME_EMAIL must be set (CERTBOT_EMAIL is accepted as an alias)" >&2 - exit 1 -fi - LEGO_BIN=${LEGO_BIN:-/usr/local/bin/lego} LEGO_PATH=${LEGO_PATH:-/etc/letsencrypt/lego} TLSALPN_ADDRESS=${TLSALPN_ADDRESS:-127.0.0.1} @@ -194,13 +194,13 @@ process_domain() { # Register the ACME account first so the CAA record we print can already # pin accounturi. Without this the operator would have to add CAA in a # second pass, after the account exists. - if ! legoman.py register --email "$ACME_EMAIL"; then + if ! legoman.py register ${ACME_EMAIL:+--email "$ACME_EMAIL"}; then echo "Error: failed to register the ACME account" >&2 exit 1 fi local account_uri caa_value - account_uri=$(legoman.py account-uri --email "$ACME_EMAIL" 2>/dev/null || true) + account_uri=$(legoman.py account-uri 2>/dev/null || true) if [ -n "$account_uri" ]; then caa_value="letsencrypt.org;validationmethods=tls-alpn-01;accounturi=$account_uri" else @@ -244,7 +244,7 @@ process_domain() { sleep "${DNS_SETTLE_SECONDS}" fi - legoman.py auto --domain "$domain" --email "$ACME_EMAIL" + legoman.py auto --domain "$domain" ${ACME_EMAIL:+--email "$ACME_EMAIL"} } # One pass over every domain: make sure the records exist, then issue or renew. From 4ff3fa781c6baaff95af9da38aafa6301608a140 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 22:46:14 -0700 Subject: [PATCH 09/16] fix(dstack-ingress): count either issuance attempt as a change --- custom-domain/dstack-ingress/README.md | 22 +++++++----- .../dstack-ingress/scripts/certman.py | 28 +++++++++------ custom-domain/dstack-ingress/scripts/dns01.sh | 34 ++++++++++++++++--- 3 files changed, 59 insertions(+), 25 deletions(-) diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index 5af2cc95..55346db4 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -159,7 +159,7 @@ environment: | `DOMAIN` | Your domain (single-domain mode). Supports wildcards (`*.example.com`) | | `TARGET_ENDPOINT` | Backend address, e.g. `app:80` or `http://app:80` | | `GATEWAY_DOMAIN` | dstack gateway domain (e.g. `_.dstack-prod5.phala.network`) | -| `CERTBOT_EMAIL` | Email for Let's Encrypt registration. Not required in tls-alpn-01 mode, where the preferred name is `ACME_EMAIL` | +| `CERTBOT_EMAIL` | *(optional)* ACME contact address. `ACME_EMAIL` is accepted too, and is the preferred name in tls-alpn-01 mode | | `DNS_PROVIDER` | DNS provider (`cloudflare`, `linode`, `namecheap`) | ### Optional @@ -172,7 +172,7 @@ environment: | `SET_CAA` | `false` | Enable CAA DNS record (dns-01 only; tls-alpn-01 cannot write DNS) | | `TXT_PREFIX` | `_dstack-app-address` | DNS TXT record prefix | | `CERTBOT_STAGING` | `false` | Use Let's Encrypt staging server. `ACME_STAGING` is the preferred name in tls-alpn-01 mode | -| `ACME_EMAIL` | | Optional ACME contact address in tls-alpn-01 mode. Falls back to `CERTBOT_EMAIL`; there is no certbot on that path | +| `ACME_EMAIL` | | ACME contact address, in either mode. Falls back to `CERTBOT_EMAIL`. Optional — see below | | `CHALLENGE_TYPE` | `dns-01` | `dns-01` (certbot + DNS credentials) or `tls-alpn-01` (lego, no DNS credentials) | | `DNS_SETUP_MODE` | `wait` | tls-alpn-01 only: `wait`, `print` or `webhook` — see below | | `DNS_SETUP_TIMEOUT` | `1800` | tls-alpn-01 only: seconds to wait for the records to appear | @@ -348,16 +348,20 @@ Two consequences: means updating DNS. `DNS_SETUP_MODE=webhook` exists so this can be automated; doing it by hand means downtime on every redeploy. -### The ACME contact address is optional, and public +## The ACME contact address is optional, and public -`ACME_EMAIL` may be left unset. RFC 8555 makes the ACME `contact` field -optional, and Let's Encrypt stopped sending expiry notification mail in 2025, so -setting one buys little. +`ACME_EMAIL` (or `CERTBOT_EMAIL`) may be left unset in **either** mode. RFC 8555 +makes the ACME `contact` field optional, and Let's Encrypt stopped sending expiry +notification mail in 2025, so setting one buys little. It also does not stay private. The ACME account document is published as -attestation evidence at `/evidences/acme-account.json`, so an address set here -is readable by anyone who fetches the evidence endpoint. Leave it unset unless -you specifically want a contact on the account. +attestation evidence at `/evidences/acme-account.json`, so an address set here is +readable by anyone who fetches the evidence endpoint. Leave it unset unless you +specifically want a contact on the account. + +Under the hood the two clients differ: lego simply omits the flag, while certbot +needs `--register-unsafely-without-email` — its "unsafely" naming predates Let's +Encrypt dropping expiry mail, and the container passes it for you. ### Limitations diff --git a/custom-domain/dstack-ingress/scripts/certman.py b/custom-domain/dstack-ingress/scripts/certman.py index d12527f8..b0c4b051 100644 --- a/custom-domain/dstack-ingress/scripts/certman.py +++ b/custom-domain/dstack-ingress/scripts/certman.py @@ -313,8 +313,18 @@ def _build_certbot_command(self, action: str, domain: str, email: str) -> List[s f"Credentials file does not exist: {credentials_file}") if action == "certonly": - base_cmd.extend(["--agree-tos", "--no-eff-email", - "--email", email, "-d", domain]) + base_cmd.extend(["--agree-tos", "--no-eff-email"]) + # The ACME contact address is optional (RFC 8555 section 7.3), and + # it is published: the account document is served as attestation + # evidence, so an address set here is readable by anyone who + # fetches it. certbot will not simply omit the flag, so ask for a + # contactless account explicitly. Its "unsafely" naming predates + # Let's Encrypt dropping expiry notification mail in 2025. + if email: + base_cmd.extend(["--email", email]) + else: + base_cmd.append("--register-unsafely-without-email") + base_cmd.extend(["-d", domain]) if os.environ.get("CERTBOT_STAGING", "false") == "true": base_cmd.extend(["--staging"]) @@ -528,15 +538,11 @@ def main(): ) sys.exit(1) - # Email is required for obtain and auto actions - if args.action in ["obtain", "auto"] and not args.email: - if not os.environ.get("CERTBOT_EMAIL"): - print( - "Error: --email is required or set CERTBOT_EMAIL environment variable", - file=sys.stderr, - ) - sys.exit(1) - args.email = os.environ["CERTBOT_EMAIL"] + # The contact address is optional; see _build_certbot_command. + if not args.email: + args.email = os.environ.get("ACME_EMAIL") or os.environ.get( + "CERTBOT_EMAIL", "" + ) success, needs_evidence = manager.run_action( args.domain, args.email, args.action diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh index 7f39e561..6ec61a15 100644 --- a/custom-domain/dstack-ingress/scripts/dns01.sh +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -11,6 +11,12 @@ source /scripts/functions.sh source /scripts/haproxy-lib.sh source /scripts/evidence-lib.sh +# ACME contact address. CERTBOT_EMAIL is the documented name on this path -- +# certbot really is the client here -- and ACME_EMAIL is accepted too so one +# variable works in either mode. Optional either way. +ACME_EMAIL=${ACME_EMAIL:-${CERTBOT_EMAIL:-}} +export ACME_EMAIL + # certbot stores certificates under /etc/letsencrypt/live//. cert_fullchain_path() { echo "/etc/letsencrypt/live/$(cert_dir_name "$1")/fullchain.pem"; } cert_privkey_path() { echo "/etc/letsencrypt/live/$(cert_dir_name "$1")/privkey.pem"; } @@ -136,20 +142,38 @@ set_caa_record() { } process_domain() { - local domain="$1" + local domain="$1" first status set_alias_record "$domain" set_txt_record "$domain" + # The CAA record names our ACME account, so the account has to exist first: - # try once (expected to fail on a fresh deployment), write CAA, try again. - issue_certificate "$domain" || echo "First certificate attempt failed for $domain, retrying after the CAA record is set" + # try once, write CAA, try again. The first attempt is expected to fail when + # a CAA record from an earlier account is still in place. + if issue_certificate "$domain"; then first=0; else first=$?; fi + if [ "$first" -eq 1 ]; then + echo "First certificate attempt failed for $domain, retrying after the CAA record is set" + fi + set_caa_record "$domain" - issue_certificate "$domain" + + if issue_certificate "$domain"; then status=0; else status=$?; fi + + # Either attempt producing a certificate counts as a change. Reporting only + # the second one hid the common case: on a clean first deployment attempt + # one obtains the certificate and attempt two reports "nothing to renew", + # so the pass looked quiet and skipped evidence generation entirely. + if [ "$first" -eq 0 ] || [ "$status" -eq 0 ]; then + return 0 + fi + return "$status" } issue_certificate() { source /opt/app-venv/bin/activate - certman.py auto --domain "$1" --email "$CERTBOT_EMAIL" + # The contact address is optional; certman.py asks for a contactless + # account when none is given. + certman.py auto --domain "$1" ${ACME_EMAIL:+--email "$ACME_EMAIL"} } build_combined_pems() { From 3a32f73be0a386f6a024e033177c5d8f8ce9f853 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 23:35:06 -0700 Subject: [PATCH 10/16] fix(dstack-ingress): stop dns-01 running the first pass twice dns-01 issues before it serves, so run_pass is called explicitly at startup and cert_loop then ran it again immediately. For a single domain that is four certbot invocations and two rounds of DNS writes on every container start, all to reach a state the first pass had already reached. The duplicate is inherited from upstream, where bootstrap and the renewal daemon both ran at startup with no initial sleep; the refactor kept it faithfully. tls-alpn-01 never had it because there is no bootstrap pass to duplicate. Also deduplicate CAA records after parsing rather than before. Resolvers disagree on the wire format -- Cloudflare returns RFC 3597 generic hex, Google the presentation form -- so one record arrives as two distinct strings and its rejection reason was printed twice. --- custom-domain/dstack-ingress/scripts/dns01.sh | 16 ++++++--- .../dstack-ingress/scripts/dnsguide.py | 10 +++++- .../scripts/tests/test_dnsguide.py | 35 +++++++++++++++++++ 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh index 6ec61a15..2e75d485 100644 --- a/custom-domain/dstack-ingress/scripts/dns01.sh +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -234,9 +234,17 @@ run_pass() { [ "$failed" -eq 0 ] } +# $1: seconds to wait before the first pass. dns-01 issues before it serves, so +# the caller has already run a pass by the time we get here; without this the +# loop would immediately repeat it -- two full passes, and for a single domain +# four certbot invocations plus a duplicate round of DNS writes, on every start. cert_loop() { - local attempt=0 delay + local attempt=0 delay="${1:-0}" while true; do + if [ "$delay" -gt 0 ]; then + echo "[$(date)] Next certificate check in ${delay}s" + sleep "$delay" + fi if run_pass; then attempt=0 delay=${RENEW_INTERVAL:-43200} @@ -244,10 +252,8 @@ cert_loop() { attempt=$((attempt + 1)) delay=$((60 * attempt)) [ "$delay" -gt 1800 ] && delay=1800 - echo "[$(date)] Pass failed (attempt ${attempt}); retrying in ${delay}s" + echo "[$(date)] Pass failed (attempt ${attempt}); retrying" >&2 fi - echo "[$(date)] Next certificate check in ${delay}s" - sleep "$delay" done } @@ -265,6 +271,6 @@ haproxy_emit_tls_frontend ":${PORT}" haproxy_emit_backends evidence_start_server -cert_loop & +cert_loop "${RENEW_INTERVAL:-43200}" & exec "$@" diff --git a/custom-domain/dstack-ingress/scripts/dnsguide.py b/custom-domain/dstack-ingress/scripts/dnsguide.py index 5c5a1747..7971e864 100644 --- a/custom-domain/dstack-ingress/scripts/dnsguide.py +++ b/custom-domain/dstack-ingress/scripts/dnsguide.py @@ -265,7 +265,15 @@ def check_caa( candidate = ".".join(labels[i:]) # Union: one resolver seeing a restriction is enough to honour it. rdatas = query_union(candidate, RR_CAA, resolvers) - parsed = [p for p in (_parse_caa(r) for r in rdatas) if p is not None] + # Deduplicate after parsing, not before: resolvers disagree on the wire + # format (one returns presentation form, another RFC 3597 generic hex), + # so the same record arrives as two distinct strings and would otherwise + # be reported, and counted, twice. + parsed = [] + for rdata in rdatas: + record = _parse_caa(rdata) + if record is not None and record not in parsed: + parsed.append(record) relevant = [p for p in parsed if p[1] == want_tag] if not parsed: continue # keep climbing; this level has no CAA record set diff --git a/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py b/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py index 0ba4db9e..8a801b33 100644 --- a/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py +++ b/custom-domain/dstack-ingress/scripts/tests/test_dnsguide.py @@ -84,6 +84,41 @@ def test_account_unknown_skips_the_comparison(self): self.assertTrue(ok) +class TestCaaCheckDeduplication(unittest.TestCase): + """One record seen twice, in two wire formats, must be reported once. + + Resolvers disagree on CAA presentation: Cloudflare hands back RFC 3597 + generic hex, Google the human-readable form. The union is taken over raw + rdata, so the same record arrives as two different strings. + """ + + PRESENTATION = '0 issue "letsencrypt.org;validationmethods=dns-01"' + # The same record, generic-encoded: flags=0, tag len 5, "issue", value. + GENERIC = ( + r"\# 44 00 05 69 73 73 75 65 6c 65 74 73 65 6e 63 72 79 70 74 2e 6f 72 67 " + r"3b 76 61 6c 69 64 61 74 69 6f 6e 6d 65 74 68 6f 64 73 3d 64 6e 73 2d 30 31" + ) + + def setUp(self): + self._saved = dnsguide.query_union + dnsguide.query_union = lambda name, rr, resolvers: ( + [self.GENERIC, self.PRESENTATION] if name == "app.example.com" else [] + ) + self.addCleanup(lambda: setattr(dnsguide, "query_union", self._saved)) + + def test_both_encodings_parse_to_the_same_record(self): + self.assertEqual( + dnsguide._parse_caa(self.GENERIC), dnsguide._parse_caa(self.PRESENTATION) + ) + + def test_reason_is_not_repeated(self): + ok, reason = dnsguide.check_caa( + "app.example.com", "issue", "tls-alpn-01", "", "r1,r2" + ) + self.assertFalse(ok) + self.assertEqual(reason.count("excludes tls-alpn-01"), 1, reason) + + class TestTxtNormalisation(unittest.TestCase): def test_strips_quotes(self): self.assertEqual(dnsguide._unquote_txt('"abc:443"'), "abc:443") From 4a822fc0fff31e7373ca3802866fc9820cec6421 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 23:41:02 -0700 Subject: [PATCH 11/16] fix(dstack-ingress): stop the placeholder path reporting itself as a failure On a first tls-alpn-01 run the log read: Generating placeholder certificate for app.example.com Warning: Cert files missing for app.example.com, skipping The warning is what build_combined_pems prints whenever lego has no certificate yet, which on the first pass is exactly the situation the placeholder exists to cover. Nothing is wrong -- skipping is what keeps the placeholder in place so haproxy can bind and forward acme-tls/1 -- but back to back with the line above it reads like the placeholder was discarded, and it is the first thing a new deployment shows. Distinguish the two cases: keeping a placeholder is normal, having neither a certificate nor a placeholder is not. dns-01 has no placeholder, so its copy keeps the original wording. --- custom-domain/dstack-ingress/scripts/tlsalpn.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh index 9c12d882..b8342d05 100644 --- a/custom-domain/dstack-ingress/scripts/tlsalpn.sh +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -161,8 +161,13 @@ build_combined_pems() { cat "$fullchain" "$privkey" > "$combined" chmod 600 "$combined" echo "Combined PEM created: ${combined}" + elif [ -f "$combined" ]; then + # Expected on a first run: ensure_placeholder_certs has already put a + # self-signed certificate here so haproxy can bind and forward + # acme-tls/1, and the real one does not exist yet. Leave it alone. + echo "No certificate for ${domain} yet; serving the placeholder" else - echo "Warning: Cert files missing for ${domain}, skipping" + echo "Warning: no certificate or placeholder for ${domain}, skipping" fi done <<<"$(get-all-domains.sh)" } From db308065a0b3ecd07e27f99b1a547b439cd2a4f0 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 27 Jul 2026 23:59:20 -0700 Subject: [PATCH 12/16] feat(dstack-ingress): honour RENEW_DAYS_BEFORE on dns-01, and document testing RENEW_DAYS_BEFORE was wired into legoman.py only, so it silently did nothing on the dns-01 path -- while the README listed it in the shared table, with every other tls-alpn-01-only setting explicitly marked as such and this one not. The asymmetry also cost test coverage. lego takes the renewal window as a flag, which makes "certificate is due" reachable on demand and lets both renewal branches be exercised. certbot has no equivalent flag: it reads `renew_before_expiry` from the lineage's renewal config. With nothing writing that, a fresh certificate could not be made due, and the dns-01 renewal path could only ever be observed in its "nothing to do" branch. Write the setting before `certbot renew` so the variable means the same thing in both modes. Unset keeps certbot's own 30-day default. TESTING.md documents how to exercise this component against real infrastructure -- three tiers by cost, what each can and cannot observe, how to reach both renewal branches, and the traps that make a test pass for the wrong reason (leftover records changing which branch runs, single-resolver negative caching, gateway TXT caching). --- custom-domain/dstack-ingress/README.md | 2 +- custom-domain/dstack-ingress/TESTING.md | 277 ++++++++++++++++++ .../dstack-ingress/scripts/certman.py | 69 +++++ 3 files changed, 347 insertions(+), 1 deletion(-) create mode 100644 custom-domain/dstack-ingress/TESTING.md diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index 55346db4..f683efe3 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -182,7 +182,7 @@ environment: | `DOH_RESOLVERS` | Google + Cloudflare | Comma-separated DoH endpoints used to verify records | | `TLSALPN_PORT` | `9443` | Loopback port lego's ACME responder binds to | | `TLS_TERMINATE_PORT` | `9444` | Loopback port the TLS frontend moves to in tls-alpn-01 mode | -| `RENEW_DAYS_BEFORE` | lego default | Days of remaining lifetime that trigger renewal | +| `RENEW_DAYS_BEFORE` | client default | Days of remaining lifetime that trigger renewal. Applies to both modes: passed to lego as `--renew-days`, and written to certbot's `renew_before_expiry` | | `RENEW_INTERVAL` | `43200` | Seconds between successful certificate passes | | `DNS_SETTLE_SECONDS` | `30` | Wait after DNS verifies, so the gateway's own TXT cache expires | | `MAXCONN` | `4096` | HAProxy max connections | diff --git a/custom-domain/dstack-ingress/TESTING.md b/custom-domain/dstack-ingress/TESTING.md new file mode 100644 index 00000000..a4267f8f --- /dev/null +++ b/custom-domain/dstack-ingress/TESTING.md @@ -0,0 +1,277 @@ +# Testing dstack-ingress + +This component's entire job is to obtain and serve TLS certificates. Almost +everything that can go wrong involves a party we do not control — a CA, a DNS +resolver, a gateway, a TEE — so a test that mocks those tests very little. What +follows is how to exercise it against the real things, cheaply enough to do +routinely. + +Use **Let's Encrypt staging** throughout (`CERTBOT_STAGING=true` / +`ACME_STAGING=true`). Its rate limits are generous and its certificates are +untrusted, which is exactly what you want: a test certificate that browsers +reject cannot be mistaken for a production one. + +## Three tiers, in increasing cost + +Pick the cheapest tier that can actually observe what you changed. + +| Tier | Setup | Can test | Cannot test | +|---|---|---|---| +| **1. Container + simulator** | Docker, dstack simulator | dns-01 end to end, config generation, negative paths, `DNS_SETUP_MODE`, evidence, webhook | Anything needing inbound traffic | +| **2. Tier 1 + real DNS credentials** | + a zone you control | Everything the DNS provider code does | Same | +| **3. Real CVM behind a gateway** | + TDX host, `dstack-vmm`, `dstack-gateway`, public port 443 | tls-alpn-01, ALPN routing, real TDX quotes, gateway interaction | — | + +dns-01 needs no inbound connection: the CA reads a TXT record and never talks to +you. That is why the whole dns-01 path fits in tier 1/2. tls-alpn-01 is the +opposite — the CA connects to port 443 and completes a TLS handshake — so it can +only be tested for real in tier 3. + +## Tier 1/2: containers + +The container asks the guest agent for its app and instance ID and for TDX +quotes, so it needs a socket at `/var/run/dstack.sock`. Outside a CVM, use the +simulator: + +```bash +cd dstack/sdk/simulator && ./build.sh && ./dstack-simulator & +``` + +Then mount its socket: + +```yaml +volumes: + - /path/to/sdk/simulator/dstack.sock:/var/run/dstack.sock +``` + +Keep provider credentials out of the compose file — put them in a `.env` that +compose reads, and `chmod 600` it: + +```bash +umask 077 && printf 'CLOUDFLARE_API_TOKEN=%s\n' "$TOKEN" > .env +``` + +Give each scenario its own project name and its own volumes +(`docker compose -p `), so a leftover certificate from a previous run +cannot make the next one pass for the wrong reason. This matters more than it +sounds — see "Start from a clean zone" below. + +## Tier 3: a real CVM + +You need a TDX host running `dstack-vmm`, `dstack-kms` and `dstack-gateway`, and +the gateway must be reachable from the internet on **port 443** — the CA picks +that port and RFC 8737 gives you no say. If the gateway listens elsewhere, +redirect: + +```bash +sudo iptables -t nat -A PREROUTING -d -p tcp --dport 443 \ + -m comment --comment 'ingress-lab' -j REDIRECT --to-ports +``` + +Tag every rule with a `--comment` so teardown can find them all later. + +Point the test hostname at the gateway (an `A` record works; the container's +CNAME check compares resolved addresses, not record types), then publish the +routing TXT: + +``` +_dstack-app-address. TXT ":443" +``` + +**Instance ID, not app ID.** In tls-alpn-01 mode the challenge is answered by +whichever instance holds the ACME order, while the gateway load-balances across +every instance of the app and the CA validates from several vantage points at +once — five distinct source IPs within ~2 seconds, in practice. Publishing the +app ID sends the challenge to the wrong replica almost every time. + +Reading container logs inside a CVM: with `--public-logs`, the guest agent serves +them over the gateway's WireGuard network. + +```bash +curl -s 'http://10.44.0.:8090/logs/dstack-dstack-ingress-1?text&bare&tail=500' +``` + +The container name is the compose-mangled one (`dstack--1`), not the +service name. Find the instance's WireGuard address by probing the peers listed +in `wg show allowed-ips`; stale peers from removed VMs stay in the +list, so probe for an open port rather than trusting it. + +## Exercising each area + +### Issuance + +Nothing special — start the container and wait. What to assert: + +- the certificate is served on port 443 and its issuer is a staging CA; +- SAN matches the requested name; +- for `ROUTING_MAP`, each hostname reaches *its own* backend. Routing is by SNI, + so send the SNI, not just a `Host` header. + +### The ALPN split (tls-alpn-01) + +The peek frontend forwards `acme-tls/1` to lego and everything else to the TLS +frontend. You can drive both sides by hand: + +```bash +openssl s_client -connect :443 -servername -alpn acme-tls/1 +openssl s_client -connect :443 -servername -alpn h2,http/1.1 +``` + +and then confirm in the container's haproxy log that they took different paths: + +``` +tls_peek be_acme/lego ← the ACME probe +tls_peek be_tls_terminate/local ← normal traffic +``` + +This is the mechanism the whole mode rests on, and it is worth checking directly +rather than inferring it from "issuance worked". + +### Renewal — both branches + +Renewal is the part most likely to break in production and the least likely to be +covered, because a fresh certificate is not due for ~60 days. Both clients take a +renewal window, and `RENEW_DAYS_BEFORE` sets it in either mode: + +- **not due**: leave `RENEW_DAYS_BEFORE` unset and restart the container. Expect + lego `Skip renewal` / certbot `No certificates need renewal`, **no haproxy + reload, and no new evidence**. A pass that reloads on every check is a bug — + it means the renewal decision is not being read correctly. +- **due**: set `RENEW_DAYS_BEFORE=365` and restart. Expect a new certificate, + a reload, and regenerated evidence. + +Then check the certificate actually changed on the wire — the serial number, not +just the log line: + +```bash +openssl s_client -connect :443 -servername /dev/null \ + | openssl x509 -noout -serial -dates +``` + +Shorten `RENEW_INTERVAL` (default 43200s) if you want the loop to come round +again without restarting. + +Do not assert on the ACME client's log wording. lego 4.x wrote `no renewal` and +5.x writes `Skip renewal`; certbot's phrasing has moved too. The container uses +lego's `--deploy-hook`, which fires only on an actual renewal, and certbot's exit +behaviour — test the observable effect (reload / no reload), not the prose. + +### Evidence + +`/evidences/` is served on the TLS port. The chain to verify: + +``` +quote.report_data == sha256(sha256sum.txt) +sha256sum.txt covers acme-account.json and every cert-.pem +cert-.pem == the certificate actually served +``` + +so: + +```bash +curl -sk https:///evidences/sha256sum.txt -o s.txt && sha256sum -c s.txt +python3 -c "import json,hashlib;print(json.load(open('quote.json'))['report_data'][:64] \ + == hashlib.sha256(open('s.txt','rb').read()).hexdigest())" +``` + +Compare the evidence copy against a live `openssl s_client` fingerprint — the +point of the chain is that the quote commits to what is *being served*, and only +that comparison tests it. + +To verify the quote itself rather than trusting it, run it through `dcap-qvl` +against Intel collateral and expect `status = UpToDate`. Re-run the whole chain +after a renewal: the evidence must track the new certificates, not go stale at +first issuance. + +### DNS setup modes (tls-alpn-01) + +- `wait` — the default. Start the container *before* creating the records and + confirm it blocks and says which record is missing. Then create them and + confirm it proceeds. This is also the only mode that runs the CAA + compatibility check. +- `print` — skips all verification, including CAA. Confirm it continues + immediately. +- `webhook` — set `DNS_WEBHOOK_URL` and `DNS_WEBHOOK_TOKEN` and point them at a + throwaway HTTP server. The body carries the records, an HMAC-SHA256 over the + payload, and a TDX quote whose `report_data` is `sha256(payload)`. Verify both: + the HMAC proves the shared secret, the quote proves the enclave. + +### Negative paths + +These fail fast and are cheap, so run them on every change: + +| Scenario | Expected | +|---|---| +| tls-alpn-01 + a wildcard domain | Refused before any ACME call, citing RFC 8737 | +| A CAA record with `validationmethods=dns-01`, mode tls-alpn-01 | Blocked with the restriction quoted back | +| TXT holding the wrong value | Reported as `want , saw `, not "missing" | +| An instance ID the gateway does not know | The CA reports a connection error — the gateway will not route to it | + +The last one is worth keeping: it is the cross-tenant impersonation defence, and +it is observable rather than merely argued. + +## Traps + +**Start from a clean zone.** Records left over from an earlier run change which +branch executes. A stale CAA record naming a previous ACME account makes dns-01's +first issuance attempt fail and the second succeed — which happens to be the path +where the code was correct, hiding a bug in the ordinary path where the first +attempt succeeds. Delete test records between runs and verify against the +authoritative nameserver, not a cached resolver. + +**One resolver is not enough to trust either answer.** Public resolvers cache +negative answers for the zone's SOA minimum, so a resolver queried before a +record existed can keep saying "no such record" for up to an hour. They also +disagree on wire format: Cloudflare returns CAA in RFC 3597 generic hex +(`\# 47 00 05 69 73 73 75 65 ...`), Google in presentation form. The container +queries two resolvers with deliberately opposite quorums — propagation needs all +of them to agree, CAA takes the union so any resolver seeing a restriction stops +issuance — and if you are checking DNS by hand you should do the same. + +**Public DNS being correct is not the same as the gateway acting on it.** The +gateway caches the app-address TXT for its TTL, so immediately after the value +changes it can still route to the previous target. `DNS_SETTLE_SECONDS` +(default 30, first pass only) covers this. A challenge that fails with +`Error getting validation data` right after a DNS change is usually this, not a +broken challenge. + +**lego buffers its output until it exits.** A renewal can show nothing at all in +the log for several minutes. Check whether the process is still running before +concluding it hung. + +**`ALPN` is not validated against the backend.** Advertising `h2` in front of an +HTTP/1.1-only backend makes haproxy negotiate HTTP/2 on its behalf and the +connection then breaks. If a test client fails but `--http1.1` works, suspect +this before suspecting the proxy. + +## Unit tests + +```bash +python3 scripts/tests/test_dnsguide.py # DNS/CAA parsing and record building +bash scripts/tests/test_sanitizers.sh # env var validation +``` + +The parsing tests are built from rdata real resolvers actually returned, in both +encodings. When you touch DNS handling, add the real string you saw rather than +one you composed — the encodings are the part that surprises people. + +## haproxy config equivalence + +The proxy config is assembled by `haproxy-lib.sh` and composed differently by +each mode. When refactoring that assembly, generate the config both before and +after your change for all four combinations — `dns-01` and `tls-alpn-01`, each +with a single domain and with `ROUTING_MAP` — and diff them. Byte-identical +output is a much stronger statement than "the tests still pass", and it is easy +to obtain because the emitters only write to stdout. + +## Cleanup + +Leftovers from a certificate test are unusually annoying: a stray CAA record can +block issuance for a domain weeks later, and a stray DNAT rule silently +intercepts port 443. Tear down in this order and verify each: + +1. containers and volumes (`docker compose -p down -v`); +2. the CVM (`vmm-cli remove`); +3. iptables rules — delete by comment tag, in a loop, until none match; +4. DNS records — list them from the provider API afterwards, not by `dig`, + which can serve a cached answer for a record you already deleted; +5. the credentials file, the simulator, and any images built for the test. diff --git a/custom-domain/dstack-ingress/scripts/certman.py b/custom-domain/dstack-ingress/scripts/certman.py index b0c4b051..7082e9b5 100644 --- a/custom-domain/dstack-ingress/scripts/certman.py +++ b/custom-domain/dstack-ingress/scripts/certman.py @@ -3,6 +3,7 @@ from dns_providers import DNSProviderFactory import argparse import os +import re import subprocess import sys import pkg_resources @@ -414,6 +415,7 @@ def renew_certificate(self, domain: str) -> Tuple[bool, bool]: print(f"Failed to install plugin for renewal", file=sys.stderr) return False, False + self.apply_renewal_window(domain) cmd = self._build_certbot_command("renew", domain, "") try: @@ -459,6 +461,73 @@ def certificate_exists(self, domain: str) -> bool: cert_path = f"/etc/letsencrypt/live/{domain}/fullchain.pem" return os.path.isfile(cert_path) + @staticmethod + def _lineage_name(domain: str) -> str: + """certbot stores a wildcard lineage under the bare name.""" + return domain[2:] if domain.startswith("*.") else domain + + def apply_renewal_window(self, domain: str) -> None: + """Make RENEW_DAYS_BEFORE mean the same thing here as it does for lego. + + lego takes the renewal window as a flag (`--renew-days`). certbot has no + equivalent: it reads `renew_before_expiry` out of the lineage's renewal + config, so the setting has to be written there before `certbot renew` + runs. Without this the variable is silently tls-alpn-01 only, which also + left the dns-01 renewal branch with no way to be exercised on demand. + + Leaving it unset keeps certbot's own default (30 days). + """ + days = os.environ.get("RENEW_DAYS_BEFORE", "").strip() + if not days: + return + if not days.isdigit() or int(days) < 1: + print( + f"Warning: ignoring invalid RENEW_DAYS_BEFORE={days!r} " + f"(expected a positive number of days)", + file=sys.stderr, + ) + return + + path = f"/etc/letsencrypt/renewal/{self._lineage_name(domain)}.conf" + if not os.path.isfile(path): + # No lineage yet: the first issuance has not happened, and certbot + # writes this file itself. Nothing to do. + return + + setting = f"renew_before_expiry = {days} days" + try: + with open(path, encoding="utf-8") as fh: + lines = fh.read().splitlines() + except OSError as exc: + print(f"Warning: cannot read {path}: {exc}", file=sys.stderr) + return + + out, replaced = [], False + for line in lines: + # The key ships commented out, so match both forms. + if re.match(r"\s*#?\s*renew_before_expiry\s*=", line): + if not replaced: + out.append(setting) + replaced = True + continue + out.append(line) + + if not replaced: + # Must land before the first section header; the key is top-level. + insert_at = next( + (i for i, line in enumerate(out) if line.strip().startswith("[")), + len(out), + ) + out.insert(insert_at, setting) + + try: + with open(path, "w", encoding="utf-8") as fh: + fh.write("\n".join(out) + "\n") + except OSError as exc: + print(f"Warning: cannot write {path}: {exc}", file=sys.stderr) + return + print(f"Renewal window for {domain} set to {days} days before expiry") + def acme_account_exists(self) -> bool: """Check if an ACME account exists for the current server (staging or production). From c7224f3264a18932f436c6a5a0ca6e9b669c54a9 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 28 Jul 2026 00:07:05 -0700 Subject: [PATCH 13/16] docs(dstack-ingress): warn that RENEW_DAYS_BEFORE keeps the cert permanently due --- custom-domain/dstack-ingress/TESTING.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/custom-domain/dstack-ingress/TESTING.md b/custom-domain/dstack-ingress/TESTING.md index a4267f8f..4ba1d820 100644 --- a/custom-domain/dstack-ingress/TESTING.md +++ b/custom-domain/dstack-ingress/TESTING.md @@ -148,7 +148,11 @@ openssl s_client -connect :443 -servername /dev/nu ``` Shorten `RENEW_INTERVAL` (default 43200s) if you want the loop to come round -again without restarting. +again without restarting — but note that `RENEW_DAYS_BEFORE=365` makes the +certificate *permanently* due, so every pass renews. Combined with a short +interval that is a loop issuing certificates as fast as the CA will allow. Set +one or the other, unset it once you have seen the branch you came for, and never +carry it into production. Do not assert on the ACME client's log wording. lego 4.x wrote `no renewal` and 5.x writes `Skip renewal`; certbot's phrasing has moved too. The container uses From 653121fe750ef0118ad04007c1bae7a8c97a4bc5 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 28 Jul 2026 00:38:01 -0700 Subject: [PATCH 14/16] fix(dstack-ingress): reject tls-alpn-01 wildcards at startup, not per pass A wildcard under tls-alpn-01 can never succeed -- RFC 8737 forbids the challenge for wildcard identifiers -- so it is a configuration error, not a runtime failure. process_domain did reject it, but only once the certificate loop reached that domain, by which point the container had generated a placeholder certificate for a name that will never get a real one, started haproxy serving it, and settled into retrying an unsatisfiable config every 60s to 1800s forever. Check it once at the top of tlsalpn.sh, before the startup sequence that does all of the above. It belongs to tls-alpn-01, not to entrypoint.sh: that file validates what both modes share and dispatches, and putting a CHALLENGE_TYPE branch back in it is the mode-specific logic the split was meant to remove -- HTTP-01 would then add a third. Deliberately fatal for the container rather than skipping the one domain. A mixed config did already contain the damage -- the other domains were issued normally -- but it left a self-signed certificate served for the wildcard indefinitely and a permanently failing loop, which reads as a TLS bug rather than a typo. The per-domain check is gone with it: the domain list comes from env vars fixed for the container's lifetime, so it saw exactly what the startup check saw and could not fire. legoman.py keeps its own check, which is a real boundary -- it guards the ACME call and can be invoked standalone. dns-01 wildcards are unaffected. --- custom-domain/dstack-ingress/README.md | 5 +++- .../dstack-ingress/scripts/tlsalpn.sh | 29 +++++++++++++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index f683efe3..93aaed62 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -366,7 +366,10 @@ Encrypt dropping expiry mail, and the container passes it for you. ### Limitations - **No wildcards.** RFC 8737 forbids tls-alpn-01 for wildcard identifiers, and - the CA will not offer the challenge. Use `dns-01` for `*.example.com`. + the CA will not offer the challenge. The container refuses to start rather + than serve a name it can never obtain a certificate for, so a wildcard + anywhere in `DOMAIN`/`DOMAINS` is a startup error even if the other names are + fine. Use `dns-01` for `*.example.com`. - **The gateway must be reachable on port 443.** The CA connects to port 443 of whatever the CNAME resolves to; the port is fixed by the protocol. - **CAA must permit `tls-alpn-01`.** A record left over from a `dns-01` diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh index b8342d05..3f820daf 100644 --- a/custom-domain/dstack-ingress/scripts/tlsalpn.sh +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -89,6 +89,26 @@ require_instance_id() { fi } +# RFC 8737 forbids tls-alpn-01 for wildcard identifiers, so a wildcard here can +# never succeed. That makes it a configuration error, and it has to be caught +# before the startup sequence below: past that point the container has generated +# a placeholder certificate for a name that will never get a real one, started +# haproxy serving it, and settled into retrying an unsatisfiable config forever. +# +# Fatal for the container rather than skipping the one domain. A mixed config +# does contain the damage -- the other domains are issued normally -- but it +# leaves a self-signed certificate served for the wildcard indefinitely and a +# permanently failing loop, which reads as a TLS bug rather than a typo. +reject_wildcards() { + local wildcards + wildcards=$(get-all-domains.sh | grep '^\*\.' || true) + [ -n "$wildcards" ] || return 0 + echo "Error: tls-alpn-01 cannot issue wildcard certificates (RFC 8737)." >&2 + echo "$wildcards" | sed 's/^/ /' >&2 + echo "Use CHALLENGE_TYPE=dns-01 for wildcards, or list the names individually." >&2 + exit 1 +} + # What the gateway should route this hostname to. # # dns-01 uses the app id, so the gateway load-balances across every instance of @@ -190,11 +210,9 @@ collect_evidence() { process_domain() { local domain="$1" - if [[ "$domain" == \*.* ]]; then - echo "Error: cannot issue a wildcard certificate for $domain with tls-alpn-01." >&2 - echo "RFC 8737 forbids it; use CHALLENGE_TYPE=dns-01 for wildcards." >&2 - exit 1 - fi + # No wildcard check here: reject_wildcards has already refused to start over + # the same domain list, which cannot change while the container runs. + # legoman.py checks independently, guarding the ACME call itself. # Register the ACME account first so the CAA record we print can already # pin accounturi. Without this the operator would have to add CAA in a @@ -303,6 +321,7 @@ cert_loop() { done } +reject_wildcards setup_py_env check_lego load_dstack_identity From 09b7d8e2931f06404b4c5f56c3f8eb90be4243e7 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 28 Jul 2026 00:52:17 -0700 Subject: [PATCH 15/16] fix(dstack-ingress): make ACME_STAGING work in both modes, not just tls-alpn-01 ACME_STAGING was read only on the tls-alpn-01 path. Under dns-01 nothing looked at it -- certman.py and functions.sh both read CERTBOT_STAGING -- so setting it there silently issued *production* certificates. The README did not mark it mode-specific, so there was nothing to warn you. Same shape as the RENEW_DAYS_BEFORE bug, and worse in effect: instead of a knob that does nothing, this one does the opposite of what it says on a path where the mistake costs a real certificate and real rate limit. Normalise both names once in entrypoint.sh and export them. This does belong there, unlike the wildcard rule: an ACME account and a staging toggle are genuinely shared by both modes, which is exactly what that file validates. The per-mode copies in dns01.sh and tlsalpn.sh are gone. Name them after ACME rather than certbot throughout. What you configure is an ACME account; which client implements a mode is an implementation detail, and this image already changed it once. CERTBOT_EMAIL and CERTBOT_STAGING keep working -- they are the historical names, not the preferred ones, and the README no longer implies the preference depends on the mode. Also fixes the webhook example, which showed the payload's keys in an order the code does not emit (it sorts them), and drops a stale duplicate comment on ensure_placeholder_certs that credited certbot with forwarding the tls-alpn-01 challenge. --- custom-domain/dstack-ingress/README.md | 7 +++---- custom-domain/dstack-ingress/scripts/certman.py | 15 +++++++++++++-- custom-domain/dstack-ingress/scripts/dns01.sh | 8 +++----- .../dstack-ingress/scripts/entrypoint.sh | 14 ++++++++++++++ custom-domain/dstack-ingress/scripts/functions.sh | 3 ++- custom-domain/dstack-ingress/scripts/tlsalpn.sh | 12 ++---------- 6 files changed, 37 insertions(+), 22 deletions(-) diff --git a/custom-domain/dstack-ingress/README.md b/custom-domain/dstack-ingress/README.md index 93aaed62..a0db066e 100644 --- a/custom-domain/dstack-ingress/README.md +++ b/custom-domain/dstack-ingress/README.md @@ -159,7 +159,7 @@ environment: | `DOMAIN` | Your domain (single-domain mode). Supports wildcards (`*.example.com`) | | `TARGET_ENDPOINT` | Backend address, e.g. `app:80` or `http://app:80` | | `GATEWAY_DOMAIN` | dstack gateway domain (e.g. `_.dstack-prod5.phala.network`) | -| `CERTBOT_EMAIL` | *(optional)* ACME contact address. `ACME_EMAIL` is accepted too, and is the preferred name in tls-alpn-01 mode | +| `ACME_EMAIL` | *(optional)* ACME contact address, in either mode. `CERTBOT_EMAIL` is the historical name and still works. See below — it is optional, and published | | `DNS_PROVIDER` | DNS provider (`cloudflare`, `linode`, `namecheap`) | ### Optional @@ -171,8 +171,7 @@ environment: | `ROUTING_MAP` | | Multi-domain routing: `domain=host:port` per line | | `SET_CAA` | `false` | Enable CAA DNS record (dns-01 only; tls-alpn-01 cannot write DNS) | | `TXT_PREFIX` | `_dstack-app-address` | DNS TXT record prefix | -| `CERTBOT_STAGING` | `false` | Use Let's Encrypt staging server. `ACME_STAGING` is the preferred name in tls-alpn-01 mode | -| `ACME_EMAIL` | | ACME contact address, in either mode. Falls back to `CERTBOT_EMAIL`. Optional — see below | +| `ACME_STAGING` | `false` | Use Let's Encrypt staging, in either mode. `CERTBOT_STAGING` is the historical name and still works | | `CHALLENGE_TYPE` | `dns-01` | `dns-01` (certbot + DNS credentials) or `tls-alpn-01` (lego, no DNS credentials) | | `DNS_SETUP_MODE` | `wait` | tls-alpn-01 only: `wait`, `print` or `webhook` — see below | | `DNS_SETUP_TIMEOUT` | `1800` | tls-alpn-01 only: seconds to wait for the records to appear | @@ -386,7 +385,7 @@ Encrypt dropping expiry mail, and the container passes it for you. ```json { - "payload": "{\"version\":1,\"domain\":\"app.example.com\",\"records\":[...]}", + "payload": "{\"app_id\":\"…\",\"challenge\":\"tls-alpn-01\",\"domain\":\"app.example.com\",\"instance_id\":\"…\",\"records\":[…],\"timestamp\":1234567890,\"version\":1}", "hmac_sha256": "…", "attestation": { "quote": "…", "report_data": "…" } } diff --git a/custom-domain/dstack-ingress/scripts/certman.py b/custom-domain/dstack-ingress/scripts/certman.py index 7082e9b5..30427294 100644 --- a/custom-domain/dstack-ingress/scripts/certman.py +++ b/custom-domain/dstack-ingress/scripts/certman.py @@ -13,6 +13,17 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +def staging_enabled() -> bool: + """Whether to use Let's Encrypt staging. + + entrypoint.sh normalises the two names, but certman.py is also runnable on + its own, so accept both here as well. ACME_STAGING wins: the setting is + about the ACME server, not about certbot. + """ + value = os.environ.get("ACME_STAGING") or os.environ.get("CERTBOT_STAGING", "false") + return value == "true" + + class CertManager: """Certificate management using DNS provider infrastructure.""" @@ -326,7 +337,7 @@ def _build_certbot_command(self, action: str, domain: str, email: str) -> List[s else: base_cmd.append("--register-unsafely-without-email") base_cmd.extend(["-d", domain]) - if os.environ.get("CERTBOT_STAGING", "false") == "true": + if staging_enabled(): base_cmd.extend(["--staging"]) if getattr(self.provider, 'CERTBOT_PROPAGATION_SECONDS'): @@ -540,7 +551,7 @@ def acme_account_exists(self) -> bool: """ import glob api_endpoint = "acme-v02.api.letsencrypt.org" - if os.environ.get("CERTBOT_STAGING", "false") == "true": + if staging_enabled(): api_endpoint = "acme-staging-v02.api.letsencrypt.org" pattern = f"/etc/letsencrypt/accounts/{api_endpoint}/directory/*/regr.json" return len(glob.glob(pattern)) > 0 diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh index 2e75d485..c22eb547 100644 --- a/custom-domain/dstack-ingress/scripts/dns01.sh +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -11,11 +11,9 @@ source /scripts/functions.sh source /scripts/haproxy-lib.sh source /scripts/evidence-lib.sh -# ACME contact address. CERTBOT_EMAIL is the documented name on this path -- -# certbot really is the client here -- and ACME_EMAIL is accepted too so one -# variable works in either mode. Optional either way. -ACME_EMAIL=${ACME_EMAIL:-${CERTBOT_EMAIL:-}} -export ACME_EMAIL +# ACME_EMAIL / ACME_STAGING are normalised by entrypoint.sh and exported. The +# contact address is optional; certman.py asks for a contactless account when +# none is given. # certbot stores certificates under /etc/letsencrypt/live//. cert_fullchain_path() { echo "/etc/letsencrypt/live/$(cert_dir_name "$1")/fullchain.pem"; } diff --git a/custom-domain/dstack-ingress/scripts/entrypoint.sh b/custom-domain/dstack-ingress/scripts/entrypoint.sh index 587bf830..1f564e75 100644 --- a/custom-domain/dstack-ingress/scripts/entrypoint.sh +++ b/custom-domain/dstack-ingress/scripts/entrypoint.sh @@ -22,6 +22,19 @@ EVIDENCE_PORT=${EVIDENCE_PORT:-80} ALPN=${ALPN:-} CHALLENGE_TYPE=${CHALLENGE_TYPE:-dns-01} +# ACME account settings, normalised once for both modes. What you are +# configuring is an ACME account, not a particular client -- which client +# implements a mode is an implementation detail, and this image already changed +# it once (certbot for dns-01, lego for tls-alpn-01). So ACME_EMAIL and +# ACME_STAGING are the names; CERTBOT_EMAIL and CERTBOT_STAGING are the +# historical ones and keep working. +# +# Normalising here rather than per mode is what fixes the real bug: ACME_STAGING +# used to be read only on the tls-alpn-01 path, so setting it under dns-01 +# silently issued *production* certificates. +ACME_EMAIL=${ACME_EMAIL:-${CERTBOT_EMAIL:-}} +ACME_STAGING=${ACME_STAGING:-${CERTBOT_STAGING:-false}} + case "$CHALLENGE_TYPE" in dns-01|tls-alpn-01) ;; *) @@ -72,6 +85,7 @@ done # the helpers it invokes see the sanitized values. export PORT DOMAIN DOMAINS TARGET_ENDPOINT ROUTING_MAP TXT_PREFIX MAXCONN export TIMEOUT_CONNECT TIMEOUT_CLIENT TIMEOUT_SERVER EVIDENCE_SERVER EVIDENCE_PORT ALPN +export ACME_EMAIL ACME_STAGING case "$CHALLENGE_TYPE" in dns-01) diff --git a/custom-domain/dstack-ingress/scripts/functions.sh b/custom-domain/dstack-ingress/scripts/functions.sh index 23d0df85..17dc396f 100644 --- a/custom-domain/dstack-ingress/scripts/functions.sh +++ b/custom-domain/dstack-ingress/scripts/functions.sh @@ -166,7 +166,8 @@ get_letsencrypt_account_path() { local base_path="/etc/letsencrypt/accounts" local api_endpoint="acme-v02.api.letsencrypt.org" - if [[ "$CERTBOT_STAGING" == "true" ]]; then + # entrypoint.sh normalises CERTBOT_STAGING into ACME_STAGING. + if [[ "$ACME_STAGING" == "true" ]]; then api_endpoint="acme-staging-v02.api.letsencrypt.org" fi diff --git a/custom-domain/dstack-ingress/scripts/tlsalpn.sh b/custom-domain/dstack-ingress/scripts/tlsalpn.sh index 3f820daf..485bce35 100644 --- a/custom-domain/dstack-ingress/scripts/tlsalpn.sh +++ b/custom-domain/dstack-ingress/scripts/tlsalpn.sh @@ -12,17 +12,13 @@ source /scripts/functions.sh source /scripts/haproxy-lib.sh source /scripts/evidence-lib.sh -# ACME account settings. CERTBOT_EMAIL / CERTBOT_STAGING are the historical -# names, kept working for anyone migrating a compose file from dns-01, but there -# is no certbot on this path -- the ACME client here is lego. +# ACME_EMAIL / ACME_STAGING are normalised by entrypoint.sh and exported, so +# they are already correct here whichever name the operator used. # # The contact address is optional. RFC 8555 makes it optional, Let's Encrypt # stopped sending expiry notifications in 2025, and it does not stay private: # the account document is published at /evidences/acme-account.json, so an # address set here is readable by anyone who fetches the evidence. -ACME_EMAIL=${ACME_EMAIL:-${CERTBOT_EMAIL:-}} -ACME_STAGING=${ACME_STAGING:-${CERTBOT_STAGING:-false}} -export ACME_EMAIL ACME_STAGING LEGO_BIN=${LEGO_BIN:-/usr/local/bin/lego} LEGO_PATH=${LEGO_PATH:-/etc/letsencrypt/lego} @@ -148,10 +144,6 @@ EOF # haproxy refuses to start when the crt directory has no PEM in it, and here it # has to be listening before the first certificate can exist -- it is what # forwards the challenge to lego. Seed a placeholder the real cert overwrites. -# haproxy refuses to start when the crt directory has no PEM in it. In -# tls-alpn-01 mode the proxy has to be listening *before* the first certificate -# exists, because it is what forwards the ACME challenge to certbot, so seed a -# self-signed placeholder that the real certificate overwrites later. ensure_placeholder_certs() { local all_domains domain pem all_domains=$(get-all-domains.sh) From 42a7bbdaf134bdd7b5d5a549207e266a5cf4e090 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 28 Jul 2026 01:22:08 -0700 Subject: [PATCH 16/16] fix(dstack-ingress): carry the #104 challenge delegation through the restructure #104 added ACME DNS-01 challenge delegation to entrypoint.sh. This branch moves that file's logic into dns01.sh, so a plain rebase drops the shell half of it: set_alias_record / set_txt_record / set_caa_record kept this branch's versions and the delegation branches went with them. Port them into dns01.sh, which is where the delegation belongs anyway -- it is a dns-01 capability, and entrypoint.sh no longer holds mode-specific logic. Everything else from #104 survived the rebase untouched: the hook script, unset_txt_record, the dnsman.py action, the certman.py branch and the README section. Two of the ports are deliberate substitutions, not copies: - The CAA helpers reuse this branch's caa_tag_for() and "${domain#\*.}" instead of re-introducing caa_domain_and_tag(). Behaviour is identical for both wildcard and plain names; the second helper would only be a second place to keep them in sync. - The delegated TXT instruction prints $(txt_record_value) rather than "$APP_ID:$PORT". APP_ID is an optional override, so the original printed ":443" whenever it was unset -- which is the common case. Printing the same function that writes the record in non-delegation mode also stops the instruction drifting from what the gateway actually expects. Two more changes are integration fixes: bugs neither PR has alone, only the combination. - The delegation branch built its certbot command with CERTBOT_STAGING, which this branch renamed to ACME_STAGING. ACME_STAGING=true plus delegation would have issued *production* certificates. - It also passed --email unconditionally, which this branch made optional; with no contact address configured that is `--email ""`. It now uses the same optional-contact handling as the plugin path. Verified against a delegation run: the served zone is untouched, all four static records print with correct values, the certbot command carries --manual with both hooks plus --staging and --register-unsafely-without-email, CAA verification fails closed with exit 1, and ALLOW_MISSING_CAA=true overrides it. --- .../dstack-ingress/scripts/certman.py | 9 ++- custom-domain/dstack-ingress/scripts/dns01.sh | 81 +++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/custom-domain/dstack-ingress/scripts/certman.py b/custom-domain/dstack-ingress/scripts/certman.py index 30427294..f9b1ede8 100644 --- a/custom-domain/dstack-ingress/scripts/certman.py +++ b/custom-domain/dstack-ingress/scripts/certman.py @@ -293,12 +293,17 @@ def _build_certbot_command(self, action: str, domain: str, email: str) -> List[s f"--manual-auth-hook={hook} auth", f"--manual-cleanup-hook={hook} cleanup", "--agree-tos", "--no-eff-email", - "--email", email, "-d", domain, ]) + # Same optional-contact handling as the plugin path below. + if email: + base_cmd.extend(["--email", email]) + else: + base_cmd.append("--register-unsafely-without-email") + base_cmd.extend(["-d", domain]) # For `renew`, certbot reuses the authenticator + hooks saved in the # renewal config from the initial `certonly`, so we don't re-specify # them here (and must not fall back to the DNS plugin). - if os.environ.get("CERTBOT_STAGING", "false") == "true": + if staging_enabled(): base_cmd.append("--staging") masked = [a if not (i > 0 and base_cmd[i - 1] == "--email") else "" for i, a in enumerate(base_cmd)] diff --git a/custom-domain/dstack-ingress/scripts/dns01.sh b/custom-domain/dstack-ingress/scripts/dns01.sh index c22eb547..ada2353e 100644 --- a/custom-domain/dstack-ingress/scripts/dns01.sh +++ b/custom-domain/dstack-ingress/scripts/dns01.sh @@ -76,6 +76,16 @@ EOF set_alias_record() { local domain="$1" + if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then + # certbot validates the challenge at the base name even for a wildcard, + # so the _acme-challenge CNAME must use the base domain (strip "*."). + local base="${domain#\*.}" + echo "[challenge-delegation] Not touching ${domain}'s own zone (token is scoped to the delegated zone)." + echo "[challenge-delegation] Set these in your production zone yourself (static, one-time):" + echo " ${domain} CNAME ${GATEWAY_DOMAIN}" + echo " _acme-challenge.${base} CNAME _acme-challenge.${base}.${ACME_CHALLENGE_ALIAS}" + return + fi echo "Setting alias record for $domain" dnsman.py set_alias \ --domain "$domain" \ @@ -95,6 +105,12 @@ set_txt_record() { local txt_domain txt_domain=$(txt_record_name "$domain") + if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then + echo "[challenge-delegation] Set this in your production zone yourself (static, one-time):" + echo " ${txt_domain} TXT \"$(txt_record_value)\"" + return + fi + dnsman.py set_txt \ --domain "$txt_domain" \ --content "$(txt_record_value)" @@ -105,8 +121,73 @@ set_txt_record() { fi } +# In delegation mode we have NO token for the served domain's zone, so we cannot +# set the accounturi CAA ourselves. That CAA is the forge-prevention (only this +# enclave's ACME account may issue), so this path is deliberately NOT gated by +# SET_CAA and fails closed if the record is confirmed absent -- otherwise anyone +# who can satisfy the delegated DNS-01 challenge could obtain a cert for the +# domain. Set ALLOW_MISSING_CAA=true to override (accept the risk). +verify_delegation_caa() { + local domain="$1" + local account_file account_uri caa_domain caa_tag caa_value resp status found + + if ! account_file=$(get_letsencrypt_account_file); then + echo "ERROR: cannot read the Let's Encrypt account file to determine the required accounturi CAA" >&2 + caa_fail_or_allow "$domain" + return + fi + caa_domain="${domain#\*.}" + caa_tag=$(caa_tag_for "$domain") + account_uri=$(jq -j '.uri' "$account_file") + caa_value="letsencrypt.org;validationmethods=dns-01;accounturi=$account_uri" + + echo "[challenge-delegation] Set this CAA in your production zone (static, one-time):" + echo " ${caa_domain} CAA 0 ${caa_tag} \"${caa_value}\"" + + # Verify via DNS-over-HTTPS (dig is not installed in the image; curl+jq are). + resp=$(curl -s --max-time 10 "https://dns.google/resolve?name=${caa_domain}&type=257" 2>/dev/null || true) + if [ -z "$resp" ]; then + echo "WARNING: could not reach dns.google to verify the CAA (network issue) — NOT confirmed; continuing" + return + fi + status=$(echo "$resp" | jq -r '.Status // empty' 2>/dev/null || true) + if [ "$status" != "0" ]; then + echo "WARNING: CAA DoH query for ${caa_domain} returned status ${status:-unknown} — NOT confirmed; continuing" + return + fi + # Literal (grep -F) match on the CAA data fields — the account URI contains + # '/' and '.', which must not be treated as regex. + found=$(echo "$resp" | jq -r '.Answer // [] | .[] | .data' 2>/dev/null | grep -F "accounturi=$account_uri" || true) + if [ -n "$found" ]; then + echo "[challenge-delegation] Verified: accounturi CAA is present for $caa_domain" + return + fi + echo "ERROR: the accounturi CAA is NOT present for $caa_domain." >&2 + echo "ERROR: without it, anyone who can satisfy the delegated DNS-01 challenge could obtain a" >&2 + echo "ERROR: publicly-trusted certificate for this domain (forged TLS termination)." >&2 + caa_fail_or_allow "$domain" +} + +caa_fail_or_allow() { + local domain="$1" + if [ "${ALLOW_MISSING_CAA:-false}" = "true" ]; then + echo "WARNING: ALLOW_MISSING_CAA=true — continuing without a verified accounturi CAA for $domain" + return 0 + fi + echo "ERROR: refusing to continue without the accounturi CAA for $domain." >&2 + echo "ERROR: set the CAA record shown above, or ALLOW_MISSING_CAA=true to override." >&2 + exit 1 +} + set_caa_record() { local domain="$1" + + # Delegation mode is handled separately and is NOT gated by SET_CAA. + if [ -n "${ACME_CHALLENGE_ALIAS:-}" ]; then + verify_delegation_caa "$domain" + return + fi + if [ "$SET_CAA" != "true" ]; then echo "Skipping CAA record setup" return