ingress: Support for multple domains

Author: kvinwang

custom-domain/dstack-ingress/Dockerfile

@@ -29,7 +29,12 @@ RUN set -e; \
         coreutils && \
         rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache /tmp/pinned-packages.txt
 
-RUN mkdir -p /etc/letsencrypt /var/www/certbot /usr/share/nginx/html
+RUN mkdir -p \
+    /etc/letsencrypt \
+    /var/www/certbot \
+    /usr/share/nginx/html \
+    /etc/nginx/conf.d \
+    /var/log/nginx
 
 # Set up Python virtual environment and install certbot
 RUN set -e; \

custom-domain/dstack-ingress/README.md

@@ -33,14 +33,26 @@ The dstack-ingress system provides a seamless way to set up custom domains for d
 3. **Certificate Management**:
 
    - SSL certificates are automatically obtained during initial setup
-   - A scheduled task runs twice daily to check for certificate renewal
+   - A simple background daemon checks for certificate renewal every 12 hours
    - When certificates are renewed, Nginx is automatically reloaded to use the new certificates
+   - Uses a simple sleep loop instead of cron for reliability and easier debugging in containers
 
 4. **Evidence Generation**:
    - The system generates evidence files for verification purposes
    - These include the ACME account information and certificate data
    - Evidence files are accessible through a dedicated endpoint
 
+## Features
+
+### Multi-Domain Support (New!)
+
+The dstack-ingress now supports multiple domains in a single container:
+
+- **Single Domain Mode** (backward compatible): Use `DOMAIN` and `TARGET_ENDPOINT` environment variables
+- **Multi-Domain Mode**: Use `DOMAINS` environment variable with custom nginx configurations in `/etc/nginx/conf.d/`
+- Each domain gets its own SSL certificate
+- Flexible nginx configuration per domain
+
 ## Usage
 
 ### Prerequisites
@@ -87,15 +99,90 @@ volumes:
   cert-data: # Persistent volume for certificates
 ```
 
+### Multi-Domain Configuration
+
+```yaml
+services:
+  ingress:
+    image: kvin/dstack-ingress:latest
+    ports:
+      - "443:443"
+    environment:
+      DNS_PROVIDER: cloudflare
+      CLOUDFLARE_API_TOKEN: ${CLOUDFLARE_API_TOKEN}
+      CERTBOT_EMAIL: ${CERTBOT_EMAIL}
+      GATEWAY_DOMAIN: _.dstack-prod5.phala.network
+      SET_CAA: true
+      DOMAINS: |
+        ${APP_DOMAIN}
+        ${API_DOMAIN}
+
+    volumes:
+      - /var/run/tappd.sock:/var/run/tappd.sock
+      - letsencrypt:/etc/letsencrypt
+
+    configs:
+      - source: app_conf
+        target: /etc/nginx/conf.d/app.conf
+        mode: 0444
+      - source: api_conf
+        target: /etc/nginx/conf.d/api.conf
+        mode: 0444
+
+    restart: unless-stopped
+
+  app-main:
+    image: nginx
+    restart: unless-stopped
+
+  app-api:
+    image: nginx
+    restart: unless-stopped
+
+volumes:
+  letsencrypt:
+
+configs:
+  app_conf:
+    content: |
+      server {
+          listen 443 ssl;
+          server_name ${APP_DOMAIN};
+          ssl_certificate /etc/letsencrypt/live/${APP_DOMAIN}/fullchain.pem;
+          ssl_certificate_key /etc/letsencrypt/live/${APP_DOMAIN}/privkey.pem;
+          location / {
+              proxy_pass http://app-main:80;
+          }
+      }
+  api_conf:
+    content: |
+      server {
+          listen 443 ssl;
+          server_name ${API_DOMAIN};
+          ssl_certificate /etc/letsencrypt/live/${API_DOMAIN}/fullchain.pem;
+          ssl_certificate_key /etc/letsencrypt/live/${API_DOMAIN}/privkey.pem;
+          location / {
+              proxy_pass http://app-api:80;
+          }
+      }
+```
+
 **Core Environment Variables:**
 
 - `DNS_PROVIDER`: DNS provider to use (cloudflare, linode)
-- `DOMAIN`: Your custom domain
+- `DOMAIN`: Your custom domain (for single domain mode)
+- `DOMAINS`: Multiple domains, one per line (supports environment variable substitution like `${APP_DOMAIN}`)
 - `GATEWAY_DOMAIN`: The dstack gateway domain (e.g. `_.dstack-prod5.phala.network` for Phala Cloud)
 - `CERTBOT_EMAIL`: Your email address used in Let's Encrypt certificate requests
-- `TARGET_ENDPOINT`: The plain HTTP endpoint of your dstack application
+- `TARGET_ENDPOINT`: The plain HTTP endpoint of your dstack application (for single domain mode)
 - `SET_CAA`: Set to `true` to enable CAA record setup
 
+**Backward Compatibility:**
+
+- If both `DOMAIN` and `TARGET_ENDPOINT` are set, the system operates in single-domain mode with auto-generated nginx config
+- If `DOMAINS` is set, the system operates in multi-domain mode and expects custom nginx configs in `/etc/nginx/conf.d/`
+- You can use both modes simultaneously
+
 For provider-specific configuration details, see [DNS Provider Configuration](DNS_PROVIDERS.md).
 
 #### Option 2: Build Your Own Image
@@ -126,7 +213,6 @@ docker push yourusername/dstack-ingress:tag
 
 4. Update the docker-compose.yaml file with your image name and deploy
 
-
 #### gRPC Support
 
 If your dstack application uses gRPC, you can set `TARGET_ENDPOINT` to `grpc://app:50051`.

custom-domain/dstack-ingress/docker-compose.multi.yaml

@@ -0,0 +1,67 @@
+services:
+  ingress:
+    image: kvin/dstack-ingress@sha256:b61d50360c7a4e5ab7d22f5ce87677714f3f64a65db34ee5eebcc54683950c89
+    ports:
+      - "443:443"
+    environment:
+      DNS_PROVIDER: cloudflare
+      CLOUDFLARE_API_TOKEN: ${CLOUDFLARE_API_TOKEN}
+      CERTBOT_EMAIL: ${CERTBOT_EMAIL}
+      GATEWAY_DOMAIN: _.dstack-prod5.phala.network
+      SET_CAA: true
+      DOMAINS: |
+        ${APP_DOMAIN}
+        ${API_DOMAIN}
+
+    volumes:
+      - /var/run/tappd.sock:/var/run/tappd.sock
+      - letsencrypt:/etc/letsencrypt
+
+    configs:
+      - source: app_conf
+        target: /etc/nginx/conf.d/app.conf
+        mode: 0444
+      - source: api_conf
+        target: /etc/nginx/conf.d/api.conf
+        mode: 0444
+
+    restart: unless-stopped
+
+  app-main:
+    image: nginx
+    restart: unless-stopped
+
+  app-api:
+    image: nginx
+    restart: unless-stopped
+
+volumes:
+  letsencrypt:
+
+configs:
+  app_conf:
+    content: |
+      server {
+          listen 443 ssl;
+          server_name ${APP_DOMAIN};
+
+          ssl_certificate /etc/letsencrypt/live/${APP_DOMAIN}/fullchain.pem;
+          ssl_certificate_key /etc/letsencrypt/live/${APP_DOMAIN}/privkey.pem;
+
+          location / {
+              proxy_pass http://app-main:80;
+          }
+      }
+  api_conf:
+    content: |
+      server {
+          listen 443 ssl;
+          server_name ${API_DOMAIN};
+
+          ssl_certificate /etc/letsencrypt/live/${API_DOMAIN}/fullchain.pem;
+          ssl_certificate_key /etc/letsencrypt/live/${API_DOMAIN}/privkey.pem;
+
+          location / {
+              proxy_pass http://app-api:80;
+          }
+      }

…in/dstack-ingress/scripts/dns_manager.py …-domain/dstack-ingress/scripts/dnsman.pycustom-domain/dstack-ingress/scripts/dns_manager.py renamed to custom-domain/dstack-ingress/scripts/dnsman.py custom-domain/dstack-ingress/scripts/dns_manager.py renamed to custom-domain/dstack-ingress/scripts/dnsman.py

@@ -8,6 +8,10 @@ if [[ -e /var/run/dstack.sock ]]; then
 else
   TXT_PREFIX=${TXT_PREFIX:-"_tapp-address"}
 fi
+PROXY_CMD="proxy"
+if [[ "${TARGET_ENDPOINT}" == grpc://* ]]; then
+    PROXY_CMD="grpc"
+fi
 
 echo "Setting up certbot environment"
 
@@ -24,10 +28,7 @@ setup_certbot_env() {
     fi
 }
 
-PROXY_CMD="proxy"
-if [[ "${TARGET_ENDPOINT}" == grpc://* ]]; then
-    PROXY_CMD="grpc"
-fi
+setup_py_env
 
 setup_nginx_conf() {
     cat <<EOF >/etc/nginx/conf.d/default.conf
@@ -84,29 +85,26 @@ server {
     }
 }
 EOF
-    mkdir -p /var/log/nginx
 }
 
-
 set_alias_record() {
-    # Use the unified DNS manager to set the alias record
-    source /opt/app-venv/bin/activate
-    echo "Setting alias record for $DOMAIN"
-    dns_manager.py set_alias \
-        --domain "$DOMAIN" \
+    local domain="$1"
+    echo "Setting alias record for $domain"
+    dnsman.py set_alias \
+        --domain "$domain" \
         --content "$GATEWAY_DOMAIN"
 
     if [ $? -ne 0 ]; then
-        echo "Error: Failed to set alias record for $DOMAIN"
+        echo "Error: Failed to set alias record for $domain"
         exit 1
     fi
-    echo "Alias record set for $DOMAIN"
+    echo "Alias record set for $domain"
 }
 
 set_txt_record() {
+    local domain="$1"
     local APP_ID
 
-    # Generate a unique app ID if not provided
     if [[ -e /var/run/dstack.sock ]]; then
         DSTACK_APP_ID=$(curl -s --unix-socket /var/run/dstack.sock http://localhost/Info | jq -j .app_id)
         export DSTACK_APP_ID
@@ -116,48 +114,72 @@ set_txt_record() {
     fi
     APP_ID=${APP_ID:-"$DSTACK_APP_ID"}
 
-    # Use the unified DNS manager to set the TXT record
-    source /opt/app-venv/bin/activate
-    dns_manager.py set_txt \
-        --domain "${TXT_PREFIX}.${DOMAIN}" \
+    dnsman.py set_txt \
+        --domain "${TXT_PREFIX}.${domain}" \
         --content "$APP_ID:$PORT"
 
     if [ $? -ne 0 ]; then
-        echo "Error: Failed to set TXT record for $DOMAIN"
+        echo "Error: Failed to set TXT record for $domain"
         exit 1
     fi
 }
 
 set_caa_record() {
+    local domain="$1"
     if [ "$SET_CAA" != "true" ]; then
         echo "Skipping CAA record setup"
         return
     fi
-    # Add CAA record for the domain
     local ACCOUNT_URI
-    ACCOUNT_URI=$(jq -j '.uri' /evidences/acme-account.json)
-    echo "Adding CAA record for $DOMAIN, accounturi=$ACCOUNT_URI"
-    source /opt/app-venv/bin/activate
-    dns_manager.py set_caa \
-        --domain "$DOMAIN" \
+    ACCOUNT_URI=$(jq -j '.uri' /etc/letsencrypt/accounts/acme-v02.api.letsencrypt.org/directory/*/regr.json)
+    echo "Adding CAA record for $domain, accounturi=$ACCOUNT_URI"
+    dnsman.py set_caa \
+        --domain "$domain" \
         --caa-tag "issue" \
         --caa-value "letsencrypt.org;validationmethods=dns-01;accounturi=$ACCOUNT_URI"
 
     if [ $? -ne 0 ]; then
-        echo "Warning: Failed to set CAA record for $DOMAIN"
+        echo "Warning: Failed to set CAA record for $domain"
         echo "This is not critical - certificates can still be issued without CAA records"
         echo "Consider disabling CAA records by setting SET_CAA=false if this continues to fail"
         # Don't exit - CAA records are optional for certificate generation
     fi
 }
 
+process_domain() {
+    local domain="$1"
+    echo "Processing domain: $domain"
+
+    set_alias_record "$domain"
+    set_txt_record "$domain"
+    renew-certificate.sh "$domain" || echo "First certificate renewal failed for $domain, will retry after set CAA record"
+    set_caa_record "$domain"
+    renew-certificate.sh "$domain"
+}
+
 bootstrap() {
-    echo "Bootstrap: Setting up $DOMAIN"
-    source /opt/app-venv/bin/activate
-    renew-certificate.sh -n
-    set_alias_record
-    set_txt_record
-    set_caa_record
+    echo "Bootstrap: Setting up domains"
+
+    local all_domains
+    all_domains=$(get-all-domains.sh)
+
+    if [ -z "$all_domains" ]; then
+        echo "Error: No domains found. Set either DOMAIN or DOMAINS environment variable"
+        exit 1
+    fi
+
+    echo "Found domains:"
+    echo "$all_domains"
+
+    while IFS= read -r domain; do
+        [[ -n "$domain" ]] || continue
+        process_domain "$domain"
+    done <<<"$all_domains"
+
+    # Generate evidences after all certificates are obtained
+    echo "Generating evidence files for all domains..."
+    generate-evidences.sh
+
     touch /.bootstrapped
 }
 
@@ -175,6 +197,10 @@ fi
 
 renewal-daemon.sh &
 
-setup_nginx_conf
+mkdir -p /var/log/nginx
+
+if [ -n "$DOMAIN" ] && [ -n "$TARGET_ENDPOINT" ]; then
+    setup_nginx_conf
+fi
 
 exec "$@"