Merge pull request #674 from Dstack-TEE/feat/gateway-admin-token

gw: add admin API token authentication

Author: kvinwang

Cargo.lock

@@ -192,6 +192,7 @@ rustls-webpki = "0.103.10"
 schnorrkel = "0.11.4"
 sha2 = { version = "0.10.8", default-features = false }
 sha3 = "0.10.8"
+subtle = "2"
 blake2 = "0.10.6"
 tokio-rustls = { version = "0.26.2", features = ["ring"] }
 x25519-dalek = { version = "2.0.1", features = ["static_secrets"] }

Cargo.toml

@@ -62,6 +62,7 @@ uuid = { workspace = true, features = ["v4"] }
 rmp-serde.workspace = true
 or-panic.workspace = true
 base64.workspace = true
+subtle.workspace = true
 
 [target.'cfg(unix)'.dependencies]
 nix = { workspace = true, features = ["resource"] }

gateway/Cargo.toml

@@ -289,12 +289,18 @@ Important:
 
 ### 2.7 Verify Cluster Sync
 
+The admin API requires a bearer token (see `core.admin.admin_token` in `gateway.toml`,
+or the `ADMIN_API_TOKEN` env injected by `deploy-to-vmm.sh`). Export it once:
+
 ```bash
+export ADMIN_API_TOKEN=...   # value from .env or gateway.toml
+ADMIN_AUTH=(-H "Authorization: Bearer $ADMIN_API_TOKEN")
+
 # Check sync status on any node (replace port with your admin port)
-curl -s http://localhost:9016/prpc/WaveKvStatus | jq .
+curl -s "${ADMIN_AUTH[@]}" http://localhost:9016/prpc/WaveKvStatus | jq .
 
 # List known cluster nodes
-curl -s http://localhost:9016/prpc/Status | jq '.nodes'
+curl -s "${ADMIN_AUTH[@]}" http://localhost:9016/prpc/Status | jq '.nodes'
 ```
 
 A healthy cluster sync shows:
@@ -567,7 +573,8 @@ $CLI info <vm-id>
 Check that the gateway sees the new app:
 
 ```bash
-curl -s http://localhost:<admin-port>/prpc/Status | jq '.hosts'
+curl -s -H "Authorization: Bearer $ADMIN_API_TOKEN" \
+  http://localhost:<admin-port>/prpc/Status | jq '.hosts'
 ```
 
 Expected output should include an entry with the app's `instance_id` and an assigned WireGuard IP:
@@ -619,8 +626,11 @@ Gateway supports automatic TLS certificate management via the ACME protocol. Con
 ### 6.1 Configure ACME Service
 
 ```bash
+ADMIN_AUTH=(-H "Authorization: Bearer $ADMIN_API_TOKEN")
+
 # Set ACME URL (Let's Encrypt production)
-curl -X POST "http://localhost:9016/prpc/SetCertbotConfig" \
+curl -X POST "${ADMIN_AUTH[@]}" \
+  "http://localhost:9016/prpc/SetCertbotConfig" \
   -H "Content-Type: application/json" \
   -d '{"acme_url": "https://acme-v02.api.letsencrypt.org/directory"}'
 
@@ -637,7 +647,8 @@ The Cloudflare API token needs the **DNS:Edit** permission on the target zone. C
 Cloudflare example:
 
 ```bash
-curl -X POST "http://localhost:9016/prpc/CreateDnsCredential" \
+curl -X POST "${ADMIN_AUTH[@]}" \
+  "http://localhost:9016/prpc/CreateDnsCredential" \
   -H "Content-Type: application/json" \
   -d '{
     "name": "cloudflare-prod",
@@ -669,15 +680,17 @@ Parameter description:
 Basic usage (using default DNS credential):
 
 ```bash
-curl -X POST "http://localhost:9016/prpc/AddZtDomain" \
+curl -X POST "${ADMIN_AUTH[@]}" \
+  "http://localhost:9016/prpc/AddZtDomain" \
   -H "Content-Type: application/json" \
   -d '{"domain": "example.com", "port": 443}'
 ```
 
 Specifying DNS credential and node binding:
 
 ```bash
-curl -X POST "http://localhost:9016/prpc/AddZtDomain" \
+curl -X POST "${ADMIN_AUTH[@]}" \
+  "http://localhost:9016/prpc/AddZtDomain" \
   -H "Content-Type: application/json" \
   -d '{
     "domain": "internal.example.com",
@@ -711,15 +724,16 @@ Note: After adding a domain, the certificate is not issued immediately. Gateway
 ### 6.4 Manually Trigger Certificate Renewal
 
 ```bash
-curl -X POST "http://localhost:9016/prpc/RenewZtDomainCert" \
+curl -X POST "${ADMIN_AUTH[@]}" \
+  "http://localhost:9016/prpc/RenewZtDomainCert" \
   -H "Content-Type: application/json" \
   -d '{"domain": "example.com", "force": true}'
 ```
 
 ### 6.5 Check Certificate Status
 
 ```bash
-curl -s http://localhost:9016/prpc/ListZtDomains | jq .
+curl -s "${ADMIN_AUTH[@]}" http://localhost:9016/prpc/ListZtDomains | jq .
 ```
 
 A healthy certificate shows `has_cert: true` and `loaded_in_memory: true`:

gateway/docs/cluster-deployment.md

@@ -14,17 +14,26 @@
 # Load .env if present
 if [ -f ".env" ]; then
   set -a
+  # shellcheck source=/dev/null
   source .env
   set +a
 fi
 
 ADMIN_ADDR="${1:-${GATEWAY_ADMIN_RPC_ADDR:-127.0.0.1:9203}}"
 
+# bootstrap-cluster.sh authenticates to the admin API as an operator. The token
+# is generated by deploy-to-vmm.sh and persisted in .env.
+if [ -z "${ADMIN_API_TOKEN:-}" ]; then
+  echo "ERROR: ADMIN_API_TOKEN must be set (check .env)" >&2
+  exit 1
+fi
+AUTH_HEADER=(-H "Authorization: Bearer $ADMIN_API_TOKEN")
+
 echo "Waiting for gateway admin API at $ADMIN_ADDR..."
 max_retries=60
 retry=0
 while [ $retry -lt $max_retries ]; do
-  if curl -sf "http://$ADMIN_ADDR/prpc/Status" >/dev/null 2>&1; then
+  if curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/Status" >/dev/null 2>&1; then
     break
   fi
   retry=$((retry + 1))
@@ -47,19 +56,19 @@ else
 fi
 
 echo "Setting certbot config (ACME URL: $ACME_URL)..."
-curl -sf -X POST "http://$ADMIN_ADDR/prpc/SetCertbotConfig" \
+curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/SetCertbotConfig" \
   -H "Content-Type: application/json" \
   -d '{"acme_url":"'"$ACME_URL"'","renew_interval_secs":3600,"renew_before_expiration_secs":864000,"renew_timeout_secs":300}' >/dev/null \
   && echo "  Certbot config set" || echo "  WARN: failed to set certbot config"
 
 # Create DNS credential if CF_API_TOKEN is provided and no credentials exist yet
 if [ -n "$CF_API_TOKEN" ]; then
-  existing=$(curl -sf "http://$ADMIN_ADDR/prpc/ListDnsCredentials" 2>/dev/null)
+  existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/ListDnsCredentials" 2>/dev/null)
   cred_count=$(echo "$existing" | jq -r '.credentials | length' 2>/dev/null || echo "0")
 
   if [ "$cred_count" = "0" ]; then
     echo "Creating default DNS credential..."
-    curl -sf -X POST "http://$ADMIN_ADDR/prpc/CreateDnsCredential" \
+    curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/CreateDnsCredential" \
       -H "Content-Type: application/json" \
       -d '{"name":"cloudflare","provider_type":"cloudflare","cf_api_token":"'"$CF_API_TOKEN"'","set_as_default":true}' >/dev/null \
       && echo "  DNS credential created" || echo "  WARN: failed to create DNS credential"
@@ -72,12 +81,12 @@ fi
 
 # Add ZT-Domain if SRV_DOMAIN is provided and domain doesn't exist yet
 if [ -n "$SRV_DOMAIN" ]; then
-  existing=$(curl -sf "http://$ADMIN_ADDR/prpc/ListZtDomains" 2>/dev/null)
+  existing=$(curl -sf "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/ListZtDomains" 2>/dev/null)
   has_domain=$(echo "$existing" | jq -r '.domains[]? | select(.domain=="'"$SRV_DOMAIN"'") | .domain' 2>/dev/null)
 
   if [ -z "$has_domain" ]; then
     echo "Adding ZT-Domain: $SRV_DOMAIN..."
-    curl -sf -X POST "http://$ADMIN_ADDR/prpc/AddZtDomain" \
+    curl -sf -X POST "${AUTH_HEADER[@]}" "http://$ADMIN_ADDR/prpc/AddZtDomain" \
       -H "Content-Type: application/json" \
       -d '{"domain":"'"$SRV_DOMAIN"'","port":443,"priority":100}' >/dev/null \
       && echo "  ZT-Domain added" || echo "  WARN: failed to add ZT-Domain"

gateway/dstack-app/bootstrap-cluster.sh

@@ -36,6 +36,12 @@ validate_env "$NODE_ID"
 validate_env "$WG_IP"
 validate_env "$WG_RESERVED_NET"
 validate_env "$WG_CLIENT_RANGE"
+validate_env "$ADMIN_API_TOKEN"
+
+if [ -z "$ADMIN_API_TOKEN" ]; then
+    echo "ADMIN_API_TOKEN must be set when admin API is enabled"
+    exit 1
+fi
 
 # Validate $NODE_ID, must be a number
 if [[ ! "$NODE_ID" =~ ^[0-9]+$ ]]; then
@@ -89,6 +95,7 @@ sync_connections_interval = "${SYNC_CONNECTIONS_INTERVAL:-30s}"
 enabled = true
 address = "${ADMIN_LISTEN_ADDR:-0.0.0.0}"
 port = ${ADMIN_LISTEN_PORT:-8001}
+admin_token = "${ADMIN_API_TOKEN}"
 
 [core.wg]
 public_key = "$PUBLIC_KEY"

gateway/dstack-app/builder/entrypoint.sh

@@ -95,6 +95,11 @@ WG_ADDR=0.0.0.0:9202
 # The token used to launch the App
 APP_LAUNCH_TOKEN=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 32 | head -n 1)
 
+# Bearer token required by the gateway admin API. Used by bootstrap-cluster.sh
+# and any operator who calls the admin API. Persisted into .env so cluster
+# bootstrap can reach the API after deploy.
+ADMIN_API_TOKEN=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 48 | head -n 1)
+
 # PROXY protocol: read v1/v2 header from inbound connections (e.g. when this
 # gateway sits behind a PP-aware L4 LB such as Cloudflare Spectrum or haproxy
 # with send-proxy). Set to "true" only if the upstream LB is configured to
@@ -117,6 +122,7 @@ required_env_vars=(
   "GATEWAY_APP_ID"
   "MY_URL"
   "APP_LAUNCH_TOKEN"
+  "ADMIN_API_TOKEN"
   "NODE_ID"
   "KMS_URL"
   # "BOOTNODE_URL"
@@ -180,6 +186,7 @@ WG_IP=$WG_IP
 WG_RESERVED_NET=$WG_RESERVED_NET
 WG_CLIENT_RANGE=$WG_CLIENT_RANGE
 APP_LAUNCH_TOKEN=$APP_LAUNCH_TOKEN
+ADMIN_API_TOKEN=$ADMIN_API_TOKEN
 RPC_DOMAIN=$RPC_DOMAIN
 NODE_ID=$NODE_ID
 PROXY_LISTEN_PORT=$PROXY_LISTEN_PORT

gateway/dstack-app/deploy-to-vmm.sh

@@ -41,6 +41,7 @@ services:
       - TIMEOUT_TOTAL=${TIMEOUT_TOTAL:-5h}
       - ADMIN_LISTEN_ADDR=${ADMIN_LISTEN_ADDR:-0.0.0.0}
       - ADMIN_LISTEN_PORT=${ADMIN_LISTEN_PORT:-8001}
+      - ADMIN_API_TOKEN=${ADMIN_API_TOKEN:-}
       - INBOUND_PP_ENABLED=${INBOUND_PP_ENABLED:-false}
       - TIMEOUT_PP_HEADER=${TIMEOUT_PP_HEADER:-5s}
       - PORT_POLICY_FETCH_TIMEOUT=${PORT_POLICY_FETCH_TIMEOUT:-10s}

gateway/dstack-app/docker-compose.yaml

@@ -24,6 +24,15 @@ timeout = "5s"
 [core.admin]
 enabled = false
 address = "127.0.0.1:8011"
+# Shared secret required by every admin endpoint (RPC + dashboard). Can also
+# be supplied via the `DSTACK_GATEWAY_ADMIN_TOKEN` or `ADMIN_API_TOKEN` env
+# vars. Clients send it as `Authorization: Bearer <token>`, `X-Admin-Token`,
+# or (GET only, for dashboard links) `?token=...`. Required unless
+# `insecure_no_auth = true`.
+admin_token = ""
+# Development/testing escape hatch only. Never enable this on an admin
+# interface that is reachable from the network.
+insecure_no_auth = false
 
 [core.debug]
 insecure_enable_debug_rpc = false