gw: harden admin auth defaults and unify ADMIN_API_TOKEN naming

Address review-style improvements lifted from PR #675:

- Fail-by-default. When `admin.enabled = true`, gateway now refuses to
  start unless either an `admin_token` is configured (via `core.admin.admin_token`,
  `DSTACK_GATEWAY_ADMIN_TOKEN`, or `ADMIN_API_TOKEN`) or `insecure_no_auth = true`
  is set explicitly. Replaces the previous "empty token = open access + WARN"
  policy. e2e configs opt into `insecure_no_auth = true`.
- Env-var fallback in Rust. `AdminAuthFairing::from_config` resolves the
  token from config first, then `DSTACK_GATEWAY_ADMIN_TOKEN`, then
  `ADMIN_API_TOKEN`. Operators running the binary directly no longer have
  to template the TOML.
- SHA-256 in-memory storage. The plaintext token is hashed at startup;
  only the 32-byte digest is retained. Verification SHA-256s the request
  token and constant-time compares the digests via `subtle::ConstantTimeEq`.
- dstack-app deployment chain renamed `ADMIN_TOKEN` -> `ADMIN_API_TOKEN`
  end-to-end (deploy-to-vmm.sh auto-generates it into .env, docker-compose
  forwards it, entrypoint.sh injects it into `gateway.toml`,
  bootstrap-cluster.sh requires it and sends `Authorization: Bearer`).
- Docs in `gateway/docs/cluster-deployment.md` now show the bearer-auth
  pattern in every admin-curl example.

Tests: 4 new cases in `admin_auth::tests` cover the from_config policy
(insecure flag, config path, env fallbacks, error message). All 13
`admin_auth` tests pass; clippy with `-D clippy::expect_used
-D clippy::unwrap_used` is clean.

Author: kvinwang

gateway/docs/cluster-deployment.md

@@ -289,12 +289,18 @@ Important:
 
 ### 2.7 Verify Cluster Sync
 
+The admin API requires a bearer token (see `core.admin.admin_token` in `gateway.toml`,
+or the `ADMIN_API_TOKEN` env injected by `deploy-to-vmm.sh`). Export it once:
+
 ```bash
+export ADMIN_API_TOKEN=...   # value from .env or gateway.toml
+ADMIN_AUTH=(-H "Authorization: Bearer $ADMIN_API_TOKEN")
+
 # Check sync status on any node (replace port with your admin port)
-curl -s http://localhost:9016/prpc/WaveKvStatus | jq .
+curl -s "${ADMIN_AUTH[@]}" http://localhost:9016/prpc/WaveKvStatus | jq .
 
 # List known cluster nodes
-curl -s http://localhost:9016/prpc/Status | jq '.nodes'
+curl -s "${ADMIN_AUTH[@]}" http://localhost:9016/prpc/Status | jq '.nodes'
 ```
 
 A healthy cluster sync shows:
@@ -567,7 +573,8 @@ $CLI info <vm-id>
 Check that the gateway sees the new app:
 
 ```bash
-curl -s http://localhost:<admin-port>/prpc/Status | jq '.hosts'
+curl -s -H "Authorization: Bearer $ADMIN_API_TOKEN" \
+  http://localhost:<admin-port>/prpc/Status | jq '.hosts'
 ```
 
 Expected output should include an entry with the app's `instance_id` and an assigned WireGuard IP:
@@ -619,8 +626,11 @@ Gateway supports automatic TLS certificate management via the ACME protocol. Con
 ### 6.1 Configure ACME Service
 
 ```bash
+ADMIN_AUTH=(-H "Authorization: Bearer $ADMIN_API_TOKEN")
+
 # Set ACME URL (Let's Encrypt production)
-curl -X POST "http://localhost:9016/prpc/SetCertbotConfig" \
+curl -X POST "${ADMIN_AUTH[@]}" \
+  "http://localhost:9016/prpc/SetCertbotConfig" \
   -H "Content-Type: application/json" \
   -d '{"acme_url": "https://acme-v02.api.letsencrypt.org/directory"}'
 
@@ -637,7 +647,8 @@ The Cloudflare API token needs the **DNS:Edit** permission on the target zone. C
 Cloudflare example:
 
 ```bash
-curl -X POST "http://localhost:9016/prpc/CreateDnsCredential" \
+curl -X POST "${ADMIN_AUTH[@]}" \
+  "http://localhost:9016/prpc/CreateDnsCredential" \
   -H "Content-Type: application/json" \
   -d '{
     "name": "cloudflare-prod",
@@ -669,15 +680,17 @@ Parameter description:
 Basic usage (using default DNS credential):
 
 ```bash
-curl -X POST "http://localhost:9016/prpc/AddZtDomain" \
+curl -X POST "${ADMIN_AUTH[@]}" \
+  "http://localhost:9016/prpc/AddZtDomain" \
   -H "Content-Type: application/json" \
   -d '{"domain": "example.com", "port": 443}'
 ```
 
 Specifying DNS credential and node binding:
 
 ```bash
-curl -X POST "http://localhost:9016/prpc/AddZtDomain" \
+curl -X POST "${ADMIN_AUTH[@]}" \
+  "http://localhost:9016/prpc/AddZtDomain" \
   -H "Content-Type: application/json" \
   -d '{
     "domain": "internal.example.com",
@@ -711,15 +724,16 @@ Note: After adding a domain, the certificate is not issued immediately. Gateway
 ### 6.4 Manually Trigger Certificate Renewal
 
 ```bash
-curl -X POST "http://localhost:9016/prpc/RenewZtDomainCert" \
+curl -X POST "${ADMIN_AUTH[@]}" \
+  "http://localhost:9016/prpc/RenewZtDomainCert" \
   -H "Content-Type: application/json" \
   -d '{"domain": "example.com", "force": true}'
 ```
 
 ### 6.5 Check Certificate Status
 
 ```bash
-curl -s http://localhost:9016/prpc/ListZtDomains | jq .
+curl -s "${ADMIN_AUTH[@]}" http://localhost:9016/prpc/ListZtDomains | jq .
 ```
 
 A healthy certificate shows `has_cert: true` and `loaded_in_memory: true`:

gateway/dstack-app/bootstrap-cluster.sh

@@ -21,12 +21,13 @@ fi
 
 ADMIN_ADDR="${1:-${GATEWAY_ADMIN_RPC_ADDR:-127.0.0.1:9203}}"
 
-# When admin auth is enabled, ADMIN_TOKEN must be set so curl can present
-# X-Admin-Token. Empty token = legacy unauthenticated mode.
-AUTH_HEADER=()
-if [ -n "$ADMIN_TOKEN" ]; then
-  AUTH_HEADER=(-H "X-Admin-Token: $ADMIN_TOKEN")
+# bootstrap-cluster.sh authenticates to the admin API as an operator. The token
+# is generated by deploy-to-vmm.sh and persisted in .env.
+if [ -z "${ADMIN_API_TOKEN:-}" ]; then
+  echo "ERROR: ADMIN_API_TOKEN must be set (check .env)" >&2
+  exit 1
 fi
+AUTH_HEADER=(-H "Authorization: Bearer $ADMIN_API_TOKEN")
 
 echo "Waiting for gateway admin API at $ADMIN_ADDR..."
 max_retries=60

gateway/dstack-app/builder/entrypoint.sh

@@ -36,7 +36,12 @@ validate_env "$NODE_ID"
 validate_env "$WG_IP"
 validate_env "$WG_RESERVED_NET"
 validate_env "$WG_CLIENT_RANGE"
-validate_env "$ADMIN_TOKEN"
+validate_env "$ADMIN_API_TOKEN"
+
+if [ -z "$ADMIN_API_TOKEN" ]; then
+    echo "ADMIN_API_TOKEN must be set when admin API is enabled"
+    exit 1
+fi
 
 # Validate $NODE_ID, must be a number
 if [[ ! "$NODE_ID" =~ ^[0-9]+$ ]]; then
@@ -90,7 +95,7 @@ sync_connections_interval = "${SYNC_CONNECTIONS_INTERVAL:-30s}"
 enabled = true
 address = "${ADMIN_LISTEN_ADDR:-0.0.0.0}"
 port = ${ADMIN_LISTEN_PORT:-8001}
-admin_token = "${ADMIN_TOKEN}"
+admin_token = "${ADMIN_API_TOKEN}"
 
 [core.wg]
 public_key = "$PUBLIC_KEY"

gateway/dstack-app/deploy-to-vmm.sh

@@ -95,6 +95,11 @@ WG_ADDR=0.0.0.0:9202
 # The token used to launch the App
 APP_LAUNCH_TOKEN=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 32 | head -n 1)
 
+# Bearer token required by the gateway admin API. Used by bootstrap-cluster.sh
+# and any operator who calls the admin API. Persisted into .env so cluster
+# bootstrap can reach the API after deploy.
+ADMIN_API_TOKEN=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | fold -w 48 | head -n 1)
+
 # PROXY protocol: read v1/v2 header from inbound connections (e.g. when this
 # gateway sits behind a PP-aware L4 LB such as Cloudflare Spectrum or haproxy
 # with send-proxy). Set to "true" only if the upstream LB is configured to
@@ -117,6 +122,7 @@ required_env_vars=(
   "GATEWAY_APP_ID"
   "MY_URL"
   "APP_LAUNCH_TOKEN"
+  "ADMIN_API_TOKEN"
   "NODE_ID"
   "KMS_URL"
   # "BOOTNODE_URL"
@@ -180,6 +186,7 @@ WG_IP=$WG_IP
 WG_RESERVED_NET=$WG_RESERVED_NET
 WG_CLIENT_RANGE=$WG_CLIENT_RANGE
 APP_LAUNCH_TOKEN=$APP_LAUNCH_TOKEN
+ADMIN_API_TOKEN=$ADMIN_API_TOKEN
 RPC_DOMAIN=$RPC_DOMAIN
 NODE_ID=$NODE_ID
 PROXY_LISTEN_PORT=$PROXY_LISTEN_PORT

gateway/dstack-app/docker-compose.yaml

@@ -41,7 +41,7 @@ services:
       - TIMEOUT_TOTAL=${TIMEOUT_TOTAL:-5h}
       - ADMIN_LISTEN_ADDR=${ADMIN_LISTEN_ADDR:-0.0.0.0}
       - ADMIN_LISTEN_PORT=${ADMIN_LISTEN_PORT:-8001}
-      - ADMIN_TOKEN=${ADMIN_TOKEN:-}
+      - ADMIN_API_TOKEN=${ADMIN_API_TOKEN:-}
       - INBOUND_PP_ENABLED=${INBOUND_PP_ENABLED:-false}
       - TIMEOUT_PP_HEADER=${TIMEOUT_PP_HEADER:-5s}
       - PORT_POLICY_FETCH_TIMEOUT=${PORT_POLICY_FETCH_TIMEOUT:-10s}

gateway/gateway.toml

@@ -24,10 +24,15 @@ timeout = "5s"
 [core.admin]
 enabled = false
 address = "127.0.0.1:8011"
-# Shared secret required by every admin endpoint (RPC + dashboard) when
-# non-empty. Clients send it via the `X-Admin-Token` header (or `?token=...`
-# for the dashboard / browser links). Empty disables auth.
+# Shared secret required by every admin endpoint (RPC + dashboard). Can also
+# be supplied via the `DSTACK_GATEWAY_ADMIN_TOKEN` or `ADMIN_API_TOKEN` env
+# vars. Clients send it as `Authorization: Bearer <token>`, `X-Admin-Token`,
+# or (GET only, for dashboard links) `?token=...`. Required unless
+# `insecure_no_auth = true`.
 admin_token = ""
+# Development/testing escape hatch only. Never enable this on an admin
+# interface that is reachable from the network.
+insecure_no_auth = false
 
 [core.debug]
 insecure_enable_debug_rpc = false

gateway/src/admin_auth.rs

@@ -17,30 +17,66 @@
 //! Rejected requests are forwarded to a sentinel route that returns HTTP 401,
 //! so all admin routes (prpc-generated and dashboard) are protected by a single
 //! attachment without modifying the route declarations.
+//!
+//! The token is only ever held in memory as its SHA-256 hash; the configured
+//! plaintext is dropped right after the fairing is constructed.
 
+use anyhow::{bail, Result};
 use rocket::{
     fairing::{Fairing, Info, Kind},
     http::{uri::Origin, Method, Status},
     Data, Request, Route,
 };
+use sha2::{Digest, Sha256};
 use subtle::ConstantTimeEq;
 
+use crate::config::AdminConfig;
+
 const UNAUTH_URI: &str = "/__admin_unauthorized";
 const HEADER_NAME: &str = "X-Admin-Token";
 const QUERY_PARAM: &str = "token";
+const ENV_ADMIN_TOKEN: &str = "DSTACK_GATEWAY_ADMIN_TOKEN";
+const ENV_ADMIN_TOKEN_COMPAT: &str = "ADMIN_API_TOKEN";
 
 pub struct AdminAuthFairing {
-    /// `None` means auth is disabled (empty config); any request is allowed.
-    token: Option<String>,
+    /// SHA-256 of the configured token. `None` = auth disabled (insecure mode).
+    token_hash: Option<[u8; 32]>,
 }
 
 impl AdminAuthFairing {
-    pub fn new(token: String) -> Self {
+    /// Build a fairing from a resolved plaintext token. Empty disables auth.
+    pub fn new(token: &str) -> Self {
         Self {
-            token: (!token.is_empty()).then_some(token),
+            token_hash: (!token.is_empty()).then(|| sha256(token.as_bytes())),
         }
     }
 
+    /// Resolve a token from config + env, applying the auth policy:
+    ///   - `insecure_no_auth = true` → disabled (caller is expected to warn)
+    ///   - else require a non-empty token from `admin_token`,
+    ///     `DSTACK_GATEWAY_ADMIN_TOKEN`, or `ADMIN_API_TOKEN`.
+    pub fn from_config(config: &AdminConfig) -> Result<Self> {
+        if config.insecure_no_auth {
+            return Ok(Self { token_hash: None });
+        }
+        let token = if !config.admin_token.is_empty() {
+            config.admin_token.clone()
+        } else {
+            std::env::var(ENV_ADMIN_TOKEN)
+                .or_else(|_| std::env::var(ENV_ADMIN_TOKEN_COMPAT))
+                .unwrap_or_default()
+        };
+        let token = token.trim();
+        if token.is_empty() {
+            bail!(
+                "admin API is enabled but no admin_token is configured; \
+                 set core.admin.admin_token, {ENV_ADMIN_TOKEN}, or {ENV_ADMIN_TOKEN_COMPAT}, \
+                 or set core.admin.insecure_no_auth = true (testing only)"
+            );
+        }
+        Ok(Self::new(token))
+    }
+
     fn extract_token(req: &Request<'_>) -> Option<String> {
         if let Some(t) = req.headers().get_one(HEADER_NAME) {
             return Some(t.to_string());
@@ -63,6 +99,10 @@ impl AdminAuthFairing {
     }
 }
 
+fn sha256(bytes: &[u8]) -> [u8; 32] {
+    Sha256::digest(bytes).into()
+}
+
 /// Rebuild the request URI without the `token` query parameter, if present.
 /// Returns `None` when there is nothing to strip.
 fn strip_token_query(uri: &Origin<'_>) -> Option<Origin<'static>> {
@@ -99,15 +139,16 @@ impl Fairing for AdminAuthFairing {
     }
 
     async fn on_request(&self, req: &mut Request<'_>, _: &mut Data<'_>) {
-        let Some(expected) = self.token.as_deref() else {
+        let Some(expected_hash) = self.token_hash.as_ref() else {
             return;
         };
         // Avoid infinite re-routing if the fairing fires on the sentinel itself.
         if req.uri().path() == UNAUTH_URI {
             return;
         }
         let provided = Self::extract_token(req).unwrap_or_default();
-        let matches: bool = provided.as_bytes().ct_eq(expected.as_bytes()).into();
+        let provided_hash = sha256(provided.as_bytes());
+        let matches: bool = provided_hash.ct_eq(expected_hash).into();
         if !matches {
             if let Ok(origin) = Origin::parse_owned(UNAUTH_URI.to_string()) {
                 req.set_uri(origin);
@@ -198,7 +239,7 @@ mod tests {
 
     async fn make_client(token: &str) -> Client {
         let r = rocket::build()
-            .attach(AdminAuthFairing::new(token.to_string()))
+            .attach(AdminAuthFairing::new(token))
             .mount("/", routes())
             .mount("/", rocket::routes![protected_get, protected_post, echo]);
         Client::tracked(r).await.unwrap()
@@ -308,6 +349,86 @@ mod tests {
         assert_eq!(body, "token=<absent> other=keep");
     }
 
+    fn hash_of(fairing: &AdminAuthFairing) -> Option<[u8; 32]> {
+        fairing.token_hash
+    }
+
+    #[test]
+    fn from_config_disabled_when_insecure_flag_set() {
+        let cfg = AdminConfig {
+            enabled: true,
+            admin_token: String::new(),
+            insecure_no_auth: true,
+        };
+        let fairing = match AdminAuthFairing::from_config(&cfg) {
+            Ok(f) => f,
+            Err(e) => panic!("expected Ok, got err: {e}"),
+        };
+        assert!(hash_of(&fairing).is_none());
+    }
+
+    #[test]
+    fn from_config_uses_config_token() {
+        let cfg = AdminConfig {
+            enabled: true,
+            admin_token: "from-config".into(),
+            insecure_no_auth: false,
+        };
+        let fairing = match AdminAuthFairing::from_config(&cfg) {
+            Ok(f) => f,
+            Err(e) => panic!("expected Ok, got err: {e}"),
+        };
+        assert_eq!(hash_of(&fairing), Some(sha256(b"from-config")));
+    }
+
+    // Env-touching cases are combined into a single test so cargo's parallel
+    // runner doesn't race on `DSTACK_GATEWAY_ADMIN_TOKEN` / `ADMIN_API_TOKEN`.
+    #[test]
+    fn from_config_env_paths() {
+        let empty_cfg = AdminConfig {
+            enabled: true,
+            admin_token: String::new(),
+            insecure_no_auth: false,
+        };
+
+        // Baseline: no env, no config token → error.
+        unsafe {
+            std::env::remove_var(ENV_ADMIN_TOKEN);
+            std::env::remove_var(ENV_ADMIN_TOKEN_COMPAT);
+        }
+        let err = match AdminAuthFairing::from_config(&empty_cfg) {
+            Err(e) => e,
+            Ok(_) => panic!("expected error, got Ok"),
+        };
+        assert!(err.to_string().contains("no admin_token is configured"));
+
+        // Primary env var picked up.
+        unsafe {
+            std::env::set_var(ENV_ADMIN_TOKEN, "from-env");
+        }
+        let fairing = match AdminAuthFairing::from_config(&empty_cfg) {
+            Ok(f) => f,
+            Err(e) => panic!("expected Ok, got err: {e}"),
+        };
+        assert_eq!(hash_of(&fairing), Some(sha256(b"from-env")));
+        unsafe {
+            std::env::remove_var(ENV_ADMIN_TOKEN);
+        }
+
+        // Compat env var picked up when primary is absent.
+        unsafe {
+            std::env::set_var(ENV_ADMIN_TOKEN_COMPAT, "from-compat");
+        }
+        let fairing = match AdminAuthFairing::from_config(&empty_cfg) {
+            Ok(f) => f,
+            Err(e) => panic!("expected Ok, got err: {e}"),
+        };
+        assert_eq!(hash_of(&fairing), Some(sha256(b"from-compat")));
+        unsafe {
+            std::env::remove_var(ENV_ADMIN_TOKEN_COMPAT);
+        }
+    }
+
     #[rocket::async_test]
     async fn unauth_returns_401_on_all_methods() {
         let client = make_client("s3cret").await;