Merge remote-tracking branch 'ds/gateway-wavekv' into gateway-wavekv

Author: kvinwang

gateway/src/main.rs

@@ -73,13 +73,23 @@ async fn maybe_gen_certs(config: &Config, tls_config: &TlsConfig) -> Result<()>
         return Ok(());
     }
 
+    // Build alt_names: include rpc_domain and hostname from my_url
+    let mut alt_names = vec![config.rpc_domain.clone()];
+    if let Ok(url) = reqwest::Url::parse(&config.sync.my_url) {
+        if let Some(host) = url.host_str() {
+            if host != config.rpc_domain {
+                alt_names.push(host.to_string());
+            }
+        }
+    }
     if !config.debug.insecure_skip_attestation {
         info!("Using dstack guest agent for certificate generation");
         let agent_client = dstack_agent().context("Failed to create dstack client")?;
+
         let response = agent_client
             .get_tls_key(GetTlsKeyArgs {
                 subject: "dstack-gateway".to_string(),
-                alt_names: vec![config.rpc_domain.clone()],
+                alt_names,
                 usage_ra_tls: true,
                 usage_server_auth: true,
                 usage_client_auth: true,
@@ -114,7 +124,7 @@ async fn maybe_gen_certs(config: &Config, tls_config: &TlsConfig) -> Result<()>
     let cert = ra_tls::cert::CertRequest::builder()
         .key(&key)
         .subject("dstack-gateway")
-        .alt_names(std::slice::from_ref(&config.rpc_domain))
+        .alt_names(&alt_names)
         .usage_server_auth(true)
         .build()
         .self_signed()

gateway/src/main_service.rs

@@ -72,6 +72,8 @@ pub struct ProxyInner {
     kv_store: Arc<KvStore>,
     /// WaveKV sync service for network synchronization
     pub(crate) wavekv_sync: Option<Arc<WaveKvSyncService>>,
+    /// HTTPS client config for mTLS (used for bootnode peer discovery)
+    https_config: Option<HttpsClientConfig>,
 }
 
 #[derive(Debug, Serialize, Deserialize, Default)]
@@ -189,7 +191,7 @@ impl ProxyInner {
 
         // Create WaveKV sync service (only if sync is enabled)
         let wavekv_sync = if config.sync.enabled {
-            match WaveKvSyncService::new(&kv_store, &config.sync, https_config) {
+            match WaveKvSyncService::new(&kv_store, &config.sync, https_config.clone()) {
                 Ok(sync_service) => Some(Arc::new(sync_service)),
                 Err(err) => {
                     error!("Failed to create WaveKV sync service: {err}");
@@ -266,6 +268,7 @@ impl ProxyInner {
             certbot,
             kv_store,
             wavekv_sync,
+            https_config: Some(https_config),
         })
     }
 
@@ -289,6 +292,7 @@ impl Proxy {
         start_certbot_task(self.clone()).await;
         start_cert_store_watch_task(self.clone());
         start_zt_domain_watch_task(self.clone());
+        start_bootnode_discovery_task(self.clone());
         Ok(())
     }
 
@@ -583,6 +587,48 @@ fn start_zt_domain_watch_task(proxy: Proxy) {
     info!("ZT-Domain watch task started");
 }
 
+/// Periodically retry bootnode peer discovery if no peers are available
+fn start_bootnode_discovery_task(proxy: Proxy) {
+    if !proxy.config.sync.enabled || proxy.config.sync.bootnode.is_empty() {
+        return;
+    }
+
+    let bootnode = proxy.config.sync.bootnode.clone();
+    let node_id = proxy.config.sync.node_id;
+    let kv_store = proxy.kv_store.clone();
+    let https_config = match &proxy.https_config {
+        Some(config) => config.clone(),
+        None => return,
+    };
+
+    tokio::spawn(async move {
+        let mut interval = tokio::time::interval(Duration::from_secs(10));
+        loop {
+            interval.tick().await;
+            // Check if we already have peers
+            let n_peers = kv_store
+                .load_all_node_statuses()
+                .keys()
+                .filter(|&id| *id != node_id)
+                .count();
+            if n_peers > 0 {
+                info!("bootnode peer discovery finished, {n_peers} peers found");
+                break;
+            }
+            // Try to fetch peers from bootnode
+            debug!("retrying bootnode peer discovery...");
+            if let Err(err) =
+                fetch_peers_from_bootnode(&bootnode, &kv_store, node_id, &https_config).await
+            {
+                debug!("bootnode discovery retry failed: {err}");
+            } else {
+                info!("bootnode peer discovery succeeded");
+            }
+        }
+    });
+    info!("Bootnode discovery task started (will retry every 10s if no peers)");
+}
+
 async fn start_wavekv_sync_task(proxy: Proxy, wavekv_sync: Arc<WaveKvSyncService>) {
     if !proxy.config.sync.enabled {
         info!("WaveKV sync is disabled");

gateway/src/proxy/tls_passthough.rs

@@ -56,7 +56,7 @@ async fn resolve_app_address(prefix: &str, sni: &str, compat: bool) -> Result<Ap
             };
             return AppAddress::parse(data).context("failed to parse app address");
         }
-        anyhow::bail!("failed to resolve app address");
+        anyhow::bail!("failed to resolve legacy app address");
     } else {
         let lookup = resolver
             .txt_lookup(txt_domain)

gateway/test-run/.gitignore

@@ -1,2 +1,3 @@
 /run/
 .env
+/e2e/dstack-gateway

gateway/test-run/e2e/certs/gateway-ca.cert

@@ -4,16 +4,16 @@ address = "0.0.0.0"
 port = 9012
 
 [tls]
-key = "/etc/gateway/certs/gateway-rpc.key"
-certs = "/etc/gateway/certs/gateway-rpc.cert"
+key = "/var/lib/gateway/certs/gateway-rpc.key"
+certs = "/var/lib/gateway/certs/gateway-rpc.cert"
 
 [tls.mutual]
-ca_certs = "/etc/gateway/certs/gateway-ca.cert"
+ca_certs = "/var/lib/gateway/certs/gateway-ca.cert"
 mandatory = false
 
 [core]
 kms_url = ""
-rpc_domain = "gateway.test.local"
+rpc_domain = "gateway-1"
 
 [core.admin]
 enabled = true
@@ -22,7 +22,7 @@ address = "0.0.0.0"
 
 [core.debug]
 insecure_enable_debug_rpc = true
-insecure_skip_attestation = true
+insecure_skip_attestation = false
 port = 9015
 address = "0.0.0.0"
 

gateway/test-run/e2e/certs/gateway-ca.key

@@ -4,16 +4,16 @@ address = "0.0.0.0"
 port = 9012
 
 [tls]
-key = "/etc/gateway/certs/gateway-rpc.key"
-certs = "/etc/gateway/certs/gateway-rpc.cert"
+key = "/var/lib/gateway/certs/gateway-rpc.key"
+certs = "/var/lib/gateway/certs/gateway-rpc.cert"
 
 [tls.mutual]
-ca_certs = "/etc/gateway/certs/gateway-ca.cert"
+ca_certs = "/var/lib/gateway/certs/gateway-ca.cert"
 mandatory = false
 
 [core]
 kms_url = ""
-rpc_domain = "gateway.test.local"
+rpc_domain = "gateway-2"
 
 [core.admin]
 enabled = true
@@ -22,7 +22,7 @@ address = "0.0.0.0"
 
 [core.debug]
 insecure_enable_debug_rpc = true
-insecure_skip_attestation = true
+insecure_skip_attestation = false
 port = 9015
 address = "0.0.0.0"