Merge pull request #412 from Dstack-TEE/gateway-wavekv

gateway: Add support for multiple nodes using the same domain

Author: kvinwang

Cargo.lock

@@ -86,9 +86,11 @@ serde-duration = { path = "serde-duration" }
 dstack-mr = { path = "dstack-mr" }
 dstack-verifier = { path = "verifier", default-features = false }
 size-parser = { path = "size-parser" }
+wavekv = "1.0.0"
 
 # Core dependencies
 anyhow = { version = "1.0.97", default-features = false }
+arc-swap = "1"
 errify = { version = "0.3.0", features = ["anyhow"] }
 or-panic = { version = "1.0", default-features = false }
 chrono = "0.4.40"
@@ -111,6 +113,7 @@ listenfd = "1.0"
 jemallocator = "0.5.4"
 
 # Serialization/Parsing
+flate2 = "1.1"
 borsh = { version = "1.5.7", default-features = false, features = ["derive"] }
 bon = { version = "3.4.0", default-features = false }
 base64 = "0.22.1"
@@ -124,9 +127,9 @@ scale = { version = "3.7.4", package = "parity-scale-codec", features = [
 ] }
 serde = { version = "1.0.228", features = ["derive"], default-features = false }
 serde-human-bytes = "0.1.2"
+rmp-serde = "1.3.1"
 serde_json = { version = "1.0.140", default-features = false }
 serde_ini = "0.2.0"
-rmp-serde = "1.3.1"
 toml = "0.8.20"
 toml_edit = { version = "0.22.24", features = ["serde"] }
 yasna = "0.5.2"
@@ -148,6 +151,11 @@ hyper-util = { version = "0.1.10", features = [
     "client-legacy",
     "http1",
 ] }
+hyper-rustls = { version = "0.27", default-features = false, features = [
+    "ring",
+    "http1",
+    "tls12",
+] }
 hyperlocal = "0.9.1"
 ipnet = { version = "2.11.0", features = ["serde"] }
 reqwest = { version = "0.12.14", default-features = false, features = [
@@ -237,7 +245,6 @@ yaml-rust2 = "0.10.4"
 
 luks2 = "0.5.0"
 scopeguard = "1.2.0"
-flate2 = "1.1"
 tar = "0.4"
 
 [profile.release]

Cargo.toml

@@ -186,3 +186,12 @@ SPDX-License-Identifier = "CC0-1.0"
 path = "guest-agent/fixtures/*"
 SPDX-FileCopyrightText = "NONE"
 SPDX-License-Identifier = "CC0-1.0"
+
+[[annotations]]
+path = [
+    "gateway/test-run/e2e/certs/*",
+    "gateway/test-run/e2e/configs/*",
+    "gateway/test-run/e2e/pebble-config.json",
+]
+SPDX-FileCopyrightText = "NONE"
+SPDX-License-Identifier = "CC0-1.0"

REUSE.toml

@@ -12,9 +12,12 @@ license.workspace = true
 [dependencies]
 anyhow.workspace = true
 bon.workspace = true
+bytes.workspace = true
 enum_dispatch.workspace = true
 fs-err.workspace = true
 hickory-resolver.workspace = true
+http.workspace = true
+http-body-util.workspace = true
 instant-acme.workspace = true
 path-absolutize.workspace = true
 rcgen.workspace = true

certbot/Cargo.toml

@@ -164,7 +164,7 @@ async fn main() -> Result<()> {
     {
         use tracing_subscriber::{fmt, EnvFilter};
         let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
-        fmt().with_env_filter(filter).init();
+        fmt().with_env_filter(filter).with_ansi(false).init();
     }
     rustls::crypto::ring::default_provider()
         .install_default()

certbot/cli/src/main.rs

@@ -21,6 +21,7 @@ use tracing::{debug, error, info};
 use x509_parser::prelude::{GeneralName, Pem};
 
 use super::dns01_client::{Dns01Api, Dns01Client};
+use super::http_client::ReqwestHttpClient;
 
 /// A AcmeClient instance.
 pub struct AcmeClient {
@@ -63,7 +64,9 @@ impl AcmeClient {
         dns_txt_ttl: u32,
     ) -> Result<Self> {
         let credentials: Credentials = serde_json::from_str(encoded_credentials)?;
-        let account = Account::from_credentials(credentials.credentials).await?;
+        let http_client = Box::new(ReqwestHttpClient::new()?);
+        let account =
+            Account::from_credentials_and_http(credentials.credentials, http_client).await?;
         let credentials: Credentials = serde_json::from_str(encoded_credentials)?;
         Ok(Self {
             account,
@@ -81,14 +84,16 @@ impl AcmeClient {
         max_dns_wait: Duration,
         dns_txt_ttl: u32,
     ) -> Result<Self> {
-        let (account, credentials) = Account::create(
+        let http_client = Box::new(ReqwestHttpClient::new()?);
+        let (account, credentials) = Account::create_with_http(
             &NewAccount {
                 contact: &[],
                 terms_of_service_agreed: true,
                 only_return_existing: false,
             },
             acme_url,
             None,
+            http_client,
         )
         .await
         .with_context(|| format!("failed to create ACME account for {acme_url}"))?;

certbot/src/acme_client.rs

@@ -10,6 +10,7 @@ async fn new_acme_client() -> Result<AcmeClient> {
     let dns01_client = Dns01Client::new_cloudflare(
         std::env::var("CLOUDFLARE_ZONE_ID").expect("CLOUDFLARE_ZONE_ID not set"),
         std::env::var("CLOUDFLARE_API_TOKEN").expect("CLOUDFLARE_API_TOKEN not set"),
+        std::env::var("CLOUDFLARE_API_URL").ok(),
     );
     let credentials =
         std::env::var("LETSENCRYPT_CREDENTIAL").expect("LETSENCRYPT_CREDENTIAL not set");

certbot/src/acme_client/tests.rs

@@ -28,6 +28,7 @@ pub struct CertBotConfig {
     credentials_file: PathBuf,
     auto_create_account: bool,
     cf_api_token: String,
+    cf_api_url: Option<String>,
     cert_file: PathBuf,
     key_file: PathBuf,
     cert_dir: PathBuf,
@@ -94,8 +95,12 @@ impl CertBot {
             .trim_start_matches("*.")
             .trim_end_matches('.')
             .to_string();
-        let dns01_client =
-            Dns01Client::new_cloudflare(config.cf_api_token.clone(), base_domain).await?;
+        let dns01_client = Dns01Client::new_cloudflare(
+            base_domain,
+            config.cf_api_token.clone(),
+            config.cf_api_url.clone(),
+        )
+        .await?;
         let acme_client = match fs::read_to_string(&config.credentials_file) {
             Ok(credentials) => {
                 if acme_matches(&credentials, &config.acme_url) {

certbot/src/bot.rs

@@ -72,9 +72,12 @@ pub enum Dns01Client {
 }
 
 impl Dns01Client {
-    pub async fn new_cloudflare(api_token: String, base_domain: String) -> Result<Self> {
-        Ok(Self::Cloudflare(
-            CloudflareClient::new(api_token, base_domain).await?,
-        ))
+    pub async fn new_cloudflare(
+        base_domain: String,
+        api_token: String,
+        api_url: Option<String>,
+    ) -> Result<Self> {
+        let client = CloudflareClient::new(base_domain, api_token, api_url).await?;
+        Ok(Self::Cloudflare(client))
     }
 }

certbot/src/dns01_client.rs

@@ -14,12 +14,18 @@ use crate::dns01_client::Record;
 
 use super::Dns01Api;
 
-const CLOUDFLARE_API_URL: &str = "https://api.cloudflare.com/client/v4";
+const DEFAULT_CLOUDFLARE_API_URL: &str = "https://api.cloudflare.com/client/v4";
 
 #[derive(Debug, Serialize, Deserialize)]
 pub struct CloudflareClient {
     zone_id: String,
     api_token: String,
+    #[serde(default = "default_api_url")]
+    api_url: String,
+}
+
+fn default_api_url() -> String {
+    DEFAULT_CLOUDFLARE_API_URL.to_string()
 }
 
 #[derive(Deserialize)]
@@ -59,20 +65,29 @@ struct ZonesResultInfo {
 }
 
 impl CloudflareClient {
-    pub async fn new(api_token: String, base_domain: String) -> Result<Self> {
-        let zone_id = Self::resolve_zone_id(&api_token, &base_domain).await?;
-        Ok(Self { api_token, zone_id })
+    pub async fn new(
+        base_domain: String,
+        api_token: String,
+        api_url: Option<String>,
+    ) -> Result<Self> {
+        let api_url = api_url.unwrap_or_else(|| DEFAULT_CLOUDFLARE_API_URL.to_string());
+        let zone_id = Self::resolve_zone_id(&api_token, &base_domain, &api_url).await?;
+        Ok(Self {
+            zone_id,
+            api_token,
+            api_url,
+        })
     }
 
-    async fn resolve_zone_id(api_token: &str, base_domain: &str) -> Result<String> {
+    async fn resolve_zone_id(api_token: &str, base_domain: &str, api_url: &str) -> Result<String> {
         let base = base_domain
             .trim()
             .trim_start_matches("*.")
             .trim_end_matches('.')
             .to_lowercase();
 
         let client = Client::new();
-        let url = format!("{CLOUDFLARE_API_URL}/zones");
+        let url = format!("{api_url}/zones");
 
         let per_page = 50u32;
         let mut page = 1u32;
@@ -150,8 +165,7 @@ impl CloudflareClient {
 
     async fn add_record(&self, record: &impl Serialize) -> Result<Response> {
         let client = Client::new();
-        let url = format!("{CLOUDFLARE_API_URL}/zones/{}/dns_records", self.zone_id);
-
+        let url = format!("{}/zones/{}/dns_records", self.api_url, self.zone_id);
         let response = client
             .post(&url)
             .header("Authorization", format!("Bearer {}", self.api_token))
@@ -176,8 +190,8 @@ impl CloudflareClient {
     async fn remove_record_inner(&self, record_id: &str) -> Result<()> {
         let client = Client::new();
         let url = format!(
-            "{CLOUDFLARE_API_URL}/zones/{zone_id}/dns_records/{record_id}",
-            zone_id = self.zone_id
+            "{}/zones/{}/dns_records/{}",
+            self.api_url, self.zone_id, record_id
         );
 
         debug!(url = %url, "cloudflare remove_record request");
@@ -201,7 +215,7 @@ impl CloudflareClient {
 
     async fn get_records_inner(&self, domain: &str) -> Result<Vec<Record>> {
         let client = Client::new();
-        let url = format!("{CLOUDFLARE_API_URL}/zones/{}/dns_records", self.zone_id);
+        let url = format!("{}/zones/{}/dns_records", self.api_url, self.zone_id);
 
         let per_page = 100u32;
         let mut records = Vec::new();
@@ -338,8 +352,9 @@ mod tests {
 
     async fn create_client() -> CloudflareClient {
         CloudflareClient::new(
-            std::env::var("CLOUDFLARE_API_TOKEN").expect("CLOUDFLARE_API_TOKEN not set"),
             std::env::var("TEST_DOMAIN").expect("TEST_DOMAIN not set"),
+            std::env::var("CLOUDFLARE_API_TOKEN").expect("CLOUDFLARE_API_TOKEN not set"),
+            std::env::var("CLOUDFLARE_API_URL").ok(),
         )
         .await
         .unwrap()