feat(spider-storage)!: Read database credentials from the environment.

Storage read the database username and password from its config file, which the
Helm chart renders into a Kubernetes ConfigMap -- leaving the password in
plaintext. It now reads them from the `SPIDER_STORAGE_DB_USERNAME` and
`SPIDER_STORAGE_DB_PASSWORD` environment variables (both required) via a
dedicated `DatabaseCredentials` type, so they can instead be injected from a
Secret and kept out of the ConfigMap.

BREAKING CHANGE: the storage service now requires the database credentials to be
provided via environment variables; they are no longer read from the config file.

Author: 20001020ycx

components/spider-storage/src/bin/grpc_server.rs

@@ -12,6 +12,7 @@ use spider_proto_rust::storage::ResourceGroupManagementServiceServer;
 use spider_proto_rust::storage::SchedulerRegistrationServiceServer;
 use spider_proto_rust::storage::SessionManagementServiceServer;
 use spider_proto_rust::storage::TaskInstanceManagementServiceServer;
+use spider_storage::DatabaseCredentials;
 use spider_storage::ServerConfig;
 use spider_storage::grpc::GrpcServiceState;
 use spider_storage::state::runtime::create_runtime;
@@ -35,7 +36,8 @@ struct Cli {
 async fn main() -> Result<(), Box<dyn Error>> {
     let _log_guard = set_up_logging();
     let cli = Cli::parse();
-    let server_config = ServerConfig::from_yaml_file(&cli.config)?;
+    let mut server_config = ServerConfig::from_yaml_file(&cli.config)?;
+    server_config.runtime.db_config.credentials = DatabaseCredentials::from_env()?;
     let listen_addr = SocketAddr::new(server_config.host, server_config.port);
     let (runtime, cancellation_token) = create_runtime(&server_config.runtime).await?;
     let grpc_service =

components/spider-storage/src/config.rs

@@ -3,6 +3,7 @@ use std::net::IpAddr;
 use secrecy::SecretString;
 use serde::Deserialize;
 use serde::Serialize;
+use thiserror::Error;
 
 use crate::state::runtime::RuntimeConfig;
 
@@ -28,8 +29,108 @@ pub struct DatabaseConfig {
     pub host: String,
     pub port: u16,
     pub name: String,
+    pub max_connections: u32,
+
+    #[serde(skip)]
+    pub credentials: DatabaseCredentials,
+}
+
+/// Credentials for authenticating with the database.
+#[derive(Clone, Debug)]
+pub struct DatabaseCredentials {
     pub username: String,
-    #[serde(skip_serializing)]
     pub password: SecretString,
-    pub max_connections: u32,
+}
+
+impl DatabaseCredentials {
+    /// Reads the database credentials from the [`DB_USERNAME_ENV`] and [`DB_PASSWORD_ENV`]
+    /// environment variables.
+    ///
+    /// # Returns
+    ///
+    /// The credentials on success.
+    ///
+    /// # Errors
+    ///
+    /// Returns an error if:
+    ///
+    /// * [`CredentialsError::MissingEnvVar`] if either environment variable is unset.
+    pub fn from_env() -> Result<Self, CredentialsError> {
+        let username = std::env::var(DB_USERNAME_ENV)
+            .map_err(|_| CredentialsError::MissingEnvVar(DB_USERNAME_ENV))?;
+        let password = std::env::var(DB_PASSWORD_ENV)
+            .map_err(|_| CredentialsError::MissingEnvVar(DB_PASSWORD_ENV))?;
+        Ok(Self {
+            username,
+            password: SecretString::from(password),
+        })
+    }
+}
+
+impl Default for DatabaseCredentials {
+    fn default() -> Self {
+        Self {
+            username: String::new(),
+            password: SecretString::from(String::new()),
+        }
+    }
+}
+
+/// An error returned while reading database credentials from the environment.
+#[derive(Debug, Error)]
+pub enum CredentialsError {
+    #[error("required environment variable `{0}` is not set")]
+    MissingEnvVar(&'static str),
+}
+
+/// Environment variable that supplies the database username.
+const DB_USERNAME_ENV: &str = "SPIDER_STORAGE_DB_USERNAME";
+
+/// Environment variable that supplies the database password.
+const DB_PASSWORD_ENV: &str = "SPIDER_STORAGE_DB_PASSWORD";
+
+#[cfg(test)]
+mod tests {
+    use secrecy::ExposeSecret;
+    use serial_test::serial;
+
+    use super::*;
+
+    #[test]
+    #[serial]
+    fn from_env_reads_credentials() -> anyhow::Result<()> {
+        // SAFETY: `#[serial]` prevents other tests from observing these process-global mutations.
+        unsafe {
+            std::env::set_var(DB_USERNAME_ENV, "env-user");
+            std::env::set_var(DB_PASSWORD_ENV, "env-pass");
+        }
+        let credentials = DatabaseCredentials::from_env();
+        unsafe {
+            std::env::remove_var(DB_USERNAME_ENV);
+            std::env::remove_var(DB_PASSWORD_ENV);
+        }
+
+        let credentials = credentials?;
+        assert_eq!(credentials.username, "env-user");
+        assert_eq!(credentials.password.expose_secret(), "env-pass");
+        Ok(())
+    }
+
+    #[test]
+    #[serial]
+    fn from_env_errors_when_a_variable_is_unset() -> anyhow::Result<()> {
+        // SAFETY: `#[serial]` prevents other tests from observing these process-global mutations.
+        unsafe {
+            std::env::set_var(DB_USERNAME_ENV, "env-user");
+            std::env::remove_var(DB_PASSWORD_ENV);
+        }
+        let result = DatabaseCredentials::from_env();
+        unsafe { std::env::remove_var(DB_USERNAME_ENV) };
+
+        assert!(matches!(
+            result,
+            Err(CredentialsError::MissingEnvVar(var)) if var == DB_PASSWORD_ENV
+        ));
+        Ok(())
+    }
 }

components/spider-storage/src/db/mariadb.rs

@@ -65,8 +65,8 @@ impl MariaDbStorageConnector {
             .host(&config.host)
             .port(config.port)
             .database(&config.name)
-            .username(&config.username)
-            .password(config.password.expose_secret());
+            .username(&config.credentials.username)
+            .password(config.credentials.password.expose_secret());
 
         let pool = sqlx::mysql::MySqlPoolOptions::new()
             .max_connections(config.max_connections)

components/spider-storage/src/lib.rs

@@ -7,5 +7,7 @@ pub mod ready_queue;
 pub mod state;
 pub mod task_instance_pool;
 
+pub use config::CredentialsError;
 pub use config::DatabaseConfig;
+pub use config::DatabaseCredentials;
 pub use config::ServerConfig;

components/spider-storage/tests/mariadb_infra.rs

@@ -1,6 +1,7 @@
 use secrecy::SecretString;
 use spider_core::types::id::ResourceGroupId;
 use spider_storage::DatabaseConfig;
+use spider_storage::DatabaseCredentials;
 use spider_storage::db::MariaDbStorageConnector;
 use spider_storage::db::ResourceGroupManagement;
 
@@ -47,9 +48,11 @@ pub fn create_mariadb_config() -> DatabaseConfig {
         host: "localhost".to_string(),
         port,
         name: database,
-        username,
-        password: SecretString::from(password),
         max_connections: 5,
+        credentials: DatabaseCredentials {
+            username,
+            password: SecretString::from(password),
+        },
     }
 }